fix:查询存在的数据增加distinct去重

service/deduplication.go

@@ -20,7 +20,6 @@ type DeduplicationService struct{}
 
 var PREFIX = "qc"
 
-
 //数据判重
 func (service *DeduplicationService) DataDeduplicateInsert(data *deduplication.Request) (*deduplication.Info, string) {
 	log.Println("开始=====")
@@ -52,7 +51,7 @@ func (service *DeduplicationService) DataDeduplicateInsert(data *deduplication.R
 	log.Println(selectSql)
 	infoIdList := strings.Split(data.InfoId, ",")
 	totalInfoCount := len(infoIdList)
-	err := orm.Table(tableName).Cols("info_id").Where(selectSql, valueList...).Find(&rs)
+	err := orm.Table(tableName).Distinct("info_id").Where(selectSql, valueList...).Find(&rs)
 	totalExist := len(rs)
 	log.Println(totalExist, "已存在")
 	if err != nil {
@@ -152,14 +151,14 @@ func (service *DeduplicationService) DataDeduplicateByAccountId(data *deduplicat
 	// 模运算取企业id  todo  需要区分是int类型还是mongodb objectid类型   看一下咋转
 	//var num01 int
 	number, errConv := strconv.Atoi(data.AccountId)
-	log.Println(55555,number,errConv)
-	if errConv!=nil{
+	log.Println(55555, number, errConv)
+	if errConv != nil {
 		log.Println("不是int类型的，hash后再取模寻表")
 		b := a(data.AccountId)
-		ss:=b[len(b)-2:]
-		bt,_:=strconv.ParseInt(ss,16,64)
-		number =int(bt)
-		log.Println("哈希取模后的表序号",number)
+		ss := b[len(b)-2:]
+		bt, _ := strconv.ParseInt(ss, 16, 64)
+		number = int(bt)
+		log.Println("哈希取模后的表序号", number)
 	}
 	tableName := PREFIX + fmt.Sprintf("%03d", number%100)
 
@@ -171,7 +170,7 @@ func (service *DeduplicationService) DataDeduplicateByAccountId(data *deduplicat
 	valueList = append(valueList, data.AccountId)
 	valueList = append(valueList, data.DataDesc)
 	for _, v := range strings.Split(data.InfoId, ",") {
-		if strings.TrimSpace(v)!=""{
+		if strings.TrimSpace(v) != "" {
 			tmpList = append(tmpList, "?")
 			valueList = append(valueList, v)
 		}
@@ -181,7 +180,7 @@ func (service *DeduplicationService) DataDeduplicateByAccountId(data *deduplicat
 	selectSql = fmt.Sprintf("account_id=? and data_desc=? and info_id in (%s)", strings.Join(tmpList, ","))
 	infoIdList := strings.Split(data.InfoId, ",")
 	totalInfoCount := len(infoIdList)
-	err := orm.Table(tableName).Cols("info_id").Where(selectSql, valueList...).Find(&rs)
+	err := orm.Table(tableName).Distinct("info_id").Where(selectSql, valueList...).Find(&rs)
 
 	if err != nil {
 		log.Println(err, "判重查询失败")
@@ -197,8 +196,8 @@ func (service *DeduplicationService) DataDeduplicateByAccountId(data *deduplicat
 	for _, v := range rs {
 		if existInfoIdMap[v.InfoId] {
 			continue
-		}else {
-			existIdList = append(existIdList,v.InfoId)
+		} else {
+			existIdList = append(existIdList, v.InfoId)
 		}
 	}
 	count := int64(len(existIdList))
@@ -209,21 +208,22 @@ func (service *DeduplicationService) DataDeduplicateByAccountId(data *deduplicat
 		IsInsert:   false,
 	}, ""
 }
+
 // 根据账户id进行判重并存入数据
 func (service *DeduplicationService) DataDeduplicateAndSave(data *deduplication.ByAccountRequest) (*deduplication.Info, string) {
-	log.Println("开始=====",time.Now())
+	log.Println("开始=====", time.Now())
 	orm := Engine.NewSession()
 	defer orm.Close()
 	// 模运算取企业id  企业id是int 类型的直接对100取模
 	// objectid类型的哈希后取模  这里使用md5后取后两位数字转10进制 然后对100取模
 	number, errConv := strconv.Atoi(data.AccountId)
-	log.Println(55555,number,errConv)
-	if errConv!=nil{
+	log.Println(55555, number, errConv)
+	if errConv != nil {
 		log.Println("不是int类型的，hash后再取模寻表")
 		b := a(data.AccountId)
-		ss:=b[len(b)-2:]
-		bt,_:=strconv.ParseInt(ss,16,64)
-		number =int(bt)
+		ss := b[len(b)-2:]
+		bt, _ := strconv.ParseInt(ss, 16, 64)
+		number = int(bt)
 		log.Println(bt)
 	}
 	tableName := PREFIX + fmt.Sprintf("%03d", number%100)
@@ -235,7 +235,7 @@ func (service *DeduplicationService) DataDeduplicateAndSave(data *deduplication.
 	valueList = append(valueList, data.AccountId)
 	valueList = append(valueList, data.DataDesc)
 	for _, v := range strings.Split(data.InfoId, ",") {
-		if strings.TrimSpace(v)!=""{
+		if strings.TrimSpace(v) != "" {
 			tmpList = append(tmpList, "?")
 			valueList = append(valueList, v)
 		}
@@ -245,7 +245,7 @@ func (service *DeduplicationService) DataDeduplicateAndSave(data *deduplication.
 	selectSql = fmt.Sprintf("account_id=? and data_desc=? and info_id in (%s)", strings.Join(tmpList, ","))
 	infoIdList := strings.Split(data.InfoId, ",")
 	totalInfoCount := len(infoIdList)
-	err := orm.Table(tableName).Cols("info_id").Where(selectSql, valueList...).Find(&rs)
+	err := orm.Table(tableName).Distinct("info_id").Where(selectSql, valueList...).Find(&rs)
 	//existIdList := []string{}
 
 	if err != nil {
@@ -257,7 +257,6 @@ func (service *DeduplicationService) DataDeduplicateAndSave(data *deduplication.
 		}, "判重查询失败"
 	}
 
-
 	existIdMap := map[string]bool{}
 	for _, v := range rs {
 		existIdMap[v.InfoId] = true
@@ -275,19 +274,20 @@ func (service *DeduplicationService) DataDeduplicateAndSave(data *deduplication.
 		}
 		log.Println("新增", id)
 		temData := entity.Deduplication{
-			InfoId:   id,
-			EntId:    "0",
-			PersonId: data.PersonId,
+			InfoId:    id,
+			EntId:     "0",
+			PersonId:  data.PersonId,
 			AccountId: data.AccountId,
-			DataDesc: data.DataDesc,
+			DataDesc:  data.DataDesc,
 		}
+		existIdMap[id] = true
 		insertList = append(insertList, temData)
 
 	}
-	log.Println(totalInfoCount,count,"88888888")
+	//log.Println(totalInfoCount, count, "88888888")
 	log.Println(len(insertList))
-	go SaveMysql(tableName,insertList)
-	log.Println("结束=====",time.Now())
+	go SaveMysql(tableName, insertList)
+	log.Println("结束=====", time.Now())
 
 	return &deduplication.Info{
 		TotalCount: int64(totalInfoCount),
@@ -295,8 +295,6 @@ func (service *DeduplicationService) DataDeduplicateAndSave(data *deduplication.
 		NewCount:   int64(totalInfoCount - count),
 	}, ""
 
-
-
 }
 func (service *DeduplicationService) EntCount(data *deduplication.GetEntCountRequest) (int64, string) {
 	log.Println("开始=====")
@@ -321,7 +319,7 @@ func a(data string) string {
 	return fmt.Sprintf("%x", t.Sum(nil))
 }
 
-func SaveMysql(tableName string,saveList []entity.Deduplication) {
+func SaveMysql(tableName string, saveList []entity.Deduplication) {
 	//
 	log.Println("保存数据开始")
 	orm := Engine.NewSession()
@@ -347,10 +345,9 @@ func SaveMysql(tableName string,saveList []entity.Deduplication) {
 
 		}
 
-
 	}
 	err2 := orm.Commit()
 	log.Println("err2", err2)
 	log.Println("保存数据结束")
 
-}
+}