协程导入数据

CMPlatform/service/data_ent.go

@@ -16,6 +16,7 @@ import (
 	"io"
 	"mime/multipart"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -99,6 +100,8 @@ func ImportDataByExcel(mf multipart.File, eid int64, entid string) (int, int64,
 	fMap := make(map[int]string) //字段位置
 	sheet := xls.Sheets[0]
 	rows := sheet.Rows
+	ch := make(chan bool, 10) // 协程处理
+	wg := &sync.WaitGroup{}
 	for rn, row := range rows {
 		if rn == 0 {
 			for index, cell := range row.Cells {
@@ -111,113 +114,120 @@ func ImportDataByExcel(mf multipart.File, eid int64, entid string) (int, int64,
 			log.Debug(fMap)
 			continue
 		}
-		tmp := map[string]interface{}{}
-
-		AreaM, IndustryM, WtM := CodeFieldFuc(eid)
-
-		for index, cell := range row.Cells {
-			if val := cell.Value; val != "" {
-				if fMap[index] == "infoid" {
-					if mongodb.IsObjectIdHex(val) {
-						tmp["infoid"] = val
-					} else {
-						tmp["infoid"] = push.SE.DecodeString(val) //解密后id
-					}
-				} else if fMap[index] == "region" {
-					// 业务地区
-					if v := AreaM[val]; v != nil && v != 0 {
-						tmp[fMap[index]] = v
-					} else {
-						id := util.EntMysql.Insert("d_yys_analyze_dimensions", bson.M{"ent_id": eid, "name": val,
-							"type": 1})
-						tmp[fMap[index]] = id
-						info := util.EntMysql.FindOne("customer_data_yys_permission_elements", bson.M{"ent_id": eid, "element_field": "region", "pcode": "$isNull"}, "", "")
-						if info != nil && len(*info) > 0 {
-							util.EntMysql.Insert("customer_data_yys_permission_elements", bson.M{"name": val, "ent_id": eid,
-								"element_field": "region", "pcode": common.IntAll((*info)["code"]), "element_value": id})
+		ch <- true
+		wg.Add(1)
+		go func(cells []*xlsx.Cell) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			tmp := map[string]interface{}{}
+			AreaM, IndustryM, WtM := CodeFieldFuc(eid)
+			for index, cell := range row.Cells {
+				if val := cell.Value; val != "" {
+					if fMap[index] == "infoid" {
+						if mongodb.IsObjectIdHex(val) {
+							tmp["infoid"] = val
+						} else {
+							tmp["infoid"] = push.SE.DecodeString(val) //解密后id
 						}
-					}
-				} else if fMap[index] == "industry" {
-					// 行业
-					if v := IndustryM[val]; v != nil && v != 0 {
-						tmp[fMap[index]] = v
-					} else {
-						id := util.EntMysql.Insert("d_yys_analyze_dimensions", bson.M{"ent_id": eid, "name": val,
-							"type": 2})
-						tmp[fMap[index]] = id
-						info := util.EntMysql.FindOne("customer_data_yys_permission_elements", bson.M{"ent_id": eid, "element_field": "industry", "pcode": "$isNull"}, "", "")
-						if info != nil && len(*info) > 0 {
-							util.EntMysql.Insert("customer_data_yys_permission_elements", bson.M{"name": val, "ent_id": eid,
-								"element_field": "industry", "pcode": common.IntAll((*info)["code"]), "element_value": id})
+					} else if fMap[index] == "region" {
+						// 业务地区
+						if v := AreaM[val]; v != nil && v != 0 {
+							tmp[fMap[index]] = v
+						} else {
+							id := util.EntMysql.Insert("d_yys_analyze_dimensions", bson.M{"ent_id": eid, "name": val,
+								"type": 1})
+							tmp[fMap[index]] = id
+							info := util.EntMysql.FindOne("customer_data_yys_permission_elements", bson.M{"ent_id": eid, "element_field": "region", "pcode": "$isNull"}, "", "")
+							if info != nil && len(*info) > 0 {
+								util.EntMysql.Insert("customer_data_yys_permission_elements", bson.M{"name": val, "ent_id": eid,
+									"element_field": "region", "pcode": common.IntAll((*info)["code"]), "element_value": id})
+							}
 						}
-					}
-				} else if fMap[index] == "winner_tag" {
-					// 中标单位标签
-					if v := WtM[val]; v != nil && v != 0 {
-						tmp[fMap[index]] = v
-					} else {
-						id := util.EntMysql.Insert("d_yys_analyze_dimensions", bson.M{"ent_id": eid, "name": val,
-							"type": 3})
-						tmp[fMap[index]] = id
-						info := util.EntMysql.FindOne("customer_data_yys_permission_elements", bson.M{"ent_id": eid, "element_field": "winner_tag", "pcode": "$isNull"}, "", "")
-						if info != nil && len(*info) > 0 {
-							util.EntMysql.Insert("customer_data_yys_permission_elements", bson.M{"name": val, "ent_id": eid,
-								"element_field": "winner_tag", "pcode": common.IntAll((*info)["code"]), "element_value": id})
+					} else if fMap[index] == "industry" {
+						// 行业
+						if v := IndustryM[val]; v != nil && v != 0 {
+							tmp[fMap[index]] = v
+						} else {
+							id := util.EntMysql.Insert("d_yys_analyze_dimensions", bson.M{"ent_id": eid, "name": val,
+								"type": 2})
+							tmp[fMap[index]] = id
+							info := util.EntMysql.FindOne("customer_data_yys_permission_elements", bson.M{"ent_id": eid, "element_field": "industry", "pcode": "$isNull"}, "", "")
+							if info != nil && len(*info) > 0 {
+								util.EntMysql.Insert("customer_data_yys_permission_elements", bson.M{"name": val, "ent_id": eid,
+									"element_field": "industry", "pcode": common.IntAll((*info)["code"]), "element_value": id})
+							}
 						}
-					}
-				} else if fMap[index] == "bidopentime" || fMap[index] == "publishtime" || fMap[index] == "docstarttime" || fMap[index] == "bidstarttime" || fMap[index] == "bidendtime" {
-					if cell.Type() == xlsx.CellTypeNumeric {
-						date, err := cell.GetTime(false)
-						if err != nil {
-							log.Debug("Error getting date:", err)
+					} else if fMap[index] == "winner_tag" {
+						// 中标单位标签
+						if v := WtM[val]; v != nil && v != 0 {
+							tmp[fMap[index]] = v
 						} else {
-							tmp[fMap[index]] = date.Format(time.DateOnly)
+							id := util.EntMysql.Insert("d_yys_analyze_dimensions", bson.M{"ent_id": eid, "name": val,
+								"type": 3})
+							tmp[fMap[index]] = id
+							info := util.EntMysql.FindOne("customer_data_yys_permission_elements", bson.M{"ent_id": eid, "element_field": "winner_tag", "pcode": "$isNull"}, "", "")
+							if info != nil && len(*info) > 0 {
+								util.EntMysql.Insert("customer_data_yys_permission_elements", bson.M{"name": val, "ent_id": eid,
+									"element_field": "winner_tag", "pcode": common.IntAll((*info)["code"]), "element_value": id})
+							}
+						}
+					} else if fMap[index] == "bidopentime" || fMap[index] == "publishtime" || fMap[index] == "docstarttime" || fMap[index] == "bidstarttime" || fMap[index] == "bidendtime" {
+						if cell.Type() == xlsx.CellTypeNumeric {
+							date, err := cell.GetTime(false)
+							if err != nil {
+								log.Debug("Error getting date:", err)
+							} else {
+								tmp[fMap[index]] = date.Format(time.DateOnly)
+							}
+						} else {
+							tmp[fMap[index]] = val
 						}
-					} else {
-						tmp[fMap[index]] = val
-					}
 
-				} else {
-					if fMap[index] != "" {
-						tmp[fMap[index]] = val
+					} else {
+						if fMap[index] != "" {
+							tmp[fMap[index]] = val
+						}
 					}
 				}
 			}
-		}
-		if tmp["infoid"] == nil {
-			return 0, now.Unix(), "文档没有唯一标识字段"
-		}
-		importDataNum++
-		if common.ObjToString(tmp["href"]) == "#" || strings.Contains(common.ObjToString(tmp["href"]), "jianyu360.cn") {
-			tmp["isCompetitors"] = "是"
-		} else {
-			tmp["isCompetitors"] = "否"
-		}
-		tmp["eid"] = eid
-		tmp["entid"] = entid
-		tmp["jybxhref"] = "https://www.jianyu360.cn/succbi/yyssjdz/app/sjdz.app/detail.spg?E_id=" + common.ObjToString(tmp["infoid"])
-		tmp["pici"] = now.Format(date.Date_Full_Layout)
-		if info := util.EntMysql.FindOne("customer_data", bson.M{"entid": entid, "infoid": tmp["infoid"]}, "", ""); info != nil && len(*info) > 0 {
-			tmp["status"] = 1
-			util.EntMysql.Update("customer_data", bson.M{"entid": entid, "infoid": tmp["infoid"]}, tmp)
-			id := (*info)["id"]
-			infoid := (*info)["infoid"]
-			tmp["id"] = id
-			//util.EsCus.DelById("bidding_customer", "", fmt.Sprint(id))
-			binfo := util.EntMysql.FindOne("customer_data_yys_gl", bson.M{"info_id": infoid}, "", "")
-			if binfo != nil && len(*binfo) > 0 {
-				esSaveFuc(tmp, common.ObjToString((*binfo)["detail"]), &saveEsArr)
+			if tmp["infoid"] == nil {
+				return
+			}
+			importDataNum++
+			if common.ObjToString(tmp["href"]) == "#" || strings.Contains(common.ObjToString(tmp["href"]), "jianyu360.cn") {
+				tmp["isCompetitors"] = "是"
 			} else {
-				esSaveFuc(tmp, "", &saveEsArr)
+				tmp["isCompetitors"] = "否"
 			}
-		} else {
-			log.Debug(tmp)
-			id := util.EntMysql.Insert("customer_data", tmp)
-			tmp["id"] = id
-			detail := detailFuc(common.ObjToString(tmp["infoid"]))
-			esSaveFuc(tmp, detail, &saveEsArr)
-		}
+			tmp["eid"] = eid
+			tmp["entid"] = entid
+			tmp["jybxhref"] = "https://www.jianyu360.cn/succbi/yyssjdz/app/sjdz.app/detail.spg?E_id=" + common.ObjToString(tmp["infoid"])
+			tmp["pici"] = now.Format(date.Date_Full_Layout)
+			if info := util.EntMysql.FindOne("customer_data", bson.M{"entid": entid, "infoid": tmp["infoid"]}, "", ""); info != nil && len(*info) > 0 {
+				tmp["status"] = 1
+				util.EntMysql.Update("customer_data", bson.M{"entid": entid, "infoid": tmp["infoid"]}, tmp)
+				id := (*info)["id"]
+				infoid := (*info)["infoid"]
+				tmp["id"] = id
+				//util.EsCus.DelById("bidding_customer", "", fmt.Sprint(id))
+				binfo := util.EntMysql.FindOne("customer_data_yys_gl", bson.M{"info_id": infoid}, "", "")
+				if binfo != nil && len(*binfo) > 0 {
+					esSaveFuc(tmp, common.ObjToString((*binfo)["detail"]), &saveEsArr)
+				} else {
+					esSaveFuc(tmp, "", &saveEsArr)
+				}
+			} else {
+				log.Debug(tmp)
+				id := util.EntMysql.Insert("customer_data", tmp)
+				tmp["id"] = id
+				detail := detailFuc(common.ObjToString(tmp["infoid"]))
+				esSaveFuc(tmp, detail, &saveEsArr)
+			}
+		}(row.Cells)
 	}
+	wg.Wait()
 	if len(saveEsArr) > 0 {
 		util.BulkSave("bidding_customer", &saveEsArr, false, util.EsCus.(*elastic.EsV7))
 	}