标讯模型相关修改 与调整

data_mgo_to_tidb/bidding.go

@@ -20,52 +20,6 @@ var (
 	regLetter = regexp.MustCompile("[a-z]*")
 )
 
-func doBiddingTask(gtid, lteid string, mapInfo map[string]interface{}) {
-	sess := MongoB.GetMgoConn()
-	defer MongoB.DestoryMongoConn(sess)
-
-	ch := make(chan bool, 10)
-	wg := &sync.WaitGroup{}
-
-	stype := util.ObjToString(mapInfo["stype"])
-	q := map[string]interface{}{"_id": map[string]interface{}{"$gt": mongodb.StringTOBsonId(gtid),
-		"$lte": mongodb.StringTOBsonId(lteid)}}
-	query := sess.DB(config.Conf.DB.MongoB.Dbname).C("bidding").Find(q).Sort("_id").Iter()
-	count := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
-		if count%1000 == 0 {
-			log.Info(fmt.Sprintf("current --- %d", count))
-		}
-		ch <- true
-		wg.Add(1)
-		go func(tmp map[string]interface{}) {
-			defer func() {
-				<-ch
-				wg.Done()
-			}()
-			if util.IntAll(tmp["dataprocess"]) != 8 {
-				return
-			}
-			if stype == "bidding_history" && tmp["history_updatetime"] == nil {
-				return
-			}
-			taskBase(tmp)
-			taskTags(tmp)
-			taskExpand(tmp)
-			taskAtts(tmp)
-			taskInfoformat(tmp)
-			taskIntent(tmp)
-			taskWinner(tmp)
-			taskPackage(tmp)
-			taskPur(tmp)
-
-		}(tmp)
-		tmp = make(map[string]interface{})
-	}
-	wg.Wait()
-	log.Info(fmt.Sprintf("over --- %d", count))
-}
-
 func taskB() {
 	sess := MongoB.GetMgoConn()
 	defer MongoB.DestoryMongoConn(sess)
@@ -87,15 +41,13 @@ func taskB() {
 			}()
 			if util.IntAll(tmp["extracttype"]) != -1 {
 				//taskBase(tmp) 			//基础标讯数据
-				//taskTags(tmp)     		//标签信息
 				//taskExpand(tmp) 		//扩展数据
-				taskDetail(tmp) //正文信息
-				//taskAtts(tmp) 			//附件信息
-				//taskInfoformat(tmp) 	//拟建
-				//taskIntent(tmp) 	  	//采购意向
+				//taskDetail(tmp) //正文信息
+				//taskAtts(tmp) //附件信息
+				//taskIntent(tmp) //采购意向
+				taskPackage(tmp) //分包
+
 				//taskWinner(tmp)
-				//taskPackage(tmp)
-				//taskPur(tmp)
 			}
 		}(tmp)
 		tmp = make(map[string]interface{})
@@ -105,7 +57,6 @@ func taskB() {
 }
 
 // @Description 基本信息
-// @Author J 2022/9/22 11:12
 func taskBase(tmp map[string]interface{}) {
 	saveM := make(map[string]interface{})
 	var errf []string // 异常字段
@@ -194,7 +145,6 @@ func getNameId(name string) string {
 }
 
 // @Description 扩展信息
-// @Author J 2022/9/22 11:13
 func taskExpand(tmp map[string]interface{}) {
 	saveM := make(map[string]interface{})
 	var errf []string // 异常字段
@@ -270,91 +220,50 @@ func taskExpand(tmp map[string]interface{}) {
 	}
 }
 
+// @Description 正文信息
 func taskDetail(tmp map[string]interface{}) {
-
-}
-
-// @Description 标签记录
-// @Author J 2022/9/22 11:13
-func taskTags(tmp map[string]interface{}) {
 	id := mongodb.BsonIdToSId(tmp["_id"])
-	if topArr, ok := tmp["topscopeclass"].([]interface{}); ok {
-		for _, i2 := range topArr {
-			tclass := regLetter.ReplaceAllString(util.ObjToString(i2), "") // 去除字母
-			code := TopScopeCode[tclass]
-			saveTagPool <- map[string]interface{}{"infoid": id, "labelcode": "1", "labelvalues": code, "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
-			//MysqlTool.Insert("bid_tags", map[string]interface{}{"infoid": id, "labelcode": "1", "labelvalues": code, "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)})
-		}
-	}
-	if subArr, ok := tmp["subscopeclass"].([]interface{}); ok {
-		for _, i2 := range subArr {
-			sc := strings.Split(util.ObjToString(i2), "_")
-			if len(sc) > 1 {
-				code := SubScopeCode[sc[1]]
-				saveTagPool <- map[string]interface{}{"infoid": id, "labelcode": "2", "labelvalues": code, "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
-			}
-			//MysqlTool.Insert("bid_tags", map[string]interface{}{"infoid": id, "labelcode": "2", "labelvalues": code, "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)})
-		}
-	}
-	if tArr, ok := tmp["certificate_class"].([]interface{}); ok {
-		for _, i2 := range tArr {
-			if util.ObjToString(i2) == "ISO" {
-				saveTagPool <- map[string]interface{}{"infoid": id, "labelcode": "3", "labelvalues": "01", "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
-				//MysqlTool.Insert("bid_tags", map[string]interface{}{"infoid": id, "labelcode": "3", "labelvalues": "01", "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)})
-			} else if util.ObjToString(i2) == "AAA" {
-				saveTagPool <- map[string]interface{}{"infoid": id, "labelcode": "3", "labelvalues": "02", "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
-				//MysqlTool.Insert("bid_tags", map[string]interface{}{"infoid": id, "labelcode": "3", "labelvalues": "02", "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)})
-			} else if util.ObjToString(i2) == "ISO,AAA" {
-				saveTagPool <- map[string]interface{}{"infoid": id, "labelcode": "3", "labelvalues": "03", "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
-				//MysqlTool.Insert("bid_tags", map[string]interface{}{"infoid": id, "labelcode": "3", "labelvalues": "03", "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)})
-			}
-		}
+	s_detail := util.ObjToString(tmp["detail"])
+	s_contenthtml := util.ObjToString(tmp["contenthtml"])
+	saveDetailPool <- map[string]interface{}{
+		"s_info_id":     id,
+		"s_detail":      s_detail,
+		"s_contenthtml": s_contenthtml,
+		"d_createtime":  time.Now().Format(util.Date_Full_Layout),
+		"d_updatetime":  time.Now().Format(util.Date_Full_Layout),
 	}
 }
 
 // @Description 附件
-// @Author J 2022/9/22 11:13
 func taskAtts(tmp map[string]interface{}) {
-	id := mongodb.BsonIdToSId(tmp["_id"])
-	if tmp["projectinfo"] != nil {
-		if pinfo, o := tmp["projectinfo"].(map[string]interface{}); o {
-			if attsMap, ok := pinfo["attachments"].(map[string]interface{}); ok {
-				for _, attr := range attsMap {
-					if at, ok := attr.(map[string]interface{}); ok {
-						if util.ObjToString(at["fid"]) != "" {
-							ftype := ""
-							for _, s := range FileTypeArr {
-								ft := strings.ToLower(util.ObjToString(tmp["ftype"]))
-								if strings.Contains(ft, s) {
-									ftype = s
-									break
-								}
-							}
-							saveAttrPool <- map[string]interface{}{"infoid": id, "org_url": at["org_url"], "size": at["size"], "fid": at["fid"],
-								"filename": at["filename"], "ftype": ftype, "file_type": 0, "createtime": time.Now().Format(util.Date_Full_Layout)}
-						}
-					}
-				}
-			}
-		}
-	}
-	if attachTxt, o := tmp["attach_text"].(map[string]interface{}); o {
-		if len(attachTxt) > 0 {
-			for _, at := range attachTxt {
-				at1 := at.(map[string]interface{})
-				if len(at1) > 0 {
-					for k, v := range at1 {
-						if reflect.TypeOf(v).String() == "string" {
-							if util.ObjToString(at1["attach_url"]) != "" {
-								saveAttrPool <- map[string]interface{}{"infoid": id, "fid": at1["attach_url"], "filename": at1["file_name"], "file_type": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
-							}
-							break
-						} else {
-							if at2, ok := at1[k].(map[string]interface{}); ok {
-								if util.ObjToString(at2["attach_url"]) != "" {
-									saveAttrPool <- map[string]interface{}{"infoid": id, "fid": at2["attach_url"], "filename": at2["file_name"], "file_type": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
-								}
-							}
+	tmpid := mongodb.BsonIdToSId(tmp["_id"])
+	f_baseInfo := map[string]interface{}{}
+	attach_text := util.ObjToMap(tmp["attach_text"])
+	if projectinfo := util.ObjToMap(tmp["projectinfo"]); projectinfo != nil {
+		attachments := util.ObjToMap((*projectinfo)["attachments"])
+		for index, attr := range *attachments {
+			if at, ok := attr.(map[string]interface{}); ok {
+				if util.ObjToString(at["fid"]) != "" {
+					//ftype := ""
+					//for _, s := range FileTypeArr {
+					//	ft := strings.ToLower(util.ObjToString(tmp["ftype"]))
+					//	if strings.Contains(ft, s) {
+					//		ftype = s
+					//		break
+					//	}
+					//}
+					f_baseInfo["s_info_id"] = tmpid
+					f_baseInfo["s_file_name"] = util.ObjToString(at["filename"])
+					f_baseInfo["s_file_url"] = util.ObjToString(at["org_url"])
+					f_baseInfo["s_file_size"] = util.ObjToString(at["size"])
+					f_baseInfo["s_file_suffix"] = util.ObjToString(at["ftype"])
+					f_baseInfo["s_file_oss_url"] = util.ObjToString(at["fid"])
+					f_baseInfo["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+					f_id := InsertGlobalMysqlData("dwd_f_bid_file_baseinfo", f_baseInfo, tmpid)
+					if f_id > 0 && util.IntAll(index) > 0 && attach_text != nil {
+						att_key := fmt.Sprintf("%d", util.IntAll(index)-1)
+						if att_info := util.ObjToMap((*attach_text)[att_key]); att_info != nil {
+							taskAttsAttach(*att_info, tmpid, f_id)
 						}
 					}
 				}
@@ -363,128 +272,80 @@ func taskAtts(tmp map[string]interface{}) {
 	}
 }
 
-// @Description 拟建
-// @Author J 2022/9/22 15:56
-func taskInfoformat(tmp map[string]interface{}) {
-	if util.IntAll(tmp["infoformat"]) != 2 && tmp["projectinfo"] != nil {
-		return
-	}
-	if info, ok := tmp["projectinfo"].(map[string]interface{}); ok {
-		delete(info, "attachments")
-		if len(info) > 0 {
-			saveM := make(map[string]interface{})
-			for _, f := range IfmField {
-				if f == "infoid" {
-					saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
-				} else if f == "createtime" || f == "updatetime" {
-					saveM[f] = time.Now().Format(util.Date_Full_Layout)
-				} else if f == "approvetime" {
-					if info[f] != nil && util.IntAll(tmp[f]) > 0 {
-						saveM[f] = info[f]
-					}
-				} else {
-					if info[f] != nil {
-						saveM[f] = info[f]
-					}
-				}
-			}
-			saveIfmPool <- saveM
+func taskAttsAttach(att_info map[string]interface{}, tmpid string, f_id int64) {
+	for _, v := range att_info {
+		if att, b := v.(map[string]interface{}); b {
+			info := map[string]interface{}{}
+			info["s_info_id"] = tmpid
+			info["s_file_id"] = f_id
+			info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+			attach_url := util.ObjToString(att["attach_url"])
+			bs := OssGetObject(attach_url)
+			//if utf8.RuneCountInString(bs) > 100000 {
+			//	bs = string(([]rune(bs))[:100000])
+			//}
+			info["s_file_text"] = bs
+			saveAttrPool <- info
 		}
 	}
 }
 
 // @Description 采购意向
-// @Author J 2022/9/22 16:27
 func taskIntent(tmp map[string]interface{}) {
 	if arr, ok := tmp["procurementlist"].([]interface{}); ok {
 		for _, p := range arr {
 			p1 := p.(map[string]interface{})
-			saveM := make(map[string]interface{})
-			for _, f := range IntentField {
-				if f == "infoid" {
-					saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
-				} else if f == "createtime" || f == "updatetime" {
-					saveM[f] = time.Now().Format(util.Date_Full_Layout)
-				} else if f == "buyer_id" {
-					if b := util.ObjToString(tmp["buyer"]); b != "" {
-						//if code := redis.GetStr("qyxy_id", b); code != "" {
-						//	saveM[f] = code
-						//}
-						if code := getNameId(b); code != "" {
-							saveM[f] = code
-						}
-					}
-				} else {
-					if p1[f] != nil {
-						saveM[f] = p1[f]
-					}
+			saveM := map[string]interface{}{}
+			saveM["s_info_id"] = mongodb.BsonIdToSId(tmp["_id"])
+			if p1["itemname"] != nil {
+				saveM["s_intention_name"] = p1["itemname"]
+			}
+			if p1["projectscope"] != nil {
+				saveM["s_intention_demand"] = p1["projectscope"]
+			}
+			if p1["item"] != nil {
+				saveM["s_item"] = p1["item"]
+			}
+			if p1["totalprice"] != nil {
+				saveM["f_totalprice"] = p1["totalprice"]
+			}
+			if p1["expurasingtime"] != nil {
+				saveM["s_expurasingtime"] = p1["expurasingtime"]
+			}
+			if p1["reserved_amount"] != nil {
+				saveM["s_reserved_amount"] = p1["reserved_amount"]
+			}
+			if b := util.ObjToString(tmp["buyer"]); b != "" {
+				if code := getNameId(b); code != "" {
+					saveM["s_buyer_id"] = code
 				}
 			}
+			saveM["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
 			saveIntentPool <- saveM
 		}
 	}
 }
 
-// @Description 中标单位
-// @Author J 2022/9/27 10:58
-func taskWinner(tmp map[string]interface{}) {
-	if wod, ok := tmp["winnerorder"].([]interface{}); ok {
-		for _, w := range wod {
-			w1 := w.(map[string]interface{})
-			if w1["sort"] != nil {
-				saveM := make(map[string]interface{})
-				for _, f := range WinnerField {
-					if f == "infoid" {
-						saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
-					} else if f == "createtime" || f == "updatetime" {
-						saveM[f] = time.Now().Format(util.Date_Full_Layout)
-					} else if f == "winnersort" {
-						saveM[f] = util.IntAll(w1["sort"])
-					} else if f == "winner_id" {
-						if b := util.ObjToString(w1["entname"]); b != "" {
-							saveM["winner"] = b
-							//if code := redis.GetStr("qyxy_id", b); code != "" {
-							//	saveM[f] = code
-							//}
-							if code := getNameId(b); code != "" {
-								saveM[f] = code
-							}
-						}
-					} else if f == "package_id" {
-
-					}
-				}
-				saveWinnerPool <- saveM
-			}
-		}
-	}
-
-	warr := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
-	if BinarySearch(warr, util.ObjToString(tmp["winner"])) == -1 {
-		warr = append(warr, util.ObjToString(tmp["winner"]))
-	}
-	for _, s := range warr {
-		saveM := make(map[string]interface{})
-		for _, f := range WinnerField {
-			if f == "infoid" {
-				saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
-			} else if f == "createtime" || f == "updatetime" {
-				saveM[f] = time.Now().Format(util.Date_Full_Layout)
-			} else if f == "winnersort" {
-				saveM[f] = 0
-			} else if f == "winner_id" {
-				if s != "" {
-					saveM["winner"] = s
-					//if code := redis.GetStr("qyxy_id", s); code != "" {
-					//	saveM[f] = code
-					//}
-					if code := getNameId(s); code != "" {
-						saveM[f] = code
-					}
-				}
-			}
+// @Description 分包基本信息
+func taskPackage(tmp map[string]interface{}) {
+	tmpid := mongodb.BsonIdToSId(tmp["_id"])
+	packages := filterPackageInfos(tmp)
+	if len(packages) <= 1 { //单包···标讯本身
+		baseInfo := CPBaseInfoFromBidding(tmp, tmpid)
+		pid := InsertGlobalMysqlData("dwd_f_bid_package_baseinfo", baseInfo, tmpid)
+		if pid > 0 {
+			//投标人信息
+			CPBidderBiddingBaseInfo(tmp, tmpid, pid)
+			//标的物信息
+			//new_purlist := CPBiddingPackageGoodsBaseInfo(tmp, tmpid, pid)
+			//for _,v := range new_purlist {
+			//
+			//}
 		}
-		saveWinnerPool <- saveM
+	} else { //多包...具体源信息
+		//for k, v := range packages {
+		//	baseInfo := CPBaseInfoFromPackage(v, tmpid)
+		//}
 	}
 }
 
@@ -503,48 +364,3 @@ func BinarySearch(s []string, k string) int {
 	}
 	return -1
 }
-
-func taskPackage(tmp map[string]interface{}) {
-
-}
-
-// @Description 标的物
-// @Author J 2022/9/29 16:48
-func taskPur(tmp map[string]interface{}) {
-	if plist, ok := tmp["purchasinglist"].([]interface{}); ok {
-		for _, p := range plist {
-			saveM := make(map[string]interface{})
-			p1 := p.(map[string]interface{})
-			for _, f := range PurField {
-				if f == "infoid" {
-					saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
-				} else if f == "unitprice" || f == "totalprice" {
-					if p1[f] != nil {
-						if reflect.TypeOf(p1[f]).String() == "string" {
-
-						} else {
-							if util.Float64All(p1[f]) <= 10000000000 {
-								saveM[f], _ = util.FormatFloat(util.Float64All(p1[f]), 4)
-							}
-						}
-					}
-				} else {
-					if p1[f] != nil {
-						if reflect.TypeOf(p1[f]).String() == "string" {
-							if f == "item" || f == "itemname" || f == "brandname" {
-								if len(util.ObjToString(p1[f])) <= 500 {
-									saveM[f] = p1[f]
-								}
-							} else {
-								saveM[f] = p1[f]
-							}
-						} else {
-							saveM[f] = p1[f]
-						}
-					}
-				}
-			}
-			savePurPool <- saveM
-		}
-	}
-}

data_mgo_to_tidb/go.mod

@@ -4,9 +4,12 @@ go 1.16
 
 require (
 	github.com/BurntSushi/toml v1.2.0
+	github.com/aliyun/alibaba-cloud-sdk-go v1.62.605 // indirect
+	github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible // indirect
 	github.com/shopspring/decimal v1.3.1
 	github.com/spf13/cobra v1.5.0
 	go.mongodb.org/mongo-driver v1.10.3
 	go.uber.org/zap v1.23.0
+	golang.org/x/time v0.4.0 // indirect
 	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061
 )

data_mgo_to_tidb/go.sum

@@ -1,9 +1,17 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
+github.com/aliyun/alibaba-cloud-sdk-go v1.62.605 h1:8qKLpS1eRZxnNrc9zTDXi1Gr9+1JO76XS2h8nw83jAk=
+github.com/aliyun/alibaba-cloud-sdk-go v1.62.605/go.mod h1:CJJYa1ZMxjlN/NbXEwmejEnBkhi0DV+Yb3B2lxf+74o=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible h1:so4m5rRA32Tc5GgKg/5gKUu0CRsYmVO3ThMP6T3CwLc=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
@@ -13,6 +21,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -22,11 +31,15 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -51,14 +64,20 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE=
+github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -66,12 +85,22 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A=
+github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -98,6 +127,8 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
+github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
@@ -115,6 +146,8 @@ go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydA
 go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
@@ -123,15 +156,26 @@ go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
 go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -153,6 +197,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cO
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -168,11 +213,16 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY=
+golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -180,6 +230,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
+gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
+gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
+gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -202,6 +256,10 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=
+gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
@@ -210,9 +268,11 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061 h1:xfmoVsDEqVv1XzAVxgGEDi+W9ojHBJc6OmTbg3b1tP0=
 jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

data_mgo_to_tidb/init.go

@@ -11,12 +11,18 @@ import (
 	"os"
 )
 
-var (
+var ( //通用
 	MongoB, MongoP *mongodb.MongodbSim
 	MysqlTool      *mysqldb.Mysql
-
-	FCriteria map[string]interface{}
-
+	FCriteria      map[string]interface{}
+	AreaCode       = make(map[string]string, 5000)
+	TopTypeCode    = make(map[string]string, 10)
+	SubTypeCode    = make(map[string]string, 40)
+	BuyerCode      = make(map[string]string, 100)
+	TopScopeCode   = make(map[string]interface{}, 20)
+	SubScopeCode   = make(map[string]interface{}, 70)
+)
+var ( //标讯
 	saveSize       = 200
 	saveBasePool   = make(chan map[string]interface{}, 5000)
 	saveBaseSp     = make(chan bool, 1)
@@ -24,65 +30,43 @@ var (
 	saveExpandSp   = make(chan bool, 1)
 	saveDetailPool = make(chan map[string]interface{}, 5000)
 	saveDetailSp   = make(chan bool, 1)
-
-	saveTagPool = make(chan map[string]interface{}, 5000)
-	saveTagSp   = make(chan bool, 1)
-
 	saveAttrPool   = make(chan map[string]interface{}, 5000)
 	saveAttrSp     = make(chan bool, 1)
-	saveIfmPool    = make(chan map[string]interface{}, 5000)
-	saveIfmSp      = make(chan bool, 1)
 	saveIntentPool = make(chan map[string]interface{}, 5000)
 	saveIntentSp   = make(chan bool, 1)
-	saveWinnerPool = make(chan map[string]interface{}, 5000)
-	saveWinnerSp   = make(chan bool, 1)
-	savePkgPool    = make(chan map[string]interface{}, 5000)
-	savePkgSp      = make(chan bool, 1)
-	savePurPool    = make(chan map[string]interface{}, 5000)
-	savePurSp      = make(chan bool, 1)
-
-	saveProPool    = make(chan map[string]interface{}, 5000)
-	saveProSp      = make(chan bool, 1)
-	saveProbPool   = make(chan map[string]interface{}, 5000)
-	saveProbSp     = make(chan bool, 1)
-	saveProTagPool = make(chan map[string]interface{}, 5000)
-	saveProTagSp   = make(chan bool, 1)
-
-	saveRelationPool = make(chan map[string]interface{}, 5000)
-	saveRelationSp   = make(chan bool, 1)
-
-	saveErrPool = make(chan map[string]interface{}, 5000)
-	saveErrSp   = make(chan bool, 1)
-
-	AreaCode     = make(map[string]string, 5000)
-	TopTypeCode  = make(map[string]string, 10)
-	SubTypeCode  = make(map[string]string, 40)
-	BuyerCode    = make(map[string]string, 100)
-	TopScopeCode = make(map[string]interface{}, 20)
-	SubScopeCode = make(map[string]interface{}, 70)
+	saveBidderPool = make(chan map[string]interface{}, 5000)
+	saveBidderSp   = make(chan bool, 1)
+	saveGoodsPool  = make(chan map[string]interface{}, 5000)
+	saveGoodsSp    = make(chan bool, 1)
 
 	BaseField   []string
 	BaseVMap    map[string]interface{}
 	ExpandField []string
 	ExpandVMap  map[string]interface{}
-
-	ProField    []string
-	ProVMap     map[string]interface{}
-	ProBusField []string
-	ProBusVMap  map[string]interface{}
-
-	TagsField    = []string{"infoid", "labelcode", "labelvalues", "labelweight", "createtime"}
-	AttrField    = []string{"infoid", "filename", "fid", "ftype", "org_url", "size", "file_type", "createtime"}
-	FileTypeArr  = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"}
-	IfmField     = []string{"infoid", "approvecode", "approvedept", "approvestatus", "approvetime", "approvenumber", "approvecontent", "projecttype", "approvecity", "updatetime", "createtime"}
-	PurField     = []string{"infoid", "item", "itemname", "brandname", "specs", "model", "unitname", "number", "unitprice", "totalprice"}
-	IntentField  = []string{"infoid", "projectname", "projectscope", "item", "buyer_id", "totalprice", "expurasingtime", "reserved_amount", "updatetime", "createtime"}
-	WinnerField  = []string{"infoid", "winner_id", "winnersort", "package_id", "createtime", "updatetime"}
-	PackageField = []string{"infoid", "packagename", "packagename", "packageorgin", "packagedetail", "budget", "bidamount", "winner_id", "updatetime", "createtime"}
-
-	ProTagsField = []string{"projectid", "labelcode", "labelvalues", "labelweight", "createtime"}
-
-	RelationField = []string{"projectid", "infoid", "name_id", "contact_id", "identity_type", "createtime"}
+	DetailField = []string{"s_info_id", "s_detail", "s_contenthtml", "d_updatetime", "d_createtime"}
+	AttrField   = []string{"s_info_id", "s_file_id", "s_file_text", "i_is_bidding_doc", "d_updatetime", "d_createtime"}
+	IntentField = []string{"s_info_id", "s_intention_name", "s_intention_demand", "s_item", "s_buyer_id", "f_totalprice", "s_expurasingtime", "s_reserved_amount", "d_updatetime", "d_createtime"}
+	BidderField = []string{"s_info_id", "i_package_id", "s_bidder_id", "s_bidder_name", "s_other_bidder_id", "s_other_bidder_name", "i_is_bidders", "i_is_winner", "d_updatetime", "d_createtime"}
+	GoodsField  = []string{"s_info_id", "i_package_id", "s_purchasing_tag", "s_itemname", "s_brand", "s_model", "s_specs", "i_num", "f_unit", "f_totalprice", "d_updatetime", "d_createtime"}
+)
+var ( //项目
+	saveProPool      = make(chan map[string]interface{}, 5000)
+	saveProSp        = make(chan bool, 1)
+	saveProbPool     = make(chan map[string]interface{}, 5000)
+	saveProbSp       = make(chan bool, 1)
+	saveProTagPool   = make(chan map[string]interface{}, 5000)
+	saveProTagSp     = make(chan bool, 1)
+	saveRelationPool = make(chan map[string]interface{}, 5000)
+	saveRelationSp   = make(chan bool, 1)
+	saveErrPool      = make(chan map[string]interface{}, 5000)
+	saveErrSp        = make(chan bool, 1)
+	ProField         []string
+	ProVMap          map[string]interface{}
+	ProBusField      []string
+	ProBusVMap       map[string]interface{}
+	ProTagsField     = []string{"projectid", "labelcode", "labelvalues", "labelweight", "createtime"}
+	RelationField    = []string{"projectid", "infoid", "name_id", "contact_id", "identity_type", "createtime"}
+	FileTypeArr      = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"}
 )
 
 // InitLog @Description
@@ -213,3 +197,12 @@ func InitField() {
 		panic("InitField, 未找到field-criteria.json文件")
 	}
 }
+
+// 插入并打印信息
+func InsertGlobalMysqlData(name string, data map[string]interface{}, mark string) int64 {
+	inb := MysqlTool.Insert(name, data)
+	if inb == -1 {
+		log.Info("插入数据异常", zap.String(name, mark))
+	}
+	return inb
+}

data_mgo_to_tidb/main.go

@@ -2,19 +2,9 @@ package main
 
 import (
 	"data_tidb/config"
-	"encoding/json"
-	"fmt"
 	"github.com/spf13/cobra"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.uber.org/zap"
-	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
-	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
-	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
-	"net"
-	"sync"
-	"time"
 )
 
 var (
@@ -27,28 +17,26 @@ func init() {
 	InitMgo()
 	InitMysql()
 	InitField()
+	InitOss(true) //线上部署记得关闭
 	//redis.InitRedis1("qyxy_id=127.0.0.1:8379", 1)
 	//redis.InitRedis1("qyxy_id=192.168.3.166:4379", 1)
 	log.Info("init success")
-
 	//启动 nohup ./MTB bidding &
 }
 
 func main() {
-
-	taskB()
-
 	//go SaveFunc()
-	//go SaveTagFunc()
 	//go SaveExpandFunc()
+	//go SaveDetailFunc()
 	//go SaveAttrFunc()
-	//go SaveImfFunc()
 	//go SaveIntentFunc()
+
 	//go SaveWinnerFunc()
-	//go SavePackageFunc()
 	//go SavePurFunc()
 	//go saveErrMethod()
 
+	taskB()
+
 	//rootCmd := &cobra.Command{Use: "my cmd"}
 	//rootCmd.AddCommand(bidding())
 	//rootCmd.AddCommand(project())
@@ -57,243 +45,14 @@ func main() {
 	//	fmt.Println("rootCmd.Execute failed", err.Error())
 	//}
 
-	//go SaveRelationFunc()
-	//taskMysql()
-
 	//UdpClient = udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
 	//UdpClient.Listen(processUdpMsg)
 	//log.Info("Udp服务监听", zap.String("port:", config.Conf.Udp.LocPort))
-
 	c := make(chan bool, 1)
 	<-c
 
 }
 
-func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
-	defer util.Catch()
-	switch act {
-	case udp.OP_TYPE_DATA: //上个节点的数据
-		var mapInfo map[string]interface{}
-		err := json.Unmarshal(data, &mapInfo)
-		log.Info("processUdpMsg", zap.Any("mapInfo:", mapInfo))
-		gtid, _ := mapInfo["gtid"].(string)
-		lteid, _ := mapInfo["lteid"].(string)
-		if err != nil {
-			UdpClient.WriteUdp([]byte("tidb udp error"), udp.OP_NOOP, ra) //udp失败回写
-		} else {
-			//udp成功回写
-			if k := util.ObjToString(mapInfo["key"]); k != "" {
-				UdpClient.WriteUdp([]byte(k), udp.OP_NOOP, ra)
-			} else {
-				k = fmt.Sprintf("%s-%s-%s", gtid, lteid, util.ObjToString(mapInfo["stype"]))
-				UdpClient.WriteUdp([]byte(k), udp.OP_NOOP, ra)
-			}
-			log.Info("start sync ...")
-			//doBiddingTask(gtid, lteid, mapInfo)
-		}
-	}
-}
-
-func taskMysql() {
-	pool := make(chan bool, 5) //控制线程数
-	wg := &sync.WaitGroup{}
-
-	finalId := 0
-	lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT * FROM %s ORDER BY id DESC LIMIT 1", "dws_f_bpmc_relation"))
-	//lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT id, projectid, infoid, name_id, identity_type+0 FROM %s ORDER BY id DESC LIMIT 1", "dws_f_bpmc_relation_new"))
-	if len(*lastInfo) > 0 {
-		finalId = util.IntAll((*lastInfo)[0]["id"])
-	}
-	log.Info("查询最后id---", zap.Int("finally id: ", finalId))
-	lastid, count := 0, 0
-	for {
-		log.Info("重新查询，lastid---", zap.Int("lastid: ", lastid))
-		q := fmt.Sprintf("SELECT * FROM %s WHERE id > %d ORDER BY id ASC limit 100000", "dws_f_bpmc_relation", lastid)
-		//q := fmt.Sprintf("SELECT id, projectid, infoid, name_id, identity_type+0 FROM %s WHERE id=61771536 ORDER BY id ASC limit 1000000", "dws_f_bpmc_relation")
-		//q := fmt.Sprintf("SELECT id, name, name_id FROM %s WHERE id>%d ORDER BY id ASC limit 1000000", "dws_f_ent_baseinfo", lastid)
-		rows, err := MysqlTool.DB.Query(q)
-		if err != nil {
-			log.Error("mysql query err ", zap.Error(err))
-		}
-		columns, err := rows.Columns()
-		if finalId == lastid {
-			log.Info("----finish-----", zap.Int("count: ", count))
-			break
-		}
-		for rows.Next() {
-			scanArgs := make([]interface{}, len(columns))
-			values := make([]interface{}, len(columns))
-			ret := make(map[string]interface{})
-			for k := range values {
-				scanArgs[k] = &values[k]
-			}
-			err = rows.Scan(scanArgs...)
-			if err != nil {
-				log.Error("mysql scan err ", zap.Error(err))
-				break
-			}
-			for i, col := range values {
-				if col == nil {
-					ret[columns[i]] = nil
-				} else {
-					switch val := (*scanArgs[i].(*interface{})).(type) {
-					case byte:
-						ret[columns[i]] = val
-						break
-					case []byte:
-						v := string(val)
-						switch v {
-						case "\x00": // 处理数据类型为bit的情况
-							ret[columns[i]] = 0
-						case "\x01": // 处理数据类型为bit的情况
-							ret[columns[i]] = 1
-						default:
-							ret[columns[i]] = v
-							break
-						}
-						break
-					case time.Time:
-						if val.IsZero() {
-							ret[columns[i]] = nil
-						} else {
-							ret[columns[i]] = val.Format("2006-01-02 15:04:05")
-						}
-						break
-					default:
-						ret[columns[i]] = val
-					}
-				}
-			}
-			lastid = util.IntAll(ret["id"])
-			count++
-			if count%20000 == 0 {
-				log.Info("current----", zap.Int("count: ", count), zap.Int("lastid: ", lastid))
-			}
-			pool <- true
-			wg.Add(1)
-			go func(tmp map[string]interface{}) {
-				defer func() {
-					<-pool
-					wg.Done()
-				}()
-				//cid := util.Int64All(tmp["id"])
-				//iid := util.ObjToString(tmp["infoid"])
-				//name_id := util.ObjToString(tmp["name_id"])
-				//identity_type := util.Int64All(tmp["identity_type+0"])
-				//if name_id != "" {
-				//	coll := "bidding"
-				//	if iid > "5a862e7040d2d9bbe88e3b1f" {
-				//		coll = "bidding"
-				//	} else {
-				//		coll = "bidding_back"
-				//	}
-				//	info, _ := MongoB.FindById(coll, iid, bson.M{"agencytel": 1, "agencyperson": 1, "buyertel": 1, "buyerperson": 1, "winnertel": 1, "winnerperson": 1})
-				//	if len(*info) > 0 {
-				//		if identity_type == 1 {
-				//			if util.ObjToString((*info)["buyertel"]) != "" {
-				//				q := make(map[string]interface{})
-				//				q["name_id"] = name_id
-				//				q["identity_type"] = identity_type
-				//				q["contact_tel"] = util.ObjToString((*info)["buyertel"])
-				//				if util.ObjToString((*info)["buyerperson"]) != "" {
-				//					q["contact_name"] = util.ObjToString((*info)["buyerperson"])
-				//				}
-				//				cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
-				//				if cinfo != nil && len(*cinfo) > 0 {
-				//					MysqlTool.Update("dws_f_bpmc_relation_new", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
-				//				}
-				//			}
-				//		} else if identity_type == 2 {
-				//			if util.ObjToString((*info)["winnertel"]) != "" {
-				//				q := make(map[string]interface{})
-				//				q["name_id"] = name_id
-				//				q["identity_type"] = identity_type
-				//				q["contact_tel"] = util.ObjToString((*info)["winnertel"])
-				//				if util.ObjToString((*info)["winnerperson"]) != "" {
-				//					q["contact_name"] = util.ObjToString((*info)["winnerperson"])
-				//				}
-				//				cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
-				//				if cinfo != nil && len(*cinfo) > 0 {
-				//					MysqlTool.Update("dws_f_bpmc_relation_new", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
-				//				}
-				//			}
-				//		} else if identity_type == 4 {
-				//			if util.ObjToString((*info)["agencytel"]) != "" {
-				//				q := make(map[string]interface{})
-				//				q["name_id"] = name_id
-				//				q["identity_type"] = identity_type
-				//				q["contact_tel"] = util.ObjToString((*info)["agencytel"])
-				//				if util.ObjToString((*info)["agencyperson"]) != "" {
-				//					q["contact_name"] = util.ObjToString((*info)["agencyperson"])
-				//				}
-				//				cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
-				//				if cinfo != nil && len(*cinfo) > 0 {
-				//					MysqlTool.Update("dws_f_bpmc_relation_new", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
-				//				}
-				//			}
-				//		}
-				//	}
-				//}
-
-				//redis.PutCKV("qyxy_id", util.ObjToString(tmp["name"]), util.ObjToString(tmp["name_id"]))
-
-				saveM := make(map[string]interface{})
-				if util.ObjToString(tmp["name_id"]) != "" {
-					saveM["name_id"] = util.ObjToString(tmp["name_id"])
-				} else {
-					return
-				}
-				if util.ObjToString(tmp["contact_id"]) != "" {
-					saveM["contact_id"] = util.IntAll(tmp["contact_id"])
-				} else {
-					return
-				}
-				saveM["projectid"] = util.ObjToString(tmp["projectid"])
-				saveM["infoid"] = util.ObjToString(tmp["infoid"])
-				saveM["identity_type"] = tmp["identity_type"]
-				saveRelationPool <- saveM
-			}(ret)
-			ret = make(map[string]interface{})
-		}
-		_ = rows.Close()
-		wg.Wait()
-	}
-}
-
-func taskMgo() {
-	sess := MongoP.GetMgoConn()
-	defer MongoP.DestoryMongoConn(sess)
-
-	ch := make(chan bool, 3)
-	wg := &sync.WaitGroup{}
-
-	q := bson.M{"_id": bson.M{"$lte": mongodb.StringTOBsonId("63411488911e1eb3459fb87e")}}
-	field := map[string]interface{}{"ids": 1}
-	query := sess.DB("qfw").C("projectset_20220721").Find(q).Select(field).Iter()
-	count := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
-		if count%20000 == 0 {
-			util.Debug("current ---", count, tmp["_id"])
-		}
-		ch <- true
-		wg.Add(1)
-		go func(tmp map[string]interface{}) {
-			defer func() {
-				<-ch
-				wg.Done()
-			}()
-			id := mongodb.BsonIdToSId(tmp["_id"])
-			for _, i := range util.ObjArrToStringArr(tmp["ids"].([]interface{})) {
-				redis.PutCKV("s_id", i, id)
-			}
-		}(tmp)
-		tmp = make(map[string]interface{})
-	}
-	wg.Wait()
-
-	util.Debug("over ---", count)
-}
-
 // @Description 标讯数据
 // @Author J 2022/9/20 17:52
 func bidding() *cobra.Command {
@@ -301,7 +60,7 @@ func bidding() *cobra.Command {
 		Use:   "bidding",
 		Short: "Start processing bidding data",
 		Run: func(cmd *cobra.Command, args []string) {
-			go SaveFunc()
+			//go SaveFunc()
 			//go SaveTagFunc()
 			//go SaveExpandFunc()
 			//go SaveAttrFunc()
@@ -328,7 +87,7 @@ func project() *cobra.Command {
 			//go SaveProFunc()
 			//go SaveProTagFunc()
 			//go SaveProbFunc()
-			go SaveRelationFunc()
+			//go SaveRelationFunc()
 			taskP()
 		},
 	}
@@ -354,529 +113,3 @@ func projectAdd() *cobra.Command {
 	cmdClient.Flags().Int64VarP(&pici, "pici", "p", 0, "")
 	return cmdClient
 }
-
-func SaveFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveBasePool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveBaseSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveBaseSp
-					}()
-					MysqlTool.InsertBulk("dwd_f_bid_baseinfo", BaseField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveBaseSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveBaseSp
-					}()
-					MysqlTool.InsertBulk("dwd_f_bid_baseinfo", BaseField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveExpandFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveExpandPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveExpandSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveExpandSp
-					}()
-					MysqlTool.InsertBulk("dwd_f_bid_expand_baseinfo", ExpandField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveExpandSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveExpandSp
-					}()
-					MysqlTool.InsertBulk("dwd_f_bid_expand_baseinfo", ExpandField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveDetailFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveExpandPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveExpandSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveExpandSp
-					}()
-					MysqlTool.InsertBulk("dwd_f_bid_detail", ExpandField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveExpandSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveExpandSp
-					}()
-					MysqlTool.InsertBulk("dwd_f_bid_detail", ExpandField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveTagFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveTagPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveTagSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveTagSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_tags", TagsField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveTagSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveTagSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_tags", TagsField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveAttrFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveAttrPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveAttrSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveAttrSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_filetext_baseinfo", AttrField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveAttrSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveAttrSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_filetext_baseinfo", AttrField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveImfFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveIfmPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveIfmSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveIfmSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_infoformat_baseinfo", IfmField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveIfmSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveIfmSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_infoformat_baseinfo", IfmField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SavePurFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-savePurPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				savePurSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-savePurSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_purchasing_baseinfo", PurField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				savePurSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-savePurSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_purchasing_baseinfo", PurField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveIntentFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveIntentPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveIntentSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveIntentSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_intention_baseinfo", IntentField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveIntentSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveIntentSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_intention_baseinfo", IntentField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveWinnerFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveWinnerPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveWinnerSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveWinnerSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_winner_baseinfo", WinnerField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveWinnerSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveWinnerSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_winner_baseinfo", WinnerField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SavePackageFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-savePkgPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				savePkgSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-savePkgSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_package_baseinfo", PackageField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				savePkgSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-savePkgSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bid_package_baseinfo", PackageField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveProFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveProPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveProSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveProSp
-					}()
-					MysqlTool.InsertBulk("dws_f_project_baseinfo", ProField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveProSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveProSp
-					}()
-					MysqlTool.InsertBulk("dws_f_project_baseinfo", ProField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveProbFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveProbPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveProbSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveProbSp
-					}()
-					MysqlTool.InsertBulk("dws_f_project_business", ProBusField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveProbSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveProbSp
-					}()
-					MysqlTool.InsertBulk("dws_f_project_business", ProBusField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveProTagFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveProTagPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveProTagSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveProTagSp
-					}()
-					MysqlTool.InsertBulk("dws_f_project_tags", ProTagsField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveProTagSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveProTagSp
-					}()
-					MysqlTool.InsertBulk("dws_f_project_tags", ProTagsField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-func SaveRelationFunc() {
-	arru := make([]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveRelationPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveRelationSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveRelationSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bpmc_relation", RelationField, arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveRelationSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveRelationSp
-					}()
-					MysqlTool.InsertBulk("dws_f_bpmc_relation", RelationField, arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}
-
-// 字段错误数据
-func saveErrMethod() {
-	arru := make([]map[string]interface{}, 200)
-	indexu := 0
-	for {
-		select {
-		case v := <-saveErrPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				saveErrSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveErrSp
-					}()
-					MongoB.SaveBulk("bidding_mgo_to_tidb_f_err", arru...)
-				}(arru)
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				saveErrSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-saveErrSp
-					}()
-					MongoB.SaveBulk("bidding_mgo_to_tidb_f_err", arru...)
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
-	}
-}

data_mgo_to_tidb/oss.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"io"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"os"
+)
+
+var (
+	ossEndpoint = "oss-cn-beijing-internal.aliyuncs.com" //http://oss-cn-beijing.aliyuncs.com"
+	//ossEndpointTest    = "topjy.oss-cn-beijing.aliyuncs.com"
+	ossEndpointTest    = "oss-cn-beijing.aliyuncs.com"
+	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
+	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
+	ossBucketName      = "topjy"
+	ossclient          *oss.Client
+)
+
+func InitOss(isTest bool) {
+	if isTest {
+		ossEndpoint = ossEndpointTest //测试服务器~阿里云
+	}
+	client, err := oss.New(ossEndpoint, ossAccessKeyId, ossAccessKeySecret)
+	if err != nil {
+		fmt.Println("Error:", err)
+		os.Exit(-1)
+	}
+	ossclient = client
+}
+
+func OssGetObject(objectName string) string {
+	util.Catch()
+	// 获取存储空间。
+	bucket, err := ossclient.Bucket(ossBucketName)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+
+	// 下载文件到流。
+	body, err := bucket.GetObject(objectName)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+	defer body.Close()
+	data, err := io.ReadAll(body)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+	return string(data)
+}

data_mgo_to_tidb/pkg.go

@@ -0,0 +1,192 @@
+package main
+
+import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+// 标讯分包基本信息-标讯版
+func CPBaseInfoFromBidding(tmp map[string]interface{}, tmpid string) map[string]interface{} {
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	info["s_packagename"] = util.ObjToString(tmp["projectname"])
+	if tmp["budget"] != nil {
+		info["f_budget"] = util.Float64All(tmp["budget"])
+	}
+	if tmp["bidamount"] != nil {
+		info["f_bidamount"] = util.Float64All(tmp["bidamount"])
+	}
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	return info
+}
+
+// 投标人相关信息-标讯版
+func CPBidderBiddingBaseInfo(tmp map[string]interface{}, tmpid string, pid int64) {
+	s_winner := util.ObjToString(tmp["s_winner"])
+	if s_winner == "" {
+		return
+	}
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	info["i_package_id"] = pid
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	wins := strings.Split(s_winner, ",")
+	bidder_id, bidder_name, other_bidder_id := getWinsNameId(wins)
+	info["s_bidder_id"] = bidder_id
+
+	info["i_is_winner"] = 1
+	if len(wins) > 1 {
+		info["s_bidder_name"] = bidder_name
+		info["s_other_bidder_name"] = s_winner
+		info["s_other_bidder_id"] = other_bidder_id
+		info["i_is_bidders"] = 1
+	}
+	//保存服务...
+	InsertGlobalMysqlData("dwd_f_bid_package_bidder_baseinfo", info, tmpid)
+
+	//候选人相关情况
+	o_win_ids, o_win_names := getWinerorder(tmp, wins[0])
+	for k, v := range o_win_ids {
+		o_info := map[string]interface{}{}
+		o_info["s_info_id"] = tmpid
+		o_info["i_package_id"] = pid
+		o_info["s_bidder_id"] = v
+		o_info["s_bidder_name"] = o_win_names[k]
+		o_info["i_is_winner"] = 0
+		o_info["d_updatetime"] = time.Now().Format(util.Date_Full_Layout)
+		//保存服务...
+		InsertGlobalMysqlData("dwd_f_bid_package_bidder_baseinfo", info, tmpid)
+	}
+}
+
+// 标讯分包标的物信息-记录唯一包
+func CPBiddingPackageGoodsBaseInfo(tmp map[string]interface{}, tmpid string, pid int64) []map[string]interface{} {
+	purchasinglist := IsMarkInterfaceMap(tmp["purchasinglist"])
+	new_purlist := []map[string]interface{}{}
+	for _, v := range purchasinglist {
+		info := map[string]interface{}{}
+		info["s_info_id"] = tmpid
+		info["i_package_id"] = pid
+		info["s_itemname"] = cleanLengthName(util.ObjToString(v["itemname"]), 255)
+		info["s_brand"] = cleanLengthName(util.ObjToString(v["brandname"]), 255)
+		info["s_model"] = cleanLengthName(util.ObjToString(v["model"]), 255)
+		if number := util.IntAll(v["number"]); number > 0 && number < 1000000000 {
+			info["i_num"] = number
+		}
+		if unitprice := util.Float64All(v["unitprice"]); unitprice > 0.0 && unitprice < 1000000000.0 {
+			info["f_unit"] = unitprice
+		}
+		if totalprice := util.Float64All(v["totalprice"]); totalprice > 0.0 && totalprice < 1000000000.0 {
+			info["f_totalprice"] = totalprice
+		}
+		info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+		new_purlist = append(new_purlist, info)
+	}
+	return new_purlist
+}
+
+// 标讯分包基本信息-分包版
+func CPBaseInfoFromPackage(pinfos map[string]interface{}, tmpid string) map[string]interface{} {
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	for k, v := range pinfos {
+		info["s_packagecode"] = k
+		if pkg := util.ObjToMap(v); pkg != nil {
+			info["s_packagename"] = util.ObjToString((*pkg)["name"])
+			if (*pkg)["budget"] != nil {
+				info["f_budget"] = util.Float64All((*pkg)["budget"])
+			}
+			if (*pkg)["bidamount"] != nil {
+				info["f_bidamount"] = util.Float64All((*pkg)["bidamount"])
+			}
+			return info
+		}
+	}
+	return info
+}
+
+// 投标人相关信息-分包版
+func CPBidderPackageBaseInfo(pinfos map[string]interface{}, tmpid string) {
+
+}
+
+// 获取所有投标人的相关标识
+func getWinsNameId(wins []string) (string, string, string) {
+	win_id, win_name, win_ids := "", "", []string{}
+	for k, v := range wins {
+		code := getNameId(v)
+		if k == 0 {
+			win_id = code
+			win_name = v
+		}
+		win_ids = append(win_ids, code)
+	}
+	return win_id, win_name, strings.Join(win_ids, ",")
+}
+
+// 获取候选人相关信息
+func getWinerorder(tmp map[string]interface{}, win string) ([]string, []string) {
+	o_win_names, o_win_ids := []string{}, []string{}
+	winnerorder := IsMarkInterfaceMap(tmp["winnerorder"])
+	for k, v := range winnerorder {
+		if k >= 5 {
+			break
+		}
+		entname := util.ObjToString(v["entname"])
+		if entname != win {
+			o_win_ids = append(o_win_ids, getNameId(entname))
+			o_win_names = append(o_win_names, entname)
+		}
+	}
+	return o_win_ids, o_win_names
+}
+
+// 分包信息清洗与过滤...
+func filterPackageInfos(tmp map[string]interface{}) []map[string]interface{} {
+	new_packages := []map[string]interface{}{}
+	s_winner := util.ObjToString(tmp["s_winner"])
+	if s_winner == "" {
+		return new_packages
+	}
+	winArr := strings.Split(s_winner, ",")
+	if packages := util.ObjToMap(tmp["package"]); packages != nil {
+		for k, v := range *packages {
+			if pkg := util.ObjToMap(v); pkg != nil && k != "" {
+				if win := util.ObjToString((*pkg)["winner"]); win != "" {
+					if isExistsWins(winArr, win) {
+						new_packages = append(new_packages, map[string]interface{}{k: v})
+					}
+				}
+			}
+		}
+	}
+	return new_packages
+}
+
+// 是否存在单位
+func isExistsWins(winArr []string, win string) bool {
+	win = cleaname(win)
+	for _, v := range winArr {
+		if v == win {
+			return true
+		}
+	}
+	return false
+}
+
+// 简易清洗...
+func cleaname(name string) string {
+	return name
+}
+
+// 通用长度清洗
+func cleanLengthName(name string, l int) string {
+	new_name := name
+	if utf8.RuneCountInString(name) > l {
+		new_name = string(([]rune(new_name))[:l])
+	}
+	return new_name
+}

data_mgo_to_tidb/save.go

@@ -0,0 +1,354 @@
+package main
+
+import "time"
+
+func SaveFunc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveBasePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveBaseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveBaseSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_baseinfo", BaseField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveBaseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveBaseSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_baseinfo", BaseField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveExpandFunc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveExpandPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveExpandSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveExpandSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_expand_baseinfo", ExpandField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveExpandSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveExpandSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_expand_baseinfo", ExpandField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveDetailFunc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveDetailPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveDetailSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveDetailSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_detail", DetailField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveDetailSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveDetailSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_detail", DetailField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveAttrFunc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveAttrPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveAttrSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveAttrSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_file_text", AttrField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveAttrSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveAttrSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_file_text", AttrField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveIntentFunc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveIntentPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveIntentSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveIntentSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_intention_baseinfo", IntentField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveIntentSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveIntentSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_intention_baseinfo", IntentField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveProFunc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveProPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveProSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveProSp
+					}()
+					MysqlTool.InsertBulk("dws_f_project_baseinfo", ProField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveProSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveProSp
+					}()
+					MysqlTool.InsertBulk("dws_f_project_baseinfo", ProField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveProbFunc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveProbPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveProbSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveProbSp
+					}()
+					MysqlTool.InsertBulk("dws_f_project_business", ProBusField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveProbSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveProbSp
+					}()
+					MysqlTool.InsertBulk("dws_f_project_business", ProBusField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveProTagFunc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveProTagPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveProTagSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveProTagSp
+					}()
+					MysqlTool.InsertBulk("dws_f_project_tags", ProTagsField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveProTagSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveProTagSp
+					}()
+					MysqlTool.InsertBulk("dws_f_project_tags", ProTagsField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveRelationFunc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveRelationPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveRelationSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveRelationSp
+					}()
+					MysqlTool.InsertBulk("dws_f_bpmc_relation", RelationField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveRelationSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveRelationSp
+					}()
+					MysqlTool.InsertBulk("dws_f_bpmc_relation", RelationField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+// 字段错误数据
+func saveErrMethod() {
+	arru := make([]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveErrPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveErrSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveErrSp
+					}()
+					MongoB.SaveBulk("bidding_mgo_to_tidb_f_err", arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveErrSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveErrSp
+					}()
+					MongoB.SaveBulk("bidding_mgo_to_tidb_f_err", arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

data_mgo_to_tidb/test.go

@@ -0,0 +1,212 @@
+package main
+
+import (
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+	"sync"
+	"time"
+)
+
+func taskMysql() {
+	pool := make(chan bool, 5) //控制线程数
+	wg := &sync.WaitGroup{}
+
+	finalId := 0
+	lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT * FROM %s ORDER BY id DESC LIMIT 1", "dws_f_bpmc_relation"))
+	//lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT id, projectid, infoid, name_id, identity_type+0 FROM %s ORDER BY id DESC LIMIT 1", "dws_f_bpmc_relation_new"))
+	if len(*lastInfo) > 0 {
+		finalId = util.IntAll((*lastInfo)[0]["id"])
+	}
+	log.Info("查询最后id---", zap.Int("finally id: ", finalId))
+	lastid, count := 0, 0
+	for {
+		log.Info("重新查询，lastid---", zap.Int("lastid: ", lastid))
+		q := fmt.Sprintf("SELECT * FROM %s WHERE id > %d ORDER BY id ASC limit 100000", "dws_f_bpmc_relation", lastid)
+		//q := fmt.Sprintf("SELECT id, projectid, infoid, name_id, identity_type+0 FROM %s WHERE id=61771536 ORDER BY id ASC limit 1000000", "dws_f_bpmc_relation")
+		//q := fmt.Sprintf("SELECT id, name, name_id FROM %s WHERE id>%d ORDER BY id ASC limit 1000000", "dws_f_ent_baseinfo", lastid)
+		rows, err := MysqlTool.DB.Query(q)
+		if err != nil {
+			log.Error("mysql query err ", zap.Error(err))
+		}
+		columns, err := rows.Columns()
+		if finalId == lastid {
+			log.Info("----finish-----", zap.Int("count: ", count))
+			break
+		}
+		for rows.Next() {
+			scanArgs := make([]interface{}, len(columns))
+			values := make([]interface{}, len(columns))
+			ret := make(map[string]interface{})
+			for k := range values {
+				scanArgs[k] = &values[k]
+			}
+			err = rows.Scan(scanArgs...)
+			if err != nil {
+				log.Error("mysql scan err ", zap.Error(err))
+				break
+			}
+			for i, col := range values {
+				if col == nil {
+					ret[columns[i]] = nil
+				} else {
+					switch val := (*scanArgs[i].(*interface{})).(type) {
+					case byte:
+						ret[columns[i]] = val
+						break
+					case []byte:
+						v := string(val)
+						switch v {
+						case "\x00": // 处理数据类型为bit的情况
+							ret[columns[i]] = 0
+						case "\x01": // 处理数据类型为bit的情况
+							ret[columns[i]] = 1
+						default:
+							ret[columns[i]] = v
+							break
+						}
+						break
+					case time.Time:
+						if val.IsZero() {
+							ret[columns[i]] = nil
+						} else {
+							ret[columns[i]] = val.Format("2006-01-02 15:04:05")
+						}
+						break
+					default:
+						ret[columns[i]] = val
+					}
+				}
+			}
+			lastid = util.IntAll(ret["id"])
+			count++
+			if count%20000 == 0 {
+				log.Info("current----", zap.Int("count: ", count), zap.Int("lastid: ", lastid))
+			}
+			pool <- true
+			wg.Add(1)
+			go func(tmp map[string]interface{}) {
+				defer func() {
+					<-pool
+					wg.Done()
+				}()
+				//cid := util.Int64All(tmp["id"])
+				//iid := util.ObjToString(tmp["infoid"])
+				//name_id := util.ObjToString(tmp["name_id"])
+				//identity_type := util.Int64All(tmp["identity_type+0"])
+				//if name_id != "" {
+				//	coll := "bidding"
+				//	if iid > "5a862e7040d2d9bbe88e3b1f" {
+				//		coll = "bidding"
+				//	} else {
+				//		coll = "bidding_back"
+				//	}
+				//	info, _ := MongoB.FindById(coll, iid, bson.M{"agencytel": 1, "agencyperson": 1, "buyertel": 1, "buyerperson": 1, "winnertel": 1, "winnerperson": 1})
+				//	if len(*info) > 0 {
+				//		if identity_type == 1 {
+				//			if util.ObjToString((*info)["buyertel"]) != "" {
+				//				q := make(map[string]interface{})
+				//				q["name_id"] = name_id
+				//				q["identity_type"] = identity_type
+				//				q["contact_tel"] = util.ObjToString((*info)["buyertel"])
+				//				if util.ObjToString((*info)["buyerperson"]) != "" {
+				//					q["contact_name"] = util.ObjToString((*info)["buyerperson"])
+				//				}
+				//				cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+				//				if cinfo != nil && len(*cinfo) > 0 {
+				//					MysqlTool.Update("dws_f_bpmc_relation_new", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
+				//				}
+				//			}
+				//		} else if identity_type == 2 {
+				//			if util.ObjToString((*info)["winnertel"]) != "" {
+				//				q := make(map[string]interface{})
+				//				q["name_id"] = name_id
+				//				q["identity_type"] = identity_type
+				//				q["contact_tel"] = util.ObjToString((*info)["winnertel"])
+				//				if util.ObjToString((*info)["winnerperson"]) != "" {
+				//					q["contact_name"] = util.ObjToString((*info)["winnerperson"])
+				//				}
+				//				cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+				//				if cinfo != nil && len(*cinfo) > 0 {
+				//					MysqlTool.Update("dws_f_bpmc_relation_new", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
+				//				}
+				//			}
+				//		} else if identity_type == 4 {
+				//			if util.ObjToString((*info)["agencytel"]) != "" {
+				//				q := make(map[string]interface{})
+				//				q["name_id"] = name_id
+				//				q["identity_type"] = identity_type
+				//				q["contact_tel"] = util.ObjToString((*info)["agencytel"])
+				//				if util.ObjToString((*info)["agencyperson"]) != "" {
+				//					q["contact_name"] = util.ObjToString((*info)["agencyperson"])
+				//				}
+				//				cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+				//				if cinfo != nil && len(*cinfo) > 0 {
+				//					MysqlTool.Update("dws_f_bpmc_relation_new", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
+				//				}
+				//			}
+				//		}
+				//	}
+				//}
+
+				//redis.PutCKV("qyxy_id", util.ObjToString(tmp["name"]), util.ObjToString(tmp["name_id"]))
+
+				saveM := make(map[string]interface{})
+				if util.ObjToString(tmp["name_id"]) != "" {
+					saveM["name_id"] = util.ObjToString(tmp["name_id"])
+				} else {
+					return
+				}
+				if util.ObjToString(tmp["contact_id"]) != "" {
+					saveM["contact_id"] = util.IntAll(tmp["contact_id"])
+				} else {
+					return
+				}
+				saveM["projectid"] = util.ObjToString(tmp["projectid"])
+				saveM["infoid"] = util.ObjToString(tmp["infoid"])
+				saveM["identity_type"] = tmp["identity_type"]
+				saveRelationPool <- saveM
+			}(ret)
+			ret = make(map[string]interface{})
+		}
+		_ = rows.Close()
+		wg.Wait()
+	}
+}
+
+func taskMgo() {
+	sess := MongoP.GetMgoConn()
+	defer MongoP.DestoryMongoConn(sess)
+
+	ch := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+	q := bson.M{"_id": bson.M{"$lte": mongodb.StringTOBsonId("63411488911e1eb3459fb87e")}}
+	field := map[string]interface{}{"ids": 1}
+	query := sess.DB("qfw").C("projectset_20220721").Find(q).Select(field).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%20000 == 0 {
+			util.Debug("current ---", count, tmp["_id"])
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			for _, i := range util.ObjArrToStringArr(tmp["ids"].([]interface{})) {
+				redis.PutCKV("s_id", i, id)
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+
+	util.Debug("over ---", count)
+}

data_mgo_to_tidb/udp.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
+	"net"
+)
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	defer util.Catch()
+	switch act {
+	case udp.OP_TYPE_DATA: //上个节点的数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Info("processUdpMsg", zap.Any("mapInfo:", mapInfo))
+		gtid, _ := mapInfo["gtid"].(string)
+		lteid, _ := mapInfo["lteid"].(string)
+		if err != nil {
+			UdpClient.WriteUdp([]byte("tidb udp error"), udp.OP_NOOP, ra) //udp失败回写
+		} else {
+			//udp成功回写
+			if k := util.ObjToString(mapInfo["key"]); k != "" {
+				UdpClient.WriteUdp([]byte(k), udp.OP_NOOP, ra)
+			} else {
+				k = fmt.Sprintf("%s-%s-%s", gtid, lteid, util.ObjToString(mapInfo["stype"]))
+				UdpClient.WriteUdp([]byte(k), udp.OP_NOOP, ra)
+			}
+			log.Info("start sync ...")
+			//doBiddingTask(gtid, lteid, mapInfo)
+		}
+	}
+}

data_mgo_to_tidb/util.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"reflect"
 	"strings"
@@ -54,3 +55,31 @@ func verifyF(f string, v interface{}, tmp1 interface{}) (interface{}, bool) {
 	}
 	return nil, true
 }
+
+func IsMarkInterfaceArr(t interface{}) []string {
+	sub_list := []string{}
+	if list_3, ok_3 := t.([]string); ok_3 {
+		sub_list = list_3
+		return sub_list
+	}
+	if list_1, ok_1 := t.(primitive.A); ok_1 {
+		sub_list = util.ObjArrToStringArr(list_1)
+	} else {
+		if list_2, ok_2 := t.([]interface{}); ok_2 {
+			sub_list = util.ObjArrToStringArr(list_2)
+		}
+	}
+	return sub_list
+}
+
+func IsMarkInterfaceMap(t interface{}) []map[string]interface{} {
+	p_list := []map[string]interface{}{}
+	if yl_list_1, ok_1 := t.(primitive.A); ok_1 {
+		p_list = util.ObjArrToMapArr(yl_list_1)
+	} else {
+		if yl_list_2, ok_2 := t.([]interface{}); ok_2 {
+			p_list = util.ObjArrToMapArr(yl_list_2)
+		}
+	}
+	return p_list
+}