Merge commit '1ac95b638c6acbd7471f553d81b96e510098727f'

* commit '1ac95b638c6acbd7471f553d81b96e510098727f':
  标讯...异常结构字段...存储修改
  保存...
  批量保存...
  标讯模型修改，配置测试
  标讯数仓...重构...备份
  调整方法...
  标讯数仓-备份
  备份...
  备份 ... 正式版
  标讯相关数仓数据灌入，测试版...备份
  标讯模型相关修改 与调整
  数仓新调整 1、基本信息 2、附件信息
  数仓

# Conflicts:
#	monitor/main.go

data_mgo_to_tidb/bidding/bidding.go

@@ -0,0 +1,300 @@
+package bidding
+
+import (
+	"data_tidb/config"
+	u "data_tidb/util"
+	"fmt"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	regLetter = regexp.MustCompile("[a-z]*")
+	regClean  = regexp.MustCompile("[^ -~\u2E80-\u2FDF\u3040-\u318F\u31A0-\u31BF\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FFF\uA960-\uA97F\uAC00-\uD7FF\u3002\u00a5\uff1f\uff01\uff0c\u3001\uff1b\uff1a\u201c\u201d\u2018\u2019\uff08\uff09\u300a\u300b\u3008\u3009\u3010\u3011\u300e\u300f\u300c\u300d\ufe43\ufe44\u3014\u3015\u2026\u2014\uff5e\ufe4f\uffe5\u00a5]+")
+)
+
+func TaskBidding(gteid string, ltid string) {
+	sess := u.MongoB.GetMgoConn()
+	defer u.MongoB.DestoryMongoConn(sess)
+	ch := make(chan bool, 6)
+	wg := &sync.WaitGroup{}
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": mongodb.StringTOBsonId(gteid),
+			"$lt":  mongodb.StringTOBsonId(ltid),
+		},
+	}
+	query := sess.DB(config.Conf.DB.MongoB.Dbname).C("bidding").Find(q).Sort("_id").Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%10000 == 0 {
+			log.Info(fmt.Sprintf("cur index - %d，%s", count, tmp["_id"]))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			if util.IntAll(tmp["extracttype"]) != -1 {
+				taskBase(tmp)    //基础标讯数据
+				taskExpand(tmp)  //扩展数据
+				taskDetail(tmp)  //正文信息
+				taskAtts(tmp)    //附件信息
+				taskIntent(tmp)  //采购意向
+				taskPackage(tmp) //分包
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("is over --- %d", count))
+}
+
+func taskBase(tmp map[string]interface{}) {
+	tmpid := mongodb.BsonIdToSId(tmp["_id"])
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	area, city, district := util.ObjToString(tmp["area"]), util.ObjToString(tmp["city"]), util.ObjToString(tmp["district"])
+	if area != "" {
+		info["s_area_code"] = u.RegionCode[area]
+		if city != "" {
+			info["s_city_code"] = u.RegionCode[area+","+city]
+			if district != "" {
+				info["s_district_code"] = u.RegionCode[area+","+city+","+district]
+			}
+		}
+	}
+	if buyerclass := util.ObjToString(tmp["buyerclass"]); buyerclass != "" {
+		info["s_buyerclass_code"] = u.BuyerClassCode[buyerclass]
+	}
+
+	info["s_toptype_code"], info["s_subtype_code"] = u.GetTopSubCode(util.ObjToString(tmp["toptype"]), util.ObjToString(tmp["subtype"]))
+
+	//特别结构
+	info["i_isValidFile"] = util.IntAll(tmp["isValidFile"])
+	info["i_multipackage"] = util.IntAll(tmp["multipackage"])
+	//文本相关
+	u.TransferTextInfo(tmp, &info, []string{"title", "projectname", "projectcode", "purchasing", "site", "href"}, []int{500, 500, 100, 1000, 100, 1000})
+	//金额相关
+	u.TransferMoneyRateInfo(tmp, &info, []string{"budget", "bidamount", "biddiscount"}, []float64{1000000000.0, 1000000000.0, 100.0})
+	//时间相关
+	u.TransferDateTimeInfo(tmp, &info, []string{"comeintime", "publishtime", "bidopentime", "bidendtime"})
+	//主体相关
+	if code := u.GetNameId(util.ObjToString(tmp["buyer"])); code != "" {
+		info["s_buyer_id"] = code
+	}
+	if code := u.GetNameId(util.ObjToString(tmp["agency"])); code != "" {
+		info["s_agency_id"] = code
+	}
+	info["d_updatetime"] = time.Now().Format(util.Date_Full_Layout)
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+
+	u.InsertGlobalMysqlData(u.T_dwd_f_bid_baseinfo, info, mongodb.BsonIdToSId(tmp["_id"]))
+	//u.SaveBasePool <- info
+}
+
+func taskExpand(tmp map[string]interface{}) {
+	tmpid := mongodb.BsonIdToSId(tmp["_id"])
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	//文本相关
+	u.TransferTextInfo(tmp, &info, []string{"projectperiod", "project_scale", "project_timeunit", "bidmethod", "getdocmethod", "currency", "funds", "payway", "bid_bond", "contract_bond", "contractcode", "buyerzipcode", "bidopenaddress", "buyeraddr", "agencyaddr", "projectaddr", "enterprise_qualification", "personnel_qualification", "performance_qualification", "enterprise_credit"}, []int{1000, 5000, 50, 255, 500, 20, 5000, 500, 1000, 500, 100, 100, 1000, 1000, 1000, 1000, 20000, 20000, 20000, 20000})
+	//时间相关
+	u.TransferDateTimeInfo(tmp, &info, []string{"project_startdate", "project_completedate", "signstarttime", "bidendtime", "bidstarttime", "docstarttime", "docendtime", "signaturedate", "signendtime"})
+	//布尔相关
+	u.TransferBoolInfo(tmp, &info, []string{"bid_guarantee", "contract_guarantee"})
+	//金额相关
+	u.TransferMoneyRateInfo(tmp, &info, []string{"docamount", "agencyfee", "agencyrate"}, []float64{10000000.0, 10000000.0, 10000000.0})
+	//投标方式
+	bidway := util.ObjToString(tmp["bidway"])
+	if bidway == "电子投标" {
+		info["i_bidway"] = 1
+	} else if bidway == "纸质投标" {
+		info["i_bidway"] = 0
+	} else {
+
+	}
+	//评审专家
+	if tmp["review_experts"] != nil {
+		if reflect.TypeOf(tmp["review_experts"]).String() == "string" {
+			info["s_review_experts"] = tmp["review_experts"]
+		} else if reflect.TypeOf(tmp["review_experts"]).String() == "[]interface {}" {
+			if arr, ok := tmp["review_experts"].([]interface{}); ok {
+				info["s_review_experts"] = strings.Join(util.ObjArrToStringArr(arr), ",")
+			}
+		}
+	}
+	//工期时长
+	if project_duration := util.IntAll(tmp["project_duration"]); project_duration > 0 && project_duration < 10000 {
+		info["i_project_duration"] = util.IntAll(tmp["project_duration"])
+	}
+
+	info["d_updatetime"] = time.Now().Format(util.Date_Full_Layout)
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	u.InsertGlobalMysqlData(u.T_dwd_f_bid_expand_baseinfo, info, tmpid)
+}
+
+func taskDetail(tmp map[string]interface{}) {
+	tmpid := mongodb.BsonIdToSId(tmp["_id"])
+	s_detail := util.ObjToString(tmp["detail"])
+	info := map[string]interface{}{
+		"s_info_id":    tmpid,
+		"s_detail":     s_detail,
+		"d_createtime": time.Now().Format(util.Date_Full_Layout),
+		"d_updatetime": time.Now().Format(util.Date_Full_Layout),
+	}
+	u.InsertGlobalMysqlData(u.T_dwd_f_bid_detail, info, tmpid)
+}
+
+func taskAtts(tmp map[string]interface{}) {
+	tmpid := mongodb.BsonIdToSId(tmp["_id"])
+	f_baseInfo := map[string]interface{}{}
+	attach_text := util.ObjToMap(tmp["attach_text"])
+	if projectinfo := util.ObjToMap(tmp["projectinfo"]); projectinfo != nil {
+		attachments := util.ObjToMap((*projectinfo)["attachments"])
+		for index, attr := range *attachments {
+			if at, ok := attr.(map[string]interface{}); ok {
+				if util.ObjToString(at["fid"]) != "" {
+					ftype := ""
+					for _, s := range u.FileTypeArr {
+						ft := strings.ToLower(util.ObjToString(tmp["ftype"]))
+						if strings.Contains(ft, s) {
+							ftype = s
+							break
+						}
+					}
+					f_baseInfo["s_info_id"] = tmpid
+					filename := util.ObjToString(at["filename"])
+					filename = regClean.ReplaceAllString(filename, "")
+					f_baseInfo["s_file_name"] = filename
+					f_baseInfo["s_file_url"] = util.ObjToString(at["org_url"])
+					f_baseInfo["s_file_size"] = util.ObjToString(at["size"])
+					f_baseInfo["s_file_suffix"] = ftype
+					f_baseInfo["s_file_oss_url"] = util.ObjToString(at["fid"])
+					f_baseInfo["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+					f_id := u.InsertGlobalMysqlData(u.T_dwd_f_bid_file_baseinfo, f_baseInfo, tmpid)
+					if f_id > 0 && util.IntAll(index) > 0 && attach_text != nil {
+						att_key := fmt.Sprintf("%d", util.IntAll(index)-1)
+						if att_info := util.ObjToMap((*attach_text)[att_key]); att_info != nil {
+							taskAttsAttach(*att_info, tmpid, f_id)
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+func taskAttsAttach(att_info map[string]interface{}, tmpid string, f_id int64) {
+	for _, v := range att_info {
+		if att, b := v.(map[string]interface{}); b {
+			info := map[string]interface{}{}
+			info["s_info_id"] = tmpid
+			info["s_file_id"] = f_id
+			info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+			attach_url := util.ObjToString(att["attach_url"])
+			if attach_url != "" {
+				//bs := OssGetObject(attach_url)
+				//if utf8.RuneCountInString(bs) > 100000 {
+				//	bs = string(([]rune(bs))[:100000])
+				//}
+				//info["s_file_text"] = bs
+			}
+			u.InsertGlobalMysqlData(u.T_dwd_f_bid_file_text, info, tmpid)
+		}
+	}
+}
+
+func taskIntent(tmp map[string]interface{}) {
+	procurementlist := u.IsMarkInterfaceMap(tmp["procurementlist"])
+	tmpid := mongodb.BsonIdToSId(tmp["_id"])
+	for _, p1 := range procurementlist {
+		info := map[string]interface{}{}
+		info["s_info_id"] = tmpid
+		if p1["itemname"] != nil {
+			info["s_intention_name"] = p1["itemname"]
+		}
+		if p1["projectscope"] != nil {
+			info["s_intention_demand"] = p1["projectscope"]
+		}
+		if p1["item"] != nil {
+			info["s_item"] = p1["item"]
+		}
+		if p1["totalprice"] != nil {
+			info["f_totalprice"] = p1["totalprice"]
+		}
+		if p1["expurasingtime"] != nil {
+			info["s_expurasingtime"] = p1["expurasingtime"]
+		}
+		if p1["reserved_amount"] != nil {
+			info["s_reserved_amount"] = p1["reserved_amount"]
+		}
+		if b := util.ObjToString(tmp["buyer"]); b != "" {
+			if code := u.GetNameId(b); code != "" {
+				info["s_buyer_id"] = code
+			}
+		}
+		info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+		u.InsertGlobalMysqlData(u.T_dwd_f_bid_intention_baseinfo, info, tmpid)
+	}
+}
+
+func taskPackage(tmp map[string]interface{}) {
+	tmpid := mongodb.BsonIdToSId(tmp["_id"])
+	//筛选分包
+	packages := FilterPackageInfos(tmp)
+	if len(packages) <= 1 { //单包···标讯本身
+		baseInfo := CPBaseInfoFromBidding(tmp, tmpid)
+		pid := u.InsertGlobalMysqlData(u.T_dwd_f_bid_package_baseinfo, baseInfo, tmpid)
+		if pid > 0 {
+			//投标人信息
+			CPBidderBiddingBaseInfo(tmp, tmpid, pid)
+			//标的物信息
+			new_purlist := CPBiddingPackageGoodsBaseInfo(tmp, tmpid, pid)
+			for _, v := range new_purlist {
+				u.InsertGlobalMysqlData(u.T_dwd_f_bid_package_goods_baseinfo, v, tmpid)
+			}
+		}
+	} else { //多包...具体源信息
+		for k, v := range packages {
+			baseInfo := CPBaseInfoFromPackage(v, tmpid)
+			pid := u.InsertGlobalMysqlData(u.T_dwd_f_bid_package_baseinfo, baseInfo, tmpid)
+			if pid > 0 { //投标人信息
+				if k == 0 { //标的物信息
+					CPBidderPackageBaseInfo(v, tmp, tmpid, pid, true)
+					new_purlist := CPBiddingPackageGoodsBaseInfo(tmp, tmpid, pid)
+					for _, v1 := range new_purlist {
+						u.InsertGlobalMysqlData(u.T_dwd_f_bid_package_goods_baseinfo, v1, tmpid)
+					}
+				} else {
+					CPBidderPackageBaseInfo(v, tmp, tmpid, pid, false)
+				}
+			}
+		}
+	}
+}
+
+func BinarySearch(s []string, k string) int {
+	sort.Strings(s)
+	lo, hi := 0, len(s)-1
+	for lo <= hi {
+		m := (lo + hi) >> 1
+		if s[m] < k {
+			lo = m + 1
+		} else if s[m] > k {
+			hi = m - 1
+		} else {
+			return m
+		}
+	}
+	return -1
+}

data_mgo_to_tidb/bidding/oss.go

@@ -0,0 +1,55 @@
+package bidding
+
+import (
+	"fmt"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"io"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"os"
+)
+
+var (
+	ossEndpoint = "oss-cn-beijing-internal.aliyuncs.com" //http://oss-cn-beijing.aliyuncs.com"
+	//ossEndpointTest    = "topjy.oss-cn-beijing.aliyuncs.com"
+	ossEndpointTest    = "oss-cn-beijing.aliyuncs.com"
+	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
+	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
+	ossBucketName      = "topjy"
+	ossclient          *oss.Client
+)
+
+func InitOss(isTest bool) {
+	if isTest {
+		ossEndpoint = ossEndpointTest //测试服务器~阿里云
+	}
+	client, err := oss.New(ossEndpoint, ossAccessKeyId, ossAccessKeySecret)
+	if err != nil {
+		fmt.Println("Error:", err)
+		os.Exit(-1)
+	}
+	ossclient = client
+}
+
+func OssGetObject(objectName string) string {
+	util.Catch()
+	// 获取存储空间。
+	bucket, err := ossclient.Bucket(ossBucketName)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+
+	// 下载文件到流。
+	body, err := bucket.GetObject(objectName)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+	defer body.Close()
+	data, err := io.ReadAll(body)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return ""
+	}
+	return string(data)
+}

data_mgo_to_tidb/bidding/pkg.go

@@ -0,0 +1,226 @@
+package bidding
+
+import (
+	u "data_tidb/util"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+// 标讯分包基本信息-标讯版
+func CPBaseInfoFromBidding(tmp map[string]interface{}, tmpid string) map[string]interface{} {
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	info["s_packagename"] = util.ObjToString(tmp["projectname"])
+	if tmp["budget"] != nil {
+		if budget := util.Float64All(tmp["budget"]); budget < 1000000000.0 {
+			info["f_budget"] = budget
+		}
+	}
+	if tmp["bidamount"] != nil {
+		if bidamount := util.Float64All(tmp["bidamount"]); bidamount < 1000000000.0 {
+			info["f_bidamount"] = bidamount
+		}
+	}
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	return info
+}
+
+// 投标人相关信息-标讯版
+func CPBidderBiddingBaseInfo(tmp map[string]interface{}, tmpid string, pid int64) {
+	s_winner := util.ObjToString(tmp["s_winner"])
+	s_winner = cleaname(s_winner)
+	if s_winner == "" {
+		return
+	}
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	info["i_package_id"] = pid
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	wins := strings.Split(s_winner, ",")
+	bidder_id, bidder_name, other_bidder_id := getWinsNameId(wins)
+	info["s_bidder_id"] = bidder_id
+	info["i_is_winner"] = 1
+	if len(wins) > 1 {
+		info["s_bidder_name"] = bidder_name
+		info["s_other_bidder_name"] = s_winner
+		info["s_other_bidder_id"] = other_bidder_id
+		info["i_is_bidders"] = 1
+	}
+	//保存服务...
+	u.InsertGlobalMysqlData(u.T_dwd_f_bid_package_bidder_baseinfo, info, tmpid)
+	//候选人相关情况
+	o_win_ids, o_win_names := getWinerorder(tmp, wins[0])
+	for k, v := range o_win_ids {
+		o_info := map[string]interface{}{}
+		o_info["s_info_id"] = tmpid
+		o_info["i_package_id"] = pid
+		o_info["s_bidder_id"] = v
+		o_info["s_bidder_name"] = o_win_names[k]
+		o_info["i_is_winner"] = 0
+		o_info["d_updatetime"] = time.Now().Format(util.Date_Full_Layout)
+		//保存服务...
+		u.InsertGlobalMysqlData(u.T_dwd_f_bid_package_bidder_baseinfo, info, tmpid)
+	}
+}
+
+// 标讯分包标的物信息
+func CPBiddingPackageGoodsBaseInfo(tmp map[string]interface{}, tmpid string, pid int64) []map[string]interface{} {
+	purchasinglist := u.IsMarkInterfaceMap(tmp["purchasinglist"])
+	new_purlist := []map[string]interface{}{}
+	for _, v := range purchasinglist {
+		info := map[string]interface{}{}
+		info["s_info_id"] = tmpid
+		info["i_package_id"] = pid
+		info["s_itemname"] = u.CleanLength(util.ObjToString(v["itemname"]), 255)
+		info["s_brand"] = u.CleanLength(util.ObjToString(v["brandname"]), 255)
+		info["s_model"] = u.CleanLength(util.ObjToString(v["model"]), 255)
+		if number := util.IntAll(v["number"]); number > 0 && number < 1000000000 {
+			info["i_num"] = number
+		}
+		if unitprice := util.Float64All(v["unitprice"]); unitprice > 0.0 && unitprice < 1000000000.0 {
+			info["f_unit"] = unitprice
+		}
+		if totalprice := util.Float64All(v["totalprice"]); totalprice > 0.0 && totalprice < 1000000000.0 {
+			info["f_totalprice"] = totalprice
+		}
+		info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+		new_purlist = append(new_purlist, info)
+	}
+	return new_purlist
+}
+
+// 标讯分包基本信息-分包版
+func CPBaseInfoFromPackage(pinfos map[string]interface{}, tmpid string) map[string]interface{} {
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	for k, v := range pinfos {
+		info["s_packagecode"] = k
+		if pkg := util.ObjToMap(v); pkg != nil {
+			info["s_packagename"] = util.ObjToString((*pkg)["name"])
+			if (*pkg)["budget"] != nil {
+				info["f_budget"] = util.Float64All((*pkg)["budget"])
+			}
+			if (*pkg)["bidamount"] != nil {
+				info["f_bidamount"] = util.Float64All((*pkg)["bidamount"])
+			}
+			return info
+		}
+	}
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	return info
+}
+
+// 投标人相关信息-分包版
+func CPBidderPackageBaseInfo(pinfos map[string]interface{}, tmp map[string]interface{}, tmpid string, pid int64, isF bool) {
+	for _, v := range pinfos {
+		pinfo := *util.ObjToMap(v)
+		winner := util.ObjToString(pinfo["winner"])
+		info := map[string]interface{}{}
+		info["s_info_id"] = tmpid
+		info["i_package_id"] = pid
+		info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+		wins := strings.Split(winner, ",")
+		bidder_id, bidder_name, other_bidder_id := getWinsNameId(wins)
+		info["s_bidder_id"] = bidder_id
+		info["i_is_winner"] = 1
+		if len(wins) > 1 {
+			info["s_bidder_name"] = bidder_name
+			info["s_other_bidder_name"] = winner
+			info["s_other_bidder_id"] = other_bidder_id
+			info["i_is_bidders"] = 1
+		}
+		//保存服务...
+		u.InsertGlobalMysqlData(u.T_dwd_f_bid_package_bidder_baseinfo, info, tmpid)
+		//候选人相关情况
+		if isF {
+			o_win_ids, o_win_names := getWinerorder(tmp, wins[0])
+			for k, v := range o_win_ids {
+				o_info := map[string]interface{}{}
+				o_info["s_info_id"] = tmpid
+				o_info["s_bidder_id"] = v
+				o_info["s_bidder_name"] = o_win_names[k]
+				o_info["i_is_winner"] = 0
+				o_info["d_updatetime"] = time.Now().Format(util.Date_Full_Layout)
+				//保存服务...
+				u.InsertGlobalMysqlData(u.T_dwd_f_bid_package_bidder_baseinfo, info, tmpid)
+			}
+		}
+	}
+}
+
+// 获取所有投标人的相关标识
+func getWinsNameId(wins []string) (string, string, string) {
+	win_id, win_name, win_ids := "", "", []string{}
+	for k, v := range wins {
+		code := u.GetNameId(v)
+		if k == 0 {
+			win_id = code
+			win_name = v
+		}
+		win_ids = append(win_ids, code)
+	}
+	return win_id, win_name, strings.Join(win_ids, ",")
+}
+
+// 获取候选人相关信息
+func getWinerorder(tmp map[string]interface{}, win string) ([]string, []string) {
+	o_win_names, o_win_ids := []string{}, []string{}
+	winnerorder := u.IsMarkInterfaceMap(tmp["winnerorder"])
+	for k, v := range winnerorder {
+		if k >= 5 {
+			break
+		}
+		entname := util.ObjToString(v["entname"])
+		if entname != win {
+			o_win_ids = append(o_win_ids, u.GetNameId(entname))
+			o_win_names = append(o_win_names, entname)
+		}
+	}
+	return o_win_ids, o_win_names
+}
+
+// 分包信息清洗与过滤...
+func FilterPackageInfos(tmp map[string]interface{}) []map[string]interface{} {
+	new_packages := []map[string]interface{}{}
+	s_winner := util.ObjToString(tmp["s_winner"])
+	if s_winner == "" {
+		return new_packages
+	}
+	winArr := strings.Split(s_winner, ",")
+	if packages := util.ObjToMap(tmp["package"]); packages != nil {
+		for k, v := range *packages {
+			if pkg := util.ObjToMap(v); pkg != nil && k != "" {
+				if win := util.ObjToString((*pkg)["winner"]); win != "" {
+					if IsExistsWins(winArr, win) {
+						new_packages = append(new_packages, map[string]interface{}{k: v})
+					}
+				}
+			}
+		}
+	}
+	return new_packages
+}
+
+// 是否存在单位
+func IsExistsWins(winArr []string, win string) bool {
+	for _, v := range winArr {
+		if v == win {
+			return true
+		}
+	}
+	return false
+}
+
+// 简易清洗...
+func cleaname(name string) string {
+	nameArr := strings.Split(name, ",")
+	new_nameArr := []string{}
+	for _, v := range nameArr {
+		if utf8.RuneCountInString(v) < 50 {
+			new_nameArr = append(new_nameArr, v)
+		}
+	}
+	return strings.Join(new_nameArr, ",")
+}

data_mgo_to_tidb/common.toml

@@ -0,0 +1,46 @@
+[udp]
+locport = ":1166"
+
+[db]
+[db.mysql]
+addr = "192.168.3.217:4000"
+dbname = "global_common_data"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+
+[db.mongob]
+addr = "192.168.3.166:27082"
+dbname = "qfw"
+size = 5
+user = ""
+password = ""
+
+[db.mongop]
+addr = "192.168.3.166:27082"
+dbname = "mixdata"
+size = 5
+user = ""
+password = ""
+
+[mail]
+send = false
+to = "wangjianghan@topnet.net.cn"
+api = "http://172.17.145.179:19281/_send/_mail"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"

data_mgo_to_tidb/config/conf.go

@@ -0,0 +1,84 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	// Conf crocodile conf
+	Conf *conf
+)
+
+// Init Config
+func Init(conf string) {
+	_, err := toml.DecodeFile(conf, &Conf)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+}
+
+type conf struct {
+	Udp  udp
+	DB   db
+	Mail mail
+	Log  log
+}
+
+type udp struct {
+	LocPort string
+}
+
+type mail struct {
+	Send bool
+	To   string
+	Api  string
+}
+
+// Log Config
+type log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type db struct {
+	MongoB mgo
+	MongoP mgo
+	Mysql  mysql
+}
+
+type mgo struct {
+	Addr     string
+	Dbname   string
+	Size     int
+	User     string
+	Password string
+}
+
+type mysql struct {
+	Addr     string
+	Dbname   string
+	Size     int
+	User     string
+	Password string
+}
+
+type duration struct {
+	time.Duration
+}
+
+// UnmarshalText parse 10s to time.Time
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}

data_mgo_to_tidb/config/conf_test.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var confs = `# log
+[log]
+logpath = ""
+maxsize = 10
+compress = true
+maxage =  7
+maxbackups = 10
+loglevel  = "info"
+format = "text"
+
+[serve]
+grpcAddr = "192.168.3.12:10021"
+udpPort = "1782"
+
+[db]
+[db.mongo]
+addr = "192.168.3.207:27092"
+dbname = "qfw"
+size = 10
+user = ""
+password = ""
+[db.mongo1]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+size = 5
+user = ""
+password = ""
+
+[db.es]
+addr = "http://192.168.3.206:9800"
+size = 5
+indexm = "medical_institution_v1"
+typem = "medical_institution"
+indexs = "supplier_product_v1"
+types = "supplier_product"
+
+`
+
+func TestInit(t *testing.T) {
+	testfile := "/tmp/crocodile.toml"
+	ioutil.WriteFile(testfile, []byte(confs), 0644)
+	Init(testfile)
+	t.Logf("%+v", Conf.Serve.GrpcAddr)
+	os.Remove(testfile)
+}

data_mgo_to_tidb/go.mod

@@ -0,0 +1,17 @@
+module data_tidb
+
+go 1.16
+
+require (
+	github.com/BurntSushi/toml v1.2.0
+	github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible
+	github.com/kr/text v0.2.0 // indirect
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
+	github.com/robfig/cron v1.2.0
+	go.mongodb.org/mongo-driver v1.10.3
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/zap v1.23.0
+	golang.org/x/time v0.4.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061
+)

data_mgo_to_tidb/go.sum

@@ -0,0 +1,218 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible h1:so4m5rRA32Tc5GgKg/5gKUu0CRsYmVO3ThMP6T3CwLc=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.1+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
+github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.10.3 h1:XDQEvmh6z1EUsXuIkXE9TaVeqHw6SwS1uf93jFs0HBA=
+go.mongodb.org/mongo-driver v1.10.3/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
+go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY=
+golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061 h1:xfmoVsDEqVv1XzAVxgGEDi+W9ojHBJc6OmTbg3b1tP0=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

data_mgo_to_tidb/main.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	u "data_tidb/util"
+	"github.com/robfig/cron"
+	"strconv"
+	"time"
+)
+
+func init() {
+	u.InitInfo() //初始化...
+	//u.InitSaveService()//批量保存...
+}
+
+func main() {
+	//标讯信息 655ff28faf7d908a8c546ec3
+	//bidding.TaskBidding("100000000000000000000000", "655ff28faf7d908a8c546ec3")
+	//定时增量...
+	cr := cron.New()
+	cr.AddFunc("0 0 9 ? * *", func() {
+		if gteid, ltid := getCurTimePiInfo(); gteid != "" && ltid != "" {
+			//bidding.TaskBidding(gteid, ltid)
+		}
+	})
+	cr.Start()
+
+	ch := make(chan bool, 1)
+	<-ch
+}
+
+func getCurTimePiInfo() (string, string) {
+	now := time.Now()
+	start := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local).Unix()
+	end := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
+	gteid := strconv.FormatInt(start, 16) + "0000000000000000"
+	ltid := strconv.FormatInt(end, 16) + "0000000000000000"
+	return gteid, ltid
+}

data_mgo_to_tidb/mark

@@ -0,0 +1,45 @@
+[udp]
+locport = ":1166"
+[db]
+
+[db.mysql]
+addr = "172.17.4.242:4000"
+dbname = "global_common_data"
+size = 5
+user = "zhengkun"
+password = "Zk#20220824"
+
+[db.mongob]
+addr = "172.17.189.140:27080,172.17.189.141:27081"
+dbname = "qfw"
+size = 5
+user = "zhengkun"
+password = "zk@123123"
+[db.mongop]
+addr = "172.17.189.140:27080,172.17.189.141:27081"
+dbname = "mixdata"
+size = 5
+user = "zhengkun"
+password = "zk@123123"
+
+[mail]
+send = false
+to = "wangjianghan@topnet.net.cn"
+api = "http://172.17.145.179:19281/_send/_mail"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"

data_mgo_to_tidb/project.go

@@ -0,0 +1,975 @@
+package main
+
+//
+//import (
+//	"data_tidb/config"
+//	"fmt"
+//	"go.mongodb.org/mongo-driver/bson"
+//	"go.uber.org/zap"
+//	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+//	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+//	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+//	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+//	"strings"
+//	"sync"
+//	"time"
+//)
+//
+//func taskP() {
+//	sess := MongoP.GetMgoConn()
+//	defer MongoP.DestoryMongoConn(sess)
+//
+//	ch := make(chan bool, 20)
+//	wg := &sync.WaitGroup{}
+//
+//	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("64e5a63855d5406905c574e6")}
+//	query := sess.DB(config.Conf.DB.MongoP.Dbname).C("projectset_20230407").Find(nil).Sort("-_id").Skip(100000).Iter()
+//	count := 0
+//	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+//		if count%20000 == 0 {
+//			log.Info(fmt.Sprintf("current --- %d", count))
+//		}
+//		ch <- true
+//		wg.Add(1)
+//		go func(tmp map[string]interface{}) {
+//			defer func() {
+//				<-ch
+//				wg.Done()
+//			}()
+//			//taskPro(tmp)
+//			//taskBusiness(tmp)
+//			//taskProTag(tmp)
+//			//taskRelation(tmp)
+//
+//		}(tmp)
+//		tmp = make(map[string]interface{})
+//	}
+//	wg.Wait()
+//	log.Info(fmt.Sprintf("over --- %d", count))
+//}
+//
+//func taskPAdd(pici int64) {
+//	sess := MongoP.GetMgoConn()
+//	defer MongoP.DestoryMongoConn(sess)
+//
+//	ch := make(chan bool, 20)
+//	wg := &sync.WaitGroup{}
+//
+//	q := bson.M{"pici": bson.M{"$gt": pici}}
+//	query := sess.DB(config.Conf.DB.MongoP.Dbname).C("projectset_20230407").Find(q).Iter()
+//	count := 0
+//	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+//		if count%20000 == 0 {
+//			log.Info(fmt.Sprintf("current --- %d", count))
+//		}
+//		ch <- true
+//		wg.Add(1)
+//		go func(tmp map[string]interface{}) {
+//			defer func() {
+//				<-ch
+//				wg.Done()
+//			}()
+//			//taskPro(tmp)
+//			//taskBusiness(tmp)
+//			//taskProTag(tmp)
+//			//taskRelation2(tmp)
+//
+//		}(tmp)
+//		tmp = make(map[string]interface{})
+//	}
+//	wg.Wait()
+//	log.Info(fmt.Sprintf("over --- %d", count))
+//}
+//
+//var BidStatus = map[string]int{
+//	"预告": 0,
+//	"拟建": 1,
+//	"招标": 2,
+//	"中标": 3,
+//	"成交": 4,
+//	"废标": 5,
+//	"流标": 6,
+//	"合同": 7,
+//	"其它": 8,
+//}
+//var BidType = map[string]int{
+//	"招标": 0,
+//	"邀标": 1,
+//	"单一": 2,
+//	"竞价": 3,
+//	"竞谈": 4,
+//	"询价": 5,
+//}
+//
+//// @Description 基础信息
+//// @Author J 2022/9/22 18:32
+//func taskPro(tmp map[string]interface{}) {
+//	saveM := make(map[string]interface{})
+//	for _, f := range ProField {
+//		if f == "projectid" {
+//			saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
+//		} else if f == "area_code" {
+//			if tmp["area"] != nil {
+//				saveM[f] = AreaCode[util.ObjToString(tmp["area"])]
+//			}
+//		} else if f == "city_code" {
+//			if tmp["area"] != nil && tmp["city"] != nil {
+//				c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"])
+//				saveM[f] = AreaCode[c]
+//			}
+//		} else if f == "district_code" {
+//			if tmp["area"] != nil && tmp["city"] != nil && tmp["district"] != nil {
+//				c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"]) + "," + util.ObjToString(tmp["district"])
+//				saveM[f] = AreaCode[c]
+//			}
+//		} else if f == "updatetime" {
+//			saveM[f] = time.Now().Format(util.Date_Full_Layout)
+//		} else if f == "buyerclass_code" {
+//			if obj := util.ObjToString(tmp["buyerclass"]); obj != "" {
+//				saveM[f] = BuyerCode[obj]
+//			}
+//		} else if f == "firsttime" || f == "zbtime" || f == "jgtime" || f == "lasttime" || f == "bidopentime" || f == "createtime" {
+//			if tmp[f] != nil && util.Int64All(tmp[f]) > 0 {
+//				t := util.Int64All(tmp[f])
+//				saveM[f] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+//			}
+//		} else if f == "bidstatus" {
+//			if b := util.ObjToString(tmp[f]); b != "" {
+//				tmp[f] = BidStatus[b]
+//			}
+//		} else if f == "bidtype" {
+//			if b := util.ObjToString(tmp[f]); b != "" {
+//				tmp[f] = BidType[b]
+//			}
+//		} else if f == "multipackage" {
+//			if tmp[f] == nil {
+//				saveM[f] = 0
+//			} else {
+//				saveM[f] = tmp[f]
+//			}
+//		} else if f == "buyer_id" {
+//			if b := util.ObjToString(tmp["buyer"]); b != "" {
+//				if code := redis.GetStr("qyxy_id", b); code != "" {
+//					saveM[f] = code
+//				}
+//			}
+//		} else if f == "agency_id" {
+//			if b := util.ObjToString(tmp["agency"]); b != "" {
+//				if code := redis.GetStr("qyxy_id", b); code != "" {
+//					saveM[f] = code
+//				}
+//			}
+//		} else {
+//			if tmp[f] != nil {
+//				if ProVMap[f] != nil {
+//					saveM[f], _ = verifyF(f, tmp[f], ProVMap[f])
+//				} else {
+//					saveM[f] = tmp[f]
+//				}
+//			}
+//		}
+//	}
+//	saveProPool <- saveM
+//}
+//
+//// @Description 项目业务表
+//// @Author J 2022/9/30 13:40
+//func taskBusiness(tmp map[string]interface{}) {
+//	warr := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
+//	if BinarySearch(warr, util.ObjToString(tmp["winner"])) == -1 {
+//		warr = append(warr, util.ObjToString(tmp["winner"]))
+//	}
+//	for _, s := range warr {
+//		saveM := make(map[string]interface{})
+//		for _, f := range ProBusField {
+//			if f == "projectid" {
+//				saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
+//			} else if f == "area_code" {
+//				if tmp["area"] != nil {
+//					saveM[f] = AreaCode[util.ObjToString(tmp["area"])]
+//				}
+//			} else if f == "city_code" {
+//				if tmp["area"] != nil && tmp["city"] != nil {
+//					c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"])
+//					saveM[f] = AreaCode[c]
+//				}
+//			} else if f == "district_code" {
+//				if tmp["area"] != nil && tmp["city"] != nil && tmp["district"] != nil {
+//					c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"]) + "," + util.ObjToString(tmp["district"])
+//					saveM[f] = AreaCode[c]
+//				}
+//			} else if f == "updatetime" {
+//				saveM[f] = time.Now().Format(util.Date_Full_Layout)
+//			} else if f == "buyerclass_code" {
+//				if obj := util.ObjToString(tmp["buyerclass"]); obj != "" {
+//					saveM[f] = BuyerCode[obj]
+//				}
+//			} else if f == "firsttime" || f == "zbtime" || f == "jgtime" || f == "lasttime" || f == "bidopentime" || f == "createtime" {
+//				if tmp[f] != nil && util.Int64All(tmp[f]) > 0 {
+//					t := util.Int64All(tmp[f])
+//					saveM[f] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+//				}
+//			} else if f == "bidstatus" {
+//				if b := util.ObjToString(tmp[f]); b != "" {
+//					tmp[f] = BidStatus[b]
+//				}
+//			} else if f == "bidtype" {
+//				if b := util.ObjToString(tmp[f]); b != "" {
+//					tmp[f] = BidType[b]
+//				}
+//			} else if f == "buyer_id" {
+//				if b := util.ObjToString(tmp["buyer"]); b != "" {
+//					saveM["buyer"] = b
+//					if code := redis.GetStr("qyxy_id", b); code != "" {
+//						saveM[f] = code
+//					}
+//				}
+//			} else if f == "agency_id" {
+//				if b := util.ObjToString(tmp["agency"]); b != "" {
+//					saveM["agency"] = b
+//					if code := redis.GetStr("qyxy_id", b); code != "" {
+//						saveM[f] = code
+//					}
+//				}
+//			} else if f == "winner_id" {
+//				if s != "" {
+//					saveM["winner"] = s
+//					if code := redis.GetStr("qyxy_id", s); code != "" {
+//						saveM[f] = code
+//					}
+//				}
+//			} else {
+//				if tmp[f] != nil {
+//					if ProBusVMap[f] != nil {
+//						saveM[f], _ = verifyF(f, tmp[f], ProBusVMap[f])
+//					} else {
+//						saveM[f] = tmp[f]
+//					}
+//				}
+//			}
+//		}
+//		saveProbPool <- saveM
+//	}
+//
+//}
+//
+//// @Description 项目信息标签
+//// @Author J 2022/9/30 13:54
+//func taskProTag(tmp map[string]interface{}) {
+//	id := mongodb.BsonIdToSId(tmp["_id"])
+//	if topArr, ok := tmp["topscopeclass"].([]interface{}); ok {
+//		for _, i2 := range topArr {
+//			tclass := regLetter.ReplaceAllString(util.ObjToString(i2), "") // 去除字母
+//			code := TopScopeCode[tclass]
+//			saveProTagPool <- map[string]interface{}{"projectid": id, "labelcode": "1", "labelvalues": code, "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
+//		}
+//	}
+//	if subArr, ok := tmp["subscopeclass"].([]interface{}); ok {
+//		for _, i2 := range subArr {
+//			sc := strings.Split(util.ObjToString(i2), "_")
+//			code := SubScopeCode[sc[1]]
+//			saveProTagPool <- map[string]interface{}{"projectid": id, "labelcode": "2", "labelvalues": code, "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
+//		}
+//	}
+//}
+//
+//// @Description 关系表
+//// @Author J 2022/9/30 13:56
+//func taskRelation(tmp map[string]interface{}) {
+//	pid := mongodb.BsonIdToSId(tmp["_id"])
+//	if tmp["ids"] == nil {
+//		log.Info("taskRelation ids err", zap.Any("id", pid))
+//		return
+//	}
+//	ids := util.ObjArrToStringArr(tmp["ids"].([]interface{}))
+//	lid := ids[len(ids)-1]
+//
+//	//if b := util.ObjToString(tmp["buyer"]); b != "" {
+//	//	saveM := make(map[string]interface{})
+//	//
+//	//	saveM["projectid"] = pid
+//	//	saveM["infoid"] = lid
+//	//	saveM["identity_type"] = 1
+//	//	saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
+//	//	if code := redis.GetStr("qyxy_id", b); code != "" {
+//	//		saveM["name_id"] = code
+//	//		if util.ObjToString(tmp["buyertel"]) != "" {
+//	//			q := make(map[string]interface{})
+//	//			q["name_id"] = code
+//	//			q["identity_type"] = 1
+//	//			q["contact_tel"] = util.ObjToString(tmp["buyertel"])
+//	//			if util.ObjToString(tmp["buyerperson"]) != "" {
+//	//				q["contact_name"] = util.ObjToString(tmp["buyerperson"])
+//	//			}
+//	//			cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//	//			if cinfo != nil && len(*cinfo) > 0 {
+//	//				saveM["contact_id"] = (*cinfo)["id"]
+//	//				saveRelationPool <- saveM
+//	//			}
+//	//		}
+//	//	}
+//	//}
+//
+//	//if a := util.ObjToString(tmp["agency"]); a != "" {
+//	//	saveM := make(map[string]interface{})
+//	//	saveM["projectid"] = pid
+//	//	saveM["infoid"] = lid
+//	//	saveM["identity_type"] = 4
+//	//	saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
+//	//	if code := redis.GetStr("qyxy_id", a); code != "" {
+//	//		saveM["name_id"] = code
+//	//		if util.ObjToString(tmp["agencytel"]) != "" {
+//	//			q := make(map[string]interface{})
+//	//			q["name_id"] = code
+//	//			q["identity_type"] = 4 // 100
+//	//			q["contact_tel"] = util.ObjToString(tmp["agencytel"])
+//	//			if util.ObjToString(tmp["agencyperson"]) != "" {
+//	//				q["contact_name"] = util.ObjToString(tmp["agencyperson"])
+//	//			}
+//	//			cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//	//			if cinfo != nil && len(*cinfo) > 0 {
+//	//				saveM["contact_id"] = (*cinfo)["id"]
+//	//				saveRelationPool <- saveM
+//	//			}
+//	//		}
+//	//	}
+//	//}
+//
+//	for _, item := range tmp["list"].([]interface{}) {
+//		item1 := item.(map[string]interface{})
+//		sw := util.ObjToString(item1["s_winner"])
+//		if !strings.Contains(sw, ",") {
+//			if code := redis.GetStr("qyxy_id", sw); code != "" {
+//				saveM := make(map[string]interface{})
+//				saveM["projectid"] = pid
+//				saveM["infoid"] = lid
+//				saveM["identity_type"] = 2
+//				saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
+//				saveM["name_id"] = code
+//				if util.ObjToString(item1["winnertel"]) != "" {
+//					q := make(map[string]interface{})
+//					q["name_id"] = code
+//					q["identity_type"] = 2 // 010
+//					q["contact_tel"] = util.ObjToString(item1["winnertel"])
+//					if util.ObjToString(item1["winnerperson"]) != "" {
+//						q["contact_name"] = util.ObjToString(item1["winnerperson"])
+//					}
+//					cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//					if cinfo != nil && len(*cinfo) > 0 {
+//						saveM["contact_id"] = (*cinfo)["id"]
+//						saveRelationPool <- saveM
+//					}
+//				}
+//			}
+//		}
+//	}
+//}
+//
+//func taskRelation2(tmp map[string]interface{}) {
+//
+//	pid := mongodb.BsonIdToSId(tmp["_id"])
+//	if tmp["ids"] == nil {
+//		log.Info("taskRelation ids err", zap.Any("id", pid))
+//		return
+//	}
+//	info := MysqlTool.Find("dws_f_bpmc_relation", bson.M{"projectid": pid}, "", "", -1, -1)
+//	if len(*info) > 0 {
+//
+//	} else {
+//		ids := util.ObjArrToStringArr(tmp["ids"].([]interface{}))
+//		lid := ids[len(ids)-1]
+//
+//		if b := util.ObjToString(tmp["buyer"]); b != "" {
+//			saveM := make(map[string]interface{})
+//			for _, f := range RelationField {
+//				if f == "projectid" {
+//					saveM[f] = pid
+//				} else if f == "infoid" {
+//					saveM[f] = lid
+//				} else if f == "name_id" {
+//					if code := redis.GetStr("qyxy_id", b); code != "" {
+//						saveM[f] = code
+//						if util.ObjToString(tmp["buyertel"]) != "" {
+//							q := make(map[string]interface{})
+//							q["name_id"] = code
+//							q["identity_type"] = 1
+//							q["contact_tel"] = util.ObjToString(tmp["buyertel"])
+//							if util.ObjToString(tmp["buyerperson"]) != "" {
+//								q["contact_name"] = util.ObjToString(tmp["buyerperson"])
+//							}
+//							cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//							if cinfo != nil && len(*cinfo) > 0 {
+//								saveM["contact_id"] = (*cinfo)["id"]
+//							}
+//						}
+//					}
+//				} else if f == "identity_type" {
+//					saveM[f] = 1 // 001
+//				} else if f == "createtime" {
+//					saveM[f] = time.Now().Format(util.Date_Full_Layout)
+//				}
+//			}
+//			saveRelationPool <- saveM
+//		}
+//
+//		if a := util.ObjToString(tmp["agency"]); a != "" {
+//			saveM := make(map[string]interface{})
+//			for _, f := range RelationField {
+//				if f == "projectid" {
+//					saveM[f] = pid
+//				} else if f == "infoid" {
+//					saveM[f] = lid
+//				} else if f == "name_id" {
+//					if code := redis.GetStr("qyxy_id", a); code != "" {
+//						saveM[f] = code
+//						if util.ObjToString(tmp["buyertel"]) != "" {
+//							q := make(map[string]interface{})
+//							q["name_id"] = code
+//							q["identity_type"] = 4
+//							q["contact_tel"] = util.ObjToString(tmp["agencytel"])
+//							if util.ObjToString(tmp["agencyperson"]) != "" {
+//								q["contact_name"] = util.ObjToString(tmp["agencyperson"])
+//							}
+//							cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//							if cinfo != nil && len(*cinfo) > 0 {
+//								saveM["contact_id"] = (*cinfo)["id"]
+//							}
+//						}
+//					}
+//				} else if f == "identity_type" {
+//					saveM[f] = 4 // 100
+//				} else if f == "createtime" {
+//					saveM[f] = time.Now().Format(util.Date_Full_Layout)
+//				}
+//			}
+//			saveRelationPool <- saveM
+//		}
+//
+//		warr := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
+//		if BinarySearch(warr, util.ObjToString(tmp["winner"])) == -1 {
+//			warr = append(warr, util.ObjToString(tmp["winner"]))
+//		}
+//		for _, ws := range warr {
+//			saveM := make(map[string]interface{})
+//			for _, f := range RelationField {
+//				if f == "projectid" {
+//					saveM[f] = pid
+//				} else if f == "infoid" {
+//					saveM[f] = lid
+//				} else if f == "name_id" {
+//					if code := redis.GetStr("qyxy_id", ws); code != "" {
+//						saveM[f] = code
+//						if util.ObjToString(tmp["buyertel"]) != "" {
+//							q := make(map[string]interface{})
+//							q["name_id"] = code
+//							q["identity_type"] = 2
+//							q["contact_tel"] = util.ObjToString(tmp["winnertel"])
+//							if util.ObjToString(tmp["winnerperson"]) != "" {
+//								q["contact_name"] = util.ObjToString(tmp["winnerperson"])
+//							}
+//							cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//							if cinfo != nil && len(*cinfo) > 0 {
+//								saveM["contact_id"] = (*cinfo)["id"]
+//							}
+//						}
+//					}
+//				} else if f == "identity_type" {
+//					saveM[f] = 2 // 010
+//				} else if f == "createtime" {
+//					saveM[f] = time.Now().Format(util.Date_Full_Layout)
+//				}
+//			}
+//			saveRelationPool <- saveM
+//		}
+//	}
+//}
+//package main
+//
+//import (
+//"data_tidb/config"
+//"fmt"
+//"go.mongodb.org/mongo-driver/bson"
+//"go.uber.org/zap"
+//util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+//"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+//"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+//"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+//"strings"
+//"sync"
+//"time"
+//)
+//
+//func taskP() {
+//	sess := MongoP.GetMgoConn()
+//	defer MongoP.DestoryMongoConn(sess)
+//
+//	ch := make(chan bool, 20)
+//	wg := &sync.WaitGroup{}
+//
+//	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("64e5a63855d5406905c574e6")}
+//	query := sess.DB(config.Conf.DB.MongoP.Dbname).C("projectset_20230407").Find(nil).Sort("-_id").Skip(100000).Iter()
+//	count := 0
+//	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+//		if count%20000 == 0 {
+//			log.Info(fmt.Sprintf("current --- %d", count))
+//		}
+//		ch <- true
+//		wg.Add(1)
+//		go func(tmp map[string]interface{}) {
+//			defer func() {
+//				<-ch
+//				wg.Done()
+//			}()
+//			//taskPro(tmp)
+//			//taskBusiness(tmp)
+//			//taskProTag(tmp)
+//			//taskRelation(tmp)
+//
+//		}(tmp)
+//		tmp = make(map[string]interface{})
+//	}
+//	wg.Wait()
+//	log.Info(fmt.Sprintf("over --- %d", count))
+//}
+//
+//func taskPAdd(pici int64) {
+//	sess := MongoP.GetMgoConn()
+//	defer MongoP.DestoryMongoConn(sess)
+//
+//	ch := make(chan bool, 20)
+//	wg := &sync.WaitGroup{}
+//
+//	q := bson.M{"pici": bson.M{"$gt": pici}}
+//	query := sess.DB(config.Conf.DB.MongoP.Dbname).C("projectset_20230407").Find(q).Iter()
+//	count := 0
+//	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+//		if count%20000 == 0 {
+//			log.Info(fmt.Sprintf("current --- %d", count))
+//		}
+//		ch <- true
+//		wg.Add(1)
+//		go func(tmp map[string]interface{}) {
+//			defer func() {
+//				<-ch
+//				wg.Done()
+//			}()
+//			//taskPro(tmp)
+//			//taskBusiness(tmp)
+//			//taskProTag(tmp)
+//			//taskRelation2(tmp)
+//
+//		}(tmp)
+//		tmp = make(map[string]interface{})
+//	}
+//	wg.Wait()
+//	log.Info(fmt.Sprintf("over --- %d", count))
+//}
+//
+//var BidStatus = map[string]int{
+//	"预告": 0,
+//	"拟建": 1,
+//	"招标": 2,
+//	"中标": 3,
+//	"成交": 4,
+//	"废标": 5,
+//	"流标": 6,
+//	"合同": 7,
+//	"其它": 8,
+//}
+//var BidType = map[string]int{
+//	"招标": 0,
+//	"邀标": 1,
+//	"单一": 2,
+//	"竞价": 3,
+//	"竞谈": 4,
+//	"询价": 5,
+//}
+//
+//// @Description 基础信息
+//// @Author J 2022/9/22 18:32
+//func taskPro(tmp map[string]interface{}) {
+//	saveM := make(map[string]interface{})
+//	for _, f := range ProField {
+//		if f == "projectid" {
+//			saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
+//		} else if f == "area_code" {
+//			if tmp["area"] != nil {
+//				saveM[f] = AreaCode[util.ObjToString(tmp["area"])]
+//			}
+//		} else if f == "city_code" {
+//			if tmp["area"] != nil && tmp["city"] != nil {
+//				c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"])
+//				saveM[f] = AreaCode[c]
+//			}
+//		} else if f == "district_code" {
+//			if tmp["area"] != nil && tmp["city"] != nil && tmp["district"] != nil {
+//				c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"]) + "," + util.ObjToString(tmp["district"])
+//				saveM[f] = AreaCode[c]
+//			}
+//		} else if f == "updatetime" {
+//			saveM[f] = time.Now().Format(util.Date_Full_Layout)
+//		} else if f == "buyerclass_code" {
+//			if obj := util.ObjToString(tmp["buyerclass"]); obj != "" {
+//				saveM[f] = BuyerCode[obj]
+//			}
+//		} else if f == "firsttime" || f == "zbtime" || f == "jgtime" || f == "lasttime" || f == "bidopentime" || f == "createtime" {
+//			if tmp[f] != nil && util.Int64All(tmp[f]) > 0 {
+//				t := util.Int64All(tmp[f])
+//				saveM[f] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+//			}
+//		} else if f == "bidstatus" {
+//			if b := util.ObjToString(tmp[f]); b != "" {
+//				tmp[f] = BidStatus[b]
+//			}
+//		} else if f == "bidtype" {
+//			if b := util.ObjToString(tmp[f]); b != "" {
+//				tmp[f] = BidType[b]
+//			}
+//		} else if f == "multipackage" {
+//			if tmp[f] == nil {
+//				saveM[f] = 0
+//			} else {
+//				saveM[f] = tmp[f]
+//			}
+//		} else if f == "buyer_id" {
+//			if b := util.ObjToString(tmp["buyer"]); b != "" {
+//				if code := redis.GetStr("qyxy_id", b); code != "" {
+//					saveM[f] = code
+//				}
+//			}
+//		} else if f == "agency_id" {
+//			if b := util.ObjToString(tmp["agency"]); b != "" {
+//				if code := redis.GetStr("qyxy_id", b); code != "" {
+//					saveM[f] = code
+//				}
+//			}
+//		} else {
+//			if tmp[f] != nil {
+//				if ProVMap[f] != nil {
+//					saveM[f], _ = verifyF(f, tmp[f], ProVMap[f])
+//				} else {
+//					saveM[f] = tmp[f]
+//				}
+//			}
+//		}
+//	}
+//	saveProPool <- saveM
+//}
+//
+//// @Description 项目业务表
+//// @Author J 2022/9/30 13:40
+//func taskBusiness(tmp map[string]interface{}) {
+//	warr := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
+//	if BinarySearch(warr, util.ObjToString(tmp["winner"])) == -1 {
+//		warr = append(warr, util.ObjToString(tmp["winner"]))
+//	}
+//	for _, s := range warr {
+//		saveM := make(map[string]interface{})
+//		for _, f := range ProBusField {
+//			if f == "projectid" {
+//				saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
+//			} else if f == "area_code" {
+//				if tmp["area"] != nil {
+//					saveM[f] = AreaCode[util.ObjToString(tmp["area"])]
+//				}
+//			} else if f == "city_code" {
+//				if tmp["area"] != nil && tmp["city"] != nil {
+//					c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"])
+//					saveM[f] = AreaCode[c]
+//				}
+//			} else if f == "district_code" {
+//				if tmp["area"] != nil && tmp["city"] != nil && tmp["district"] != nil {
+//					c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"]) + "," + util.ObjToString(tmp["district"])
+//					saveM[f] = AreaCode[c]
+//				}
+//			} else if f == "updatetime" {
+//				saveM[f] = time.Now().Format(util.Date_Full_Layout)
+//			} else if f == "buyerclass_code" {
+//				if obj := util.ObjToString(tmp["buyerclass"]); obj != "" {
+//					saveM[f] = BuyerCode[obj]
+//				}
+//			} else if f == "firsttime" || f == "zbtime" || f == "jgtime" || f == "lasttime" || f == "bidopentime" || f == "createtime" {
+//				if tmp[f] != nil && util.Int64All(tmp[f]) > 0 {
+//					t := util.Int64All(tmp[f])
+//					saveM[f] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+//				}
+//			} else if f == "bidstatus" {
+//				if b := util.ObjToString(tmp[f]); b != "" {
+//					tmp[f] = BidStatus[b]
+//				}
+//			} else if f == "bidtype" {
+//				if b := util.ObjToString(tmp[f]); b != "" {
+//					tmp[f] = BidType[b]
+//				}
+//			} else if f == "buyer_id" {
+//				if b := util.ObjToString(tmp["buyer"]); b != "" {
+//					saveM["buyer"] = b
+//					if code := redis.GetStr("qyxy_id", b); code != "" {
+//						saveM[f] = code
+//					}
+//				}
+//			} else if f == "agency_id" {
+//				if b := util.ObjToString(tmp["agency"]); b != "" {
+//					saveM["agency"] = b
+//					if code := redis.GetStr("qyxy_id", b); code != "" {
+//						saveM[f] = code
+//					}
+//				}
+//			} else if f == "winner_id" {
+//				if s != "" {
+//					saveM["winner"] = s
+//					if code := redis.GetStr("qyxy_id", s); code != "" {
+//						saveM[f] = code
+//					}
+//				}
+//			} else {
+//				if tmp[f] != nil {
+//					if ProBusVMap[f] != nil {
+//						saveM[f], _ = verifyF(f, tmp[f], ProBusVMap[f])
+//					} else {
+//						saveM[f] = tmp[f]
+//					}
+//				}
+//			}
+//		}
+//		saveProbPool <- saveM
+//	}
+//
+//}
+//
+//// @Description 项目信息标签
+//// @Author J 2022/9/30 13:54
+//func taskProTag(tmp map[string]interface{}) {
+//	id := mongodb.BsonIdToSId(tmp["_id"])
+//	if topArr, ok := tmp["topscopeclass"].([]interface{}); ok {
+//		for _, i2 := range topArr {
+//			tclass := regLetter.ReplaceAllString(util.ObjToString(i2), "") // 去除字母
+//			code := TopScopeCode[tclass]
+//			saveProTagPool <- map[string]interface{}{"projectid": id, "labelcode": "1", "labelvalues": code, "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
+//		}
+//	}
+//	if subArr, ok := tmp["subscopeclass"].([]interface{}); ok {
+//		for _, i2 := range subArr {
+//			sc := strings.Split(util.ObjToString(i2), "_")
+//			code := SubScopeCode[sc[1]]
+//			saveProTagPool <- map[string]interface{}{"projectid": id, "labelcode": "2", "labelvalues": code, "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)}
+//		}
+//	}
+//}
+//
+//// @Description 关系表
+//// @Author J 2022/9/30 13:56
+//func taskRelation(tmp map[string]interface{}) {
+//	pid := mongodb.BsonIdToSId(tmp["_id"])
+//	if tmp["ids"] == nil {
+//		log.Info("taskRelation ids err", zap.Any("id", pid))
+//		return
+//	}
+//	ids := util.ObjArrToStringArr(tmp["ids"].([]interface{}))
+//	lid := ids[len(ids)-1]
+//
+//	//if b := util.ObjToString(tmp["buyer"]); b != "" {
+//	//	saveM := make(map[string]interface{})
+//	//
+//	//	saveM["projectid"] = pid
+//	//	saveM["infoid"] = lid
+//	//	saveM["identity_type"] = 1
+//	//	saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
+//	//	if code := redis.GetStr("qyxy_id", b); code != "" {
+//	//		saveM["name_id"] = code
+//	//		if util.ObjToString(tmp["buyertel"]) != "" {
+//	//			q := make(map[string]interface{})
+//	//			q["name_id"] = code
+//	//			q["identity_type"] = 1
+//	//			q["contact_tel"] = util.ObjToString(tmp["buyertel"])
+//	//			if util.ObjToString(tmp["buyerperson"]) != "" {
+//	//				q["contact_name"] = util.ObjToString(tmp["buyerperson"])
+//	//			}
+//	//			cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//	//			if cinfo != nil && len(*cinfo) > 0 {
+//	//				saveM["contact_id"] = (*cinfo)["id"]
+//	//				saveRelationPool <- saveM
+//	//			}
+//	//		}
+//	//	}
+//	//}
+//
+//	//if a := util.ObjToString(tmp["agency"]); a != "" {
+//	//	saveM := make(map[string]interface{})
+//	//	saveM["projectid"] = pid
+//	//	saveM["infoid"] = lid
+//	//	saveM["identity_type"] = 4
+//	//	saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
+//	//	if code := redis.GetStr("qyxy_id", a); code != "" {
+//	//		saveM["name_id"] = code
+//	//		if util.ObjToString(tmp["agencytel"]) != "" {
+//	//			q := make(map[string]interface{})
+//	//			q["name_id"] = code
+//	//			q["identity_type"] = 4 // 100
+//	//			q["contact_tel"] = util.ObjToString(tmp["agencytel"])
+//	//			if util.ObjToString(tmp["agencyperson"]) != "" {
+//	//				q["contact_name"] = util.ObjToString(tmp["agencyperson"])
+//	//			}
+//	//			cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//	//			if cinfo != nil && len(*cinfo) > 0 {
+//	//				saveM["contact_id"] = (*cinfo)["id"]
+//	//				saveRelationPool <- saveM
+//	//			}
+//	//		}
+//	//	}
+//	//}
+//
+//	for _, item := range tmp["list"].([]interface{}) {
+//		item1 := item.(map[string]interface{})
+//		sw := util.ObjToString(item1["s_winner"])
+//		if !strings.Contains(sw, ",") {
+//			if code := redis.GetStr("qyxy_id", sw); code != "" {
+//				saveM := make(map[string]interface{})
+//				saveM["projectid"] = pid
+//				saveM["infoid"] = lid
+//				saveM["identity_type"] = 2
+//				saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
+//				saveM["name_id"] = code
+//				if util.ObjToString(item1["winnertel"]) != "" {
+//					q := make(map[string]interface{})
+//					q["name_id"] = code
+//					q["identity_type"] = 2 // 010
+//					q["contact_tel"] = util.ObjToString(item1["winnertel"])
+//					if util.ObjToString(item1["winnerperson"]) != "" {
+//						q["contact_name"] = util.ObjToString(item1["winnerperson"])
+//					}
+//					cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//					if cinfo != nil && len(*cinfo) > 0 {
+//						saveM["contact_id"] = (*cinfo)["id"]
+//						saveRelationPool <- saveM
+//					}
+//				}
+//			}
+//		}
+//	}
+//}
+//
+//func taskRelation2(tmp map[string]interface{}) {
+//
+//	pid := mongodb.BsonIdToSId(tmp["_id"])
+//	if tmp["ids"] == nil {
+//		log.Info("taskRelation ids err", zap.Any("id", pid))
+//		return
+//	}
+//	info := MysqlTool.Find("dws_f_bpmc_relation", bson.M{"projectid": pid}, "", "", -1, -1)
+//	if len(*info) > 0 {
+//
+//	} else {
+//		ids := util.ObjArrToStringArr(tmp["ids"].([]interface{}))
+//		lid := ids[len(ids)-1]
+//
+//		if b := util.ObjToString(tmp["buyer"]); b != "" {
+//			saveM := make(map[string]interface{})
+//			for _, f := range RelationField {
+//				if f == "projectid" {
+//					saveM[f] = pid
+//				} else if f == "infoid" {
+//					saveM[f] = lid
+//				} else if f == "name_id" {
+//					if code := redis.GetStr("qyxy_id", b); code != "" {
+//						saveM[f] = code
+//						if util.ObjToString(tmp["buyertel"]) != "" {
+//							q := make(map[string]interface{})
+//							q["name_id"] = code
+//							q["identity_type"] = 1
+//							q["contact_tel"] = util.ObjToString(tmp["buyertel"])
+//							if util.ObjToString(tmp["buyerperson"]) != "" {
+//								q["contact_name"] = util.ObjToString(tmp["buyerperson"])
+//							}
+//							cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//							if cinfo != nil && len(*cinfo) > 0 {
+//								saveM["contact_id"] = (*cinfo)["id"]
+//							}
+//						}
+//					}
+//				} else if f == "identity_type" {
+//					saveM[f] = 1 // 001
+//				} else if f == "createtime" {
+//					saveM[f] = time.Now().Format(util.Date_Full_Layout)
+//				}
+//			}
+//			saveRelationPool <- saveM
+//		}
+//
+//		if a := util.ObjToString(tmp["agency"]); a != "" {
+//			saveM := make(map[string]interface{})
+//			for _, f := range RelationField {
+//				if f == "projectid" {
+//					saveM[f] = pid
+//				} else if f == "infoid" {
+//					saveM[f] = lid
+//				} else if f == "name_id" {
+//					if code := redis.GetStr("qyxy_id", a); code != "" {
+//						saveM[f] = code
+//						if util.ObjToString(tmp["buyertel"]) != "" {
+//							q := make(map[string]interface{})
+//							q["name_id"] = code
+//							q["identity_type"] = 4
+//							q["contact_tel"] = util.ObjToString(tmp["agencytel"])
+//							if util.ObjToString(tmp["agencyperson"]) != "" {
+//								q["contact_name"] = util.ObjToString(tmp["agencyperson"])
+//							}
+//							cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//							if cinfo != nil && len(*cinfo) > 0 {
+//								saveM["contact_id"] = (*cinfo)["id"]
+//							}
+//						}
+//					}
+//				} else if f == "identity_type" {
+//					saveM[f] = 4 // 100
+//				} else if f == "createtime" {
+//					saveM[f] = time.Now().Format(util.Date_Full_Layout)
+//				}
+//			}
+//			saveRelationPool <- saveM
+//		}
+//
+//		warr := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
+//		if BinarySearch(warr, util.ObjToString(tmp["winner"])) == -1 {
+//			warr = append(warr, util.ObjToString(tmp["winner"]))
+//		}
+//		for _, ws := range warr {
+//			saveM := make(map[string]interface{})
+//			for _, f := range RelationField {
+//				if f == "projectid" {
+//					saveM[f] = pid
+//				} else if f == "infoid" {
+//					saveM[f] = lid
+//				} else if f == "name_id" {
+//					if code := redis.GetStr("qyxy_id", ws); code != "" {
+//						saveM[f] = code
+//						if util.ObjToString(tmp["buyertel"]) != "" {
+//							q := make(map[string]interface{})
+//							q["name_id"] = code
+//							q["identity_type"] = 2
+//							q["contact_tel"] = util.ObjToString(tmp["winnertel"])
+//							if util.ObjToString(tmp["winnerperson"]) != "" {
+//								q["contact_name"] = util.ObjToString(tmp["winnerperson"])
+//							}
+//							cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+//							if cinfo != nil && len(*cinfo) > 0 {
+//								saveM["contact_id"] = (*cinfo)["id"]
+//							}
+//						}
+//					}
+//				} else if f == "identity_type" {
+//					saveM[f] = 2 // 010
+//				} else if f == "createtime" {
+//					saveM[f] = time.Now().Format(util.Date_Full_Layout)
+//				}
+//			}
+//			saveRelationPool <- saveM
+//		}
+//	}
+//}
+//
+//
+//
+//
+//
+//

data_mgo_to_tidb/util/init.go

@@ -0,0 +1,151 @@
+package util
+
+import (
+	"data_tidb/config"
+	"fmt"
+	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
+	"os"
+)
+
+var ( //通用
+	MongoB, MongoP *mongodb.MongodbSim
+	MysqlTool      *mysqldb.Mysql
+	RegionCode     = make(map[string]string, 5000)
+	BuyerClassCode = make(map[string]string, 100)
+	TopSubtypeCode = make(map[string]map[string]string, 100)
+	FileTypeArr    = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"}
+)
+
+const (
+	T_dwd_f_bid_baseinfo                = "dwd_f_bid_baseinfo"
+	T_dwd_f_bid_detail                  = "dwd_f_bid_detail"
+	T_dwd_f_bid_expand_baseinfo         = "dwd_f_bid_expand_baseinfo"
+	T_dwd_f_bid_file_baseinfo           = "dwd_f_bid_file_baseinfo"
+	T_dwd_f_bid_file_text               = "dwd_f_bid_file_text"
+	T_dwd_f_bid_intention_baseinfo      = "dwd_f_bid_intention_baseinfo"
+	T_dwd_f_bid_package_baseinfo        = "dwd_f_bid_package_baseinfo"
+	T_dwd_f_bid_package_bidder_baseinfo = "dwd_f_bid_package_bidder_baseinfo"
+	T_dwd_f_bid_package_goods_baseinfo  = "dwd_f_bid_package_goods_baseinfo"
+)
+
+func InitInfo() {
+	config.Init("./common.toml")
+	InitLog()
+	InitMgo()
+	InitMysql()
+	InitField()
+	log.Info("init success")
+}
+
+// InitLog @Description
+// @Author J 2022/7/26 15:30
+func InitLog() {
+	logcfg := config.Conf.Log
+	err := log.InitLog(
+		log.Path(logcfg.LogPath),
+		log.Level(logcfg.LogLevel),
+		log.Compress(logcfg.Compress),
+		log.MaxSize(logcfg.MaxSize),
+		log.MaxBackups(logcfg.MaxBackups),
+		log.MaxAge(logcfg.MaxAge),
+		log.Format(logcfg.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func InitMgo() {
+	MongoB = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoB.Addr,
+		DbName:      config.Conf.DB.MongoB.Dbname,
+		Size:        config.Conf.DB.MongoB.Size,
+		UserName:    config.Conf.DB.MongoB.User,
+		Password:    config.Conf.DB.MongoB.Password,
+	}
+	MongoB.InitPool()
+	MongoP = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoP.Addr,
+		DbName:      config.Conf.DB.MongoP.Dbname,
+		Size:        config.Conf.DB.MongoP.Size,
+		UserName:    config.Conf.DB.MongoP.User,
+		Password:    config.Conf.DB.MongoP.Password,
+	}
+	MongoP.InitPool()
+}
+
+func InitMysql() {
+	dbcfg := config.Conf.DB.Mysql
+	MysqlTool = &mysqldb.Mysql{
+		Address:  dbcfg.Addr,
+		DBName:   dbcfg.Dbname,
+		UserName: dbcfg.User,
+		PassWord: dbcfg.Password,
+	}
+	MysqlTool.Init()
+}
+
+func InitField() {
+	info := MysqlTool.Find("code_area", nil, "", "", -1, -1)
+	for _, m := range *info {
+		var key string
+		for i, v := range []string{"area", "city", "district"} {
+			if i == 0 && util.ObjToString(m[v]) != "" {
+				key = util.ObjToString(m[v])
+			} else if util.ObjToString(m[v]) != "" {
+				key += "," + util.ObjToString(m[v])
+			}
+		}
+		RegionCode[key] = util.ObjToString(m["code"])
+	}
+	log.Info("InitField", zap.Int("RegionCode", len(RegionCode)))
+
+	info1 := MysqlTool.Find("code_bidtopsubtype", nil, "", "", -1, -1)
+	for _, v := range *info1 {
+		name := util.ObjToString(v["name"])
+		code := util.ObjToString(v["code"])
+		pcode := util.ObjToString(v["pcode"])
+		level := util.IntAll(v["level"])
+		if level == 1 {
+			TopSubtypeCode[name] = map[string]string{
+				"code": code,
+			}
+		} else if level == 2 {
+			match_name := ""
+			for k1, v1 := range TopSubtypeCode {
+				if v1["code"] == pcode {
+					match_name = k1
+					break
+				}
+			}
+			data := TopSubtypeCode[match_name]
+			data[name] = code
+			TopSubtypeCode[match_name] = data
+		} else {
+
+		}
+
+	}
+	log.Info("InitField", zap.Int("TopSubtypeCode", len(TopSubtypeCode)))
+
+	info2 := MysqlTool.Find("code_buyerclass", nil, "", "", -1, -1)
+	for _, m := range *info2 {
+		BuyerClassCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
+	}
+	log.Info("InitField", zap.Int("BuyerClassCode", len(BuyerClassCode)))
+
+}
+
+// 插入并打印信息
+func InsertGlobalMysqlData(name string, data map[string]interface{}, mark string) int64 {
+	inb := MysqlTool.Insert(name, data)
+	if inb == -1 {
+		log.Info("插入数据异常", zap.String(name, mark))
+	}
+	return inb
+}

data_mgo_to_tidb/util/save.go

@@ -0,0 +1,50 @@
+package util
+
+import "time"
+
+var (
+	SaveSize     = 200
+	SaveBasePool = make(chan map[string]interface{}, 5000)
+	SaveBaseSp   = make(chan bool, 1)
+
+	BaseField = []string{"s_info_id", "s_area_code", "s_city_code", "s_district_code", "f_budget", "f_bidamount", "f_biddiscount", "s_title", "s_toptype_code", "s_subtype_code", "s_projectname", "s_projectcode", "s_buyerclass_code", "d_publishtime", "d_comeintime", "d_bidopentime", "d_bidendtime", "i_isvalidfile", "s_href", "s_purchasing", "i_multipackage", "s_site", "s_buyer_id", "s_agency_id", "d_updatetime", "d_createtime"}
+)
+
+func InitSaveService() {
+	go SaveBaseFunc()
+}
+
+func SaveBaseFunc() {
+	arru := make([]map[string]interface{}, SaveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-SaveBasePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == SaveSize {
+				SaveBaseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-SaveBaseSp
+					}()
+					MysqlTool.InsertBulk(T_dwd_f_bid_baseinfo, BaseField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, SaveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				SaveBaseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-SaveBaseSp
+					}()
+					MysqlTool.InsertBulk(T_dwd_f_bid_baseinfo, BaseField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, SaveSize)
+				indexu = 0
+			}
+		}
+	}
+}

data_mgo_to_tidb/util/util.go

@@ -0,0 +1,127 @@
+package util
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"unicode/utf8"
+)
+
+// 转换时间
+func TransferDateTimeInfo(tmp map[string]interface{}, info *map[string]interface{}, keys []string) {
+	for _, v := range keys {
+		if t := util.Int64All(tmp[v]); t > 0 {
+			key := "d_" + v
+			(*info)[key] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+		}
+	}
+}
+
+// 转换文本
+func TransferTextInfo(tmp map[string]interface{}, info *map[string]interface{}, keys []string, maxs []int) {
+	for k, v := range keys {
+		if value := util.ObjToString(tmp[v]); value != "" {
+			key := "s_" + v
+			if len(maxs) > k {
+				(*info)[key] = CleanLength(value, maxs[k])
+			} else {
+				(*info)[key] = value
+			}
+		}
+	}
+}
+
+// 转换布尔
+func TransferBoolInfo(tmp map[string]interface{}, info *map[string]interface{}, keys []string) {
+	for _, v := range keys {
+		if tmp[v] != nil {
+			key := "i_" + v
+			if tmp[v].(bool) {
+				(*info)[key] = 1
+			} else {
+				(*info)[key] = 0
+			}
+		}
+	}
+}
+
+// 转换金额费用
+func TransferMoneyRateInfo(tmp map[string]interface{}, info *map[string]interface{}, keys []string, maxs []float64) {
+	for k, v := range keys {
+		key := "f_" + v
+		if value, b := CleanFloat64(util.Float64All(tmp[v]), maxs[k]); tmp[v] != nil && b {
+			(*info)[key] = value
+		}
+	}
+}
+
+// 通用长度清洗
+func CleanLength(name string, l int) string {
+	if l == -1 {
+		return name
+	}
+	new_name := name
+	if utf8.RuneCountInString(name) > l {
+		new_name = string(([]rune(new_name))[:l])
+	}
+	return new_name
+}
+
+// 通用浮点清洗
+func CleanFloat64(value float64, max float64) (float64, bool) {
+	if value <= max {
+		return value, true
+	}
+	return float64(0), false
+}
+
+func GetNameId(name string) string {
+	if name == "" {
+		return ""
+	}
+	info := MysqlTool.FindOne("dws_f_ent_baseinfo", map[string]interface{}{"name": name}, "name_id", "")
+	if info != nil && (*info)["name_id"] != nil {
+		return util.ObjToString((*info)["name_id"])
+	} else {
+		return ""
+	}
+}
+
+func GetTopSubCode(toptype string, subtype string) (string, string) {
+	top_code, sub_code := "", ""
+	datas := TopSubtypeCode[toptype]
+	if datas != nil {
+		top_code = datas["code"]
+		if datas[subtype] != "" {
+			sub_code = datas[subtype]
+		}
+	}
+	return top_code, sub_code
+}
+
+func IsMarkInterfaceArr(t interface{}) []string {
+	sub_list := []string{}
+	if list_3, ok_3 := t.([]string); ok_3 {
+		sub_list = list_3
+		return sub_list
+	}
+	if list_1, ok_1 := t.(primitive.A); ok_1 {
+		sub_list = util.ObjArrToStringArr(list_1)
+	} else {
+		if list_2, ok_2 := t.([]interface{}); ok_2 {
+			sub_list = util.ObjArrToStringArr(list_2)
+		}
+	}
+	return sub_list
+}
+
+func IsMarkInterfaceMap(t interface{}) []map[string]interface{} {
+	p_list := []map[string]interface{}{}
+	if yl_list_1, ok_1 := t.(primitive.A); ok_1 {
+		p_list = util.ObjArrToMapArr(yl_list_1)
+	} else {
+		if yl_list_2, ok_2 := t.([]interface{}); ok_2 {
+			p_list = util.ObjArrToMapArr(yl_list_2)
+		}
+	}
+	return p_list
+}

data_monitor/config.go

@@ -0,0 +1,20 @@
+package main
+
+type (
+	Config struct {
+		CornExp string `json:"cornexp"`
+		Es      struct {
+			Address      string `json:"address"`
+			DbSize       int    `json:"dbSize"`
+			BiddingIndex string `json:"biddingIndex"`
+			ProjectIndex string `json:"projectIndex"`
+			Version      string `json:"version"`
+			UserName     string `json:"userName"`
+			Password     string `json:"password"`
+		} `json:"es"`
+		WxKey           string `json:"wxKey"`
+		WxApi           string `json:"wxApi"`
+		LastProjectTime int64  `json:"lastProjectTime"`
+		LastBiddingTime int64  `json:"lastBiddingTime"`
+	}
+)

data_monitor/config.json

@@ -0,0 +1,16 @@
+{
+	"cornexp": "0 0 */2 * * ?",
+	"es": {
+		"address": "http://172.17.4.184:19800",
+		"dbSize": 10,
+		"biddingIndex": "bidding",
+		"projectIndex": "projectset",
+		"version": "v7",
+		"userName": "",
+  		"password":
+	},
+	"wxKey": "45962efc-ca87-4996-9ffa-08bf6608ab7a",
+	"wxApi": "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=%s",
+	"lastProjectTime": 1660438800,
+	"lastBiddingTime": 1660438800,
+}

data_monitor/main.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	elastic "es"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	common "qfw/util"
+
+	"github.com/robfig/cron"
+)
+
+var (
+	Es                       elastic.Es
+	cfg                      = new(Config)
+	projectTime, biddingTime = 0, 0
+)
+
+func main() {
+	common.ReadConfig(&cfg)
+	log.Println("cfg ", cfg)
+	Es = elastic.NewEs(cfg.Es.Version, cfg.Es.Address, cfg.Es.DbSize, cfg.Es.UserName, cfg.Es.Password)
+	runJob()
+	c := cron.New()
+	c.AddFunc(cfg.CornExp, func() {
+		runJob()
+	})
+	c.Start()
+	select {}
+}
+
+func runJob() {
+	log.Println("增量数据查询开始")
+	lastProjectTime, isOk := getData(cfg.LastProjectTime, cfg.Es.ProjectIndex)
+	if isOk {
+		cfg.LastProjectTime = lastProjectTime
+	}
+	lastBiddingTime, isOks := getData(cfg.LastBiddingTime, cfg.Es.BiddingIndex)
+	if isOks {
+		cfg.LastBiddingTime = lastBiddingTime
+	}
+	common.WriteSysConfig(cfg)
+	log.Println("增量数据查询结束")
+}
+
+// Send("cbs告警：超过"+fmt.Sprint(SysConfig.TimeExpire)+"个小时未更新数据", WxKey)
+func getData(LastTime int64, index string) (int64, bool) {
+	endTime, isOk := int64(0), true
+	esquery := `{"query":{"bool":{"must":{"range":{"pici":{"gte":"%d"}}}}},"_source":["pici"],"sort":{"pici":"desc"},"from":0,"size":1}`
+	idQuery := fmt.Sprintf(esquery, LastTime)
+	res := Es.Get(index, index, idQuery)
+	if res != nil && *res != nil && len(*res) == 1 {
+		endTime = common.Int64All((*res)[0]["pici"])
+		if index == cfg.Es.BiddingIndex {
+			biddingTime = 0
+		} else if index == cfg.Es.ProjectIndex {
+			projectTime = 0
+		}
+		log.Println("本次任务查找到数据...", endTime)
+	} else {
+		endTime = LastTime
+		isOk = false
+		log.Println("本次任务未查找到数据...", idQuery)
+		if index == cfg.Es.BiddingIndex {
+			biddingTime += 2
+		} else if index == cfg.Es.ProjectIndex {
+			projectTime += 2
+		}
+		Send(index+fmt.Sprint(projectTime)+"个小时未更新数据", cfg.WxKey)
+	}
+	return endTime, isOk
+}
+
+func Send(msg, key string) {
+	m := map[string]interface{}{
+		"msgtype": "text",
+		"text": map[string]string{
+			"content": msg,
+		},
+	}
+	b, _ := json.Marshal(m)
+	res, err := http.Post(fmt.Sprintf(cfg.WxApi, key), "application/json", bytes.NewReader(b))
+	if err != nil {
+		log.Println("发送出错", err)
+	} else {
+		defer res.Body.Close()
+		resByte, _ := ioutil.ReadAll(res.Body)
+		log.Println("发送结果", string(resByte))
+	}
+}