xg

data_fusion/data_fusion

@@ -1 +0,0 @@
-ELF

data_fusion/main.go

@@ -10,7 +10,6 @@ import (
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
 	"reflect"
-	"strings"
 	"sync"
 	"time"
 )
@@ -48,8 +47,9 @@ func main() {
 	wg := &sync.WaitGroup{}
 
 	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("635ff15d631ff1ac3d095c41")}
-	//f := map[string]interface{}{"contenthtml": 0}
-	it := sess.DB(config.Conf.DB.Mongo.Dbname).C(config.Conf.DB.Mongo.Coll).Find(nil).Select(nil).Iter()
+	q := bson.M{"extracttype": -1}
+	f := map[string]interface{}{"detail": 0, "contenthtml": 0, "summary": 0, "description": 0}
+	it := sess.DB(config.Conf.DB.Mongo.Dbname).C(config.Conf.DB.Mongo.Coll).Find(q).Select(f).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); it.Next(tmp); count++ {
 		if count%20000 == 0 {
@@ -57,185 +57,68 @@ func main() {
 		}
 		ch <- true
 		wg.Add(1)
-
 		go func(tmp map[string]interface{}) {
 			defer func() {
 				<-ch
 				wg.Done()
 			}()
 
-			if repeat := util.IntAll(tmp["extracttype"]); repeat != -1 {
-				updatePool <- []map[string]interface{}{
-					{"_id": tmp["_id"]},
-					{"$set": bson.M{"fusion_tag": 1}},
+			repeatId := util.ObjToString(tmp["repeat_id"])
+			ArrLock[util.HashCode(repeatId)%config.Conf.Serve.Thread].Lock()
+			if str := redis.GetStr("fusion_id", repeatId); str != "" {
+				tmp1 := findData(repeatId)
+				var update map[string]interface{}
+				var fs []string
+				update, fs = mergeTmp(tmp1, tmp)
+				update1 := util.DeepCopy(update).(map[string]interface{})
+				update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
+				if fs != nil && len(fs) > 0 {
+					updatePool <- []map[string]interface{}{
+						{"_id": tmp1["_id"]},
+						{"$set": update, "$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
+					}
+				}
+				record := make(map[string]interface{})
+				record["$push"] = map[string]interface{}{
+					"ids":    mongodb.BsonIdToSId(tmp["_id"]),
+					"record": update1,
 				}
+				recordPool <- []map[string]interface{}{
+					{"_id": mongodb.StringTOBsonId(repeatId)},
+					record,
+				}
+				redis.PutCKV("fusion_id", repeatId, fmt.Sprintf("%s-%s", str, mongodb.BsonIdToSId(tmp["_id"])))
 			} else {
-				repeatId := util.ObjToString(tmp["repeat_id"])
-				ArrLock[util.HashCode(repeatId)%config.Conf.Serve.Thread].Lock()
-				if str := redis.GetStr("fusion_id", repeatId); str != "" {
-					mid := strings.Split(str, "-")[0]
-					tmp1 := findData(mid)
-					w, s := getWeight(tmp)
-					w1, _ := getWeight(tmp1)
-					var update map[string]interface{}
-					var fs []string
-					if w > w1 {
-						update, fs = mergeTmp(tmp, tmp1)
-						update1 := util.DeepCopy(update).(map[string]interface{})
-						update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
-						update1["weight"] = w
-						update["fusion_tag"] = 1
-						if fs != nil && len(fs) > 0 {
-							updatePool <- []map[string]interface{}{
-								{"_id": tmp["_id"]},
-								{"$set": update, "$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
-							}
-						} else {
-							updatePool <- []map[string]interface{}{
-								{"_id": tmp["_id"]},
-								{"$set": update},
-							}
-						}
-						updatePool <- []map[string]interface{}{
-							{"_id": tmp1["_id"]},
-							{"$set": bson.M{"fusion_tag": 0}},
-						}
-						record := make(map[string]interface{})
-						record["$set"] = map[string]interface{}{
-							"template_id":     mongodb.BsonIdToSId(tmp["_id"]),
-							"template_weight": w,
-						}
-						if w == 0 {
-							update1["remark"] = s
-						}
-						record["$push"] = map[string]interface{}{
-							"ids":    mongodb.BsonIdToSId(tmp["_id"]),
-							"record": update1,
-						}
-						recordPool <- []map[string]interface{}{
-							{"_id": mongodb.StringTOBsonId(repeatId)},
-							record,
-						}
-						redis.PutCKV("fusion_id", repeatId, fmt.Sprintf("%s-%s", mongodb.BsonIdToSId(tmp["_id"]), str))
-					} else {
-						update, fs = mergeTmp(tmp1, tmp)
-						update1 := util.DeepCopy(update).(map[string]interface{})
-						update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
-						update1["weight"] = w
-						update["fusion_tag"] = 1
-						if fs != nil && len(fs) > 0 {
-							updatePool <- []map[string]interface{}{
-								{"_id": (tmp1)["_id"]},
-								{"$set": update, "$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
-							}
-						} else {
-							updatePool <- []map[string]interface{}{
-								{"_id": (tmp1)["_id"]},
-								{"$set": update},
-							}
-						}
-						updatePool <- []map[string]interface{}{
-							{"_id": (tmp)["_id"]},
-							{"$set": bson.M{"fusion_tag": 0}},
-						}
-						record := make(map[string]interface{})
-						record["$set"] = map[string]interface{}{
-							"template_weight": w1,
-						}
-
-						if w == 0 {
-							update1["remark"] = s
-						}
-						record["$push"] = map[string]interface{}{
-							"ids":    mongodb.BsonIdToSId(tmp["_id"]),
-							"record": update1,
-						}
-						recordPool <- []map[string]interface{}{
-							{"_id": mongodb.StringTOBsonId(repeatId)},
-							record,
-						}
-						redis.PutCKV("fusion_id", mid, fmt.Sprintf("%s-%s", str, mongodb.BsonIdToSId(tmp["_id"])))
-					}
-				} else {
-					tmp1 := findData(repeatId)
-					w, s := getWeight(tmp)
-					w1, s1 := getWeight(tmp1)
-					var update map[string]interface{}
-					var fs []string
-					if w > w1 {
-						update, fs = mergeTmp(tmp, tmp1)
-						set := util.DeepCopy(update).(map[string]interface{})
-						if fs != nil && len(fs) > 0 {
-							set["fusion_fields"] = fs
-						}
-						set["fusion_tag"] = 1
-						updatePool <- []map[string]interface{}{
-							{"_id": tmp["_id"]},
-							{"$set": set},
-						}
-						updatePool <- []map[string]interface{}{
-							{"_id": tmp1["_id"]},
-							{"$set": bson.M{"fusion_tag": 0}},
-						}
-						record := make(map[string]interface{})
-						record["_id"] = tmp1["_id"]
-						record["template_id"] = mongodb.BsonIdToSId(tmp["_id"])
-						record["template_weight"] = w
-						record["ids"] = []string{mongodb.BsonIdToSId(tmp["_id"]), mongodb.BsonIdToSId(repeatId)}
-						var recordlist []map[string]interface{}
-						recordlist = append(recordlist, map[string]interface{}{"infoid": mongodb.BsonIdToSId(tmp["_id"]), "weight": w})
-						update1 := util.DeepCopy(update).(map[string]interface{})
-						update1["infoid"] = mongodb.BsonIdToSId(tmp1["_id"])
-						update1["weight"] = w1
-						if w1 == 0 {
-							update1["remark"] = s1
-						}
-						recordlist = append(recordlist, update1)
-						record["record"] = recordlist
-						recordPool <- []map[string]interface{}{
-							{"_id": tmp1["_id"]},
-							{"$set": record},
-						}
-						redis.PutCKV("fusion_id", repeatId, fmt.Sprintf("%s-%s", mongodb.BsonIdToSId(tmp["_id"]), mongodb.BsonIdToSId(tmp1["_id"])))
-					} else {
-						update, fs = mergeTmp(tmp1, tmp)
-						set := util.DeepCopy(update).(map[string]interface{})
-						if fs != nil && len(fs) > 0 {
-							set["fusion_fields"] = fs
-						}
-						set["fusion_tag"] = 1
-						updatePool <- []map[string]interface{}{
-							{"_id": tmp1["_id"]},
-							{"$set": set},
-						}
-						updatePool <- []map[string]interface{}{
-							{"_id": tmp["_id"]},
-							{"$set": bson.M{"fusion_tag": 0}},
-						}
-						record := make(map[string]interface{})
-						record["_id"] = tmp1["_id"]
-						record["template_id"] = mongodb.BsonIdToSId(tmp1["_id"])
-						record["template_weight"] = w1
-						record["ids"] = []string{mongodb.BsonIdToSId(tmp["_id"]), mongodb.BsonIdToSId(tmp1["_id"])}
-						var recordlist []map[string]interface{}
-						recordlist = append(recordlist, map[string]interface{}{"infoid": mongodb.BsonIdToSId(tmp1["_id"]), "weight": w1})
-						update1 := util.DeepCopy(update).(map[string]interface{})
-						update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
-						update1["weight"] = w
-						if w == 0 {
-							update1["remark"] = s
-						}
-						recordlist = append(recordlist, update1)
-						record["record"] = recordlist
-						recordPool <- []map[string]interface{}{
-							{"_id": tmp1["_id"]},
-							{"$set": record},
-						}
-						redis.PutCKV("fusion_id", repeatId, fmt.Sprintf("%s-%s", mongodb.BsonIdToSId(tmp1["_id"]), mongodb.BsonIdToSId(tmp["_id"])))
+				tmp1 := findData(repeatId)
+				var update map[string]interface{}
+				var fs []string
+				update, fs = mergeTmp(tmp1, tmp)
+				set := util.DeepCopy(update).(map[string]interface{})
+				if fs != nil && len(fs) > 0 {
+					set["fusion_fields"] = fs
+				}
+				if len(set) > 0 {
+					updatePool <- []map[string]interface{}{
+						{"_id": tmp1["_id"]},
+						{"$set": set},
 					}
 				}
-				ArrLock[util.HashCode(repeatId)%config.Conf.Serve.Thread].Unlock()
+				record := make(map[string]interface{})
+				record["_id"] = tmp1["_id"]
+				record["template_id"] = mongodb.BsonIdToSId(tmp1["_id"])
+				record["ids"] = []string{mongodb.BsonIdToSId(tmp["_id"]), mongodb.BsonIdToSId(tmp1["_id"])}
+				var recordlist []map[string]interface{}
+				update1 := util.DeepCopy(update).(map[string]interface{})
+				update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
+				recordlist = append(recordlist, update1)
+				record["record"] = recordlist
+				recordPool <- []map[string]interface{}{
+					{"_id": tmp1["_id"]},
+					{"$set": record},
+				}
+				redis.PutCKV("fusion_id", repeatId, mongodb.BsonIdToSId(tmp["_id"]))
 			}
+			ArrLock[util.HashCode(repeatId)%config.Conf.Serve.Thread].Unlock()
 
 		}(tmp)
 		tmp = map[string]interface{}{}

data_project/init.go

@@ -259,7 +259,7 @@ func NewKeyMap() *KeyMap {
 	}
 }
 
-//招标信息实体类
+// 招标信息实体类
 type Info struct {
 	Id                  string                   `json:"_id"`
 	Href                string                   `json:"href"` //源地址
@@ -388,12 +388,10 @@ type Project struct {
 	Qualifies          string                 `json:"qualifies"`               //资质条件
 	EntIdList          []string               `json:"entidlist"`               //企业id
 	//FirstCooperation   []string               `json:"first_cooperation"`       //first_cooperation
-	InfoFiled    map[string]InfoField `json:"infofield"`    //逻辑处理需要的info字段
-	Budgettag    int                  `json:"budgettag"`    //预算是否有效标记
-	Bidamounttag int                  `json:"bidamounttag"` //中标金额是否有效标记
+	//InfoFiled map[string]InfoField `json:"infofield"` //逻辑处理需要的info字段
 }
 
-//存储部分招标信息字段，业务逻辑处理需要
+// 存储部分招标信息字段，业务逻辑处理需要
 type InfoField struct {
 	Budget       float64 `json:"budget"`
 	Bidamount    float64 `json:"bidamount"`
@@ -403,7 +401,7 @@ type InfoField struct {
 	Bidstatus    string  `json:"bidstatus"`
 }
 
-//站点信息
+// 站点信息
 type Site struct {
 	Id       string `json:"_id"`
 	Site     string `json:"site"`     //站点名字
@@ -414,7 +412,7 @@ type Site struct {
 	Status   int    `json:"status"`   //
 }
 
-//二分字符串查找
+// 二分字符串查找
 func BinarySearch(s []string, k string) int {
 	sort.Strings(s)
 	lo, hi := 0, len(s)-1
@@ -442,7 +440,7 @@ func Duplicate(a interface{}) (ret []interface{}) {
 	return ret
 }
 
-//计算文本相似度
+// 计算文本相似度
 func CosineSimilar(srcWords1, dstWords1 string) float64 {
 	srcWords, dstWords := strings.Split(srcWords1, ""), strings.Split(dstWords1, "")
 	// get all words

data_project/main.go

@@ -168,8 +168,8 @@ func main() {
 }
 
 func mainT() {
-	sid = "5c24a939a5cb26b9b75446e6"
-	eid = "5c24a939a5cb26b9b75446e7"
+	sid = "626cccaa631ff1ac3d29289e"
+	eid = "646cd2abc994e88c251e5dae"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()

data_project/merge_comparepnc.go

@@ -17,14 +17,6 @@ func comparePNC(info *Info, compareProject *ProjectCache) (compareStr string, sc
 		}
 		ifind := 0
 		templen := 0
-		buyer := info.Buyer
-		if buyer == "" {
-			buyer = compareProject.Buyer
-		}
-		pn := info.ProjectName
-		if buyer != "" {
-			pn = strings.Replace(pn, buyer, "", -1)
-		}
 		for _, v := range pns {
 			if info.ProjectName == v {
 				ifind = 1
@@ -36,16 +28,15 @@ func comparePNC(info *Info, compareProject *ProjectCache) (compareStr string, sc
 					ifind = 1
 					break
 				} else {
-					if buyer != "" {
-						v = strings.Replace(v, buyer, "", -1)
-					}
 					//v1 := CosineSimilar(pn, v)
 					if retv == 2 {
 						templen = len([]rune(v))
 						ifind = 2
-					} else if ifind == 0 {
-						ifind = 3
+						break
 					}
+					//else if ifind == 0 {
+					//	ifind = 3
+					//}
 				}
 			}
 		}

data_project/project.go

@@ -590,29 +590,35 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 		FirstCooperation(set, p1.Buyer, thisinfo.Winners, thisinfo.EntIdList)
 	}
 
-	infofield := InfoField{
-		Budget:       thisinfo.Budget,
-		Bidamount:    thisinfo.Bidamount,
-		ContractCode: thisinfo.ContractCode,
-		ProjectName:  thisinfo.ProjectName,
-		ProjectCode:  thisinfo.ProjectCode,
-		Bidstatus:    bs,
-	}
-	res := StructToMap(infofield)
-	set["infofield"] = map[string]interface{}{
-		thisinfo.Id: res,
-	}
-	if tmp["budget"] != nil && tmp["budget"] != "" {
+	//infofield := InfoField{
+	//	Budget:       thisinfo.Budget,
+	//	Bidamount:    thisinfo.Bidamount,
+	//	ContractCode: thisinfo.ContractCode,
+	//	ProjectName:  thisinfo.ProjectName,
+	//	ProjectCode:  thisinfo.ProjectCode,
+	//	Bidstatus:    bs,
+	//}
+	//res := StructToMap(infofield)
+	//set["infofield"] = map[string]interface{}{
+	//	thisinfo.Id: res,
+	//}
+	//if tmp["budget"] != nil && tmp["budget"] != "" {
+	//	set["budget"] = thisinfo.Budget
+	//	set["budgettag"] = 0
+	//} else {
+	//	set["budgettag"] = 1
+	//}
+	//if tmp["bidamount"] != nil && tmp["bidamount"] != "" {
+	//	set["bidamount"] = thisinfo.Bidamount
+	//	set["bidamounttag"] = 0
+	//} else {
+	//	set["bidamounttag"] = 1
+	//}
+	if thisinfo.Budget > 0 {
 		set["budget"] = thisinfo.Budget
-		set["budgettag"] = 0
-	} else {
-		set["budgettag"] = 1
 	}
-	if tmp["bidamount"] != nil && tmp["bidamount"] != "" {
+	if thisinfo.Bidamount > 0 {
 		set["bidamount"] = thisinfo.Bidamount
-		set["bidamounttag"] = 0
-	} else {
-		set["bidamounttag"] = 1
 	}
 	if p1.Bidamount > 0 {
 		set["sortprice"] = p1.Bidamount
@@ -625,6 +631,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 		set["bid_field"] = tmp["bid_field"]
 	}
 	redis.PutCKV("project", thisinfo.Id, set)
+	set["size"] = 1
 	push := p.PushListInfo(tmp, thisinfo.Id)
 	if len(thisinfo.Winners) > 0 {
 		push["s_winner"] = strings.Join(thisinfo.Winners, ",")
@@ -1090,16 +1097,26 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		set["package"] = project.Package
 	}
 	//处理多包后，计算预算金额、中标金额
-	CountAmount(project, thisinfo, tmp)
-	if project.Budget >= 0 && project.Budgettag != 1 {
-		pInfo.Budget = project.Budget
-		set["budget"] = project.Budget
-		set["budgettag"] = 0
+	//CountAmount(project, thisinfo, tmp)
+	//if project.Budget >= 0 && project.Budgettag != 1 {
+	//	pInfo.Budget = project.Budget
+	//	set["budget"] = project.Budget
+	//	set["budgettag"] = 0
+	//}
+	//if project.Bidamount >= 0 && project.Bidamounttag != 1 {
+	//	pInfo.Bidamount = project.Bidamount
+	//	set["bidamount"] = pInfo.Bidamount
+	//	set["bidamounttag"] = 0
+	//}
+	if thisinfo.Budget > 0 {
+		pInfo.Budget = thisinfo.Budget
+		project.Budget = thisinfo.Budget
+		set["budget"] = thisinfo.Budget
 	}
-	if project.Bidamount >= 0 && project.Bidamounttag != 1 {
-		pInfo.Bidamount = project.Bidamount
-		set["bidamount"] = pInfo.Bidamount
-		set["bidamounttag"] = 0
+	if thisinfo.Bidamount > 0 {
+		pInfo.Bidamount = thisinfo.Bidamount
+		project.Bidamount = thisinfo.Bidamount
+		set["bidamount"] = thisinfo.Bidamount
 	}
 	if pInfo.Bidamount > 0 {
 		set["sortprice"] = pInfo.Bidamount
@@ -1111,26 +1128,27 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	if tmp["bid_field"] != nil {
 		set["bid_field"] = tmp["bid_field"]
 	}
-	infofield := InfoField{
-		Budget:       thisinfo.Budget,
-		Bidamount:    thisinfo.Bidamount,
-		ContractCode: thisinfo.ContractCode,
-		ProjectName:  thisinfo.ProjectName,
-		ProjectCode:  thisinfo.ProjectCode,
-		Bidstatus:    bs,
-	}
-	copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
-	copyMap[thisinfo.Id] = infofield
-	tmpMap := make(map[string]interface{})
-	for k, v := range copyMap {
-		tmpMap[k] = StructToMap(v)
-	}
-	tmpMap[thisinfo.Id] = StructToMap(infofield)
-	project.InfoFiled = copyMap
-	set["infofield"] = tmpMap
+	//infofield := InfoField{
+	//	Budget:       thisinfo.Budget,
+	//	Bidamount:    thisinfo.Bidamount,
+	//	ContractCode: thisinfo.ContractCode,
+	//	ProjectName:  thisinfo.ProjectName,
+	//	ProjectCode:  thisinfo.ProjectCode,
+	//	Bidstatus:    bs,
+	//}
+	//copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
+	//copyMap[thisinfo.Id] = infofield
+	//tmpMap := make(map[string]interface{})
+	//for k, v := range copyMap {
+	//	tmpMap[k] = StructToMap(v)
+	//}
+	//tmpMap[thisinfo.Id] = StructToMap(infofield)
+	//project.InfoFiled = copyMap
+	//set["infofield"] = tmpMap
 
 	set["mpn"] = pInfo.MPN
 	set["mpc"] = pInfo.MPC
+	set["size"] = len(pInfo.Ids) + 1
 	if p.currentType == "project" || p.currentType == "project_history" {
 		set["pici"] = p.pici
 	} else {
@@ -1290,163 +1308,163 @@ func PackageFormat(info *Info, project *Project) map[string]interface{} {
 }
 
 // 计算预算(budget)、中标金额(bidamount)
-func CountAmount(project *Project, info *Info, tmp map[string]interface{}) {
-	if info.HasPackage {
-		budget := 0.0
-		for _, v := range project.Package {
-			v1, _ := v.([]map[string]interface{})
-			for _, v2 := range v1 {
-				if v2["budget"] != nil {
-					b1 := util.Float64All(v2["budget"])
-					if b1 > 0 {
-						budget = budget + b1
-						break
-					}
-				} else {
-					project.Budgettag = 1
-				}
-			}
-		}
-		if budget > 0 {
-			project.Budget = budget
-			project.Budgettag = 0
-		} else if budget == 0 && info.Budget > 0 {
-			project.Budget = info.Budget
-			project.Budgettag = 0
-		}
-	} else {
-		//招标没有多包
-		k := KeyPackage.FindStringSubmatch(info.ProjectName)
-		if len(k) > 0 {
-			//招标是单包
-			if len(project.Package) > 0 {
-				//项目有多包
-				flag := false
-				for _, v := range project.Package {
-					v1, _ := v.([]map[string]interface{})
-					if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
-						flag = true
-					}
-				}
-				if !flag {
-					project.Budget = project.Budget + info.Budget
-					project.Budgettag = 0
-				}
-			} else {
-				//项目没有多包
-				if info.Budget > 0 {
-					project.Budget = project.Budget + info.Budget
-					project.Budgettag = 0
-				} else if info.Budget == 0 && tmp["budget"] != nil {
-					project.Budgettag = 0
-				}
-			}
-		} else {
-			if info.Budget > 0 && project.Budget < info.Budget {
-				project.Budget = info.Budget
-				project.Budgettag = 0
-			}
-		}
-	}
-	if info.SubType == "中标" || info.SubType == "成交" || info.SubType == "合同" {
-		if info.HasPackage {
-			bidamount := 0.0
-			for _, v := range project.Package {
-				v1, _ := v.([]map[string]interface{})
-				for _, v2 := range v1 {
-					b1 := util.Float64All(v2["bidamount"])
-					if b1 > 0 {
-						bidamount = bidamount + b1
-						break
-					}
-				}
-			}
-			if bidamount > 0 {
-				project.Bidamount = bidamount
-				project.Bidamounttag = 0
-			} else if bidamount == 0 && info.Bidamount > 0 {
-				project.Bidamount = info.Bidamount
-				project.Bidamounttag = 0
-			}
-		} else {
-			//招标没有多包
-			k := KeyPackage.FindStringSubmatch(info.ProjectName)
-			if len(k) > 0 {
-				//招标是单包
-				if len(project.Package) > 0 {
-					//项目有多包
-					flag := false
-					for _, v := range project.Package {
-						v1, _ := v.([]map[string]interface{})
-						if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
-							flag = true
-						}
-					}
-					if !flag {
-						project.Bidamount = project.Bidamount + info.Bidamount
-						project.Bidamounttag = 0
-					}
-				} else {
-					//项目没有多包
-					if info.Bidamount > 0 {
-						project.Bidamount = project.Bidamount + info.Bidamount
-						project.Bidamounttag = 0
-					} else if info.Bidamount == 0 && tmp["bidamount"] != nil {
-						project.Bidamounttag = 0
-					} else {
-						project.Bidamounttag = 1
-					}
-				}
-			} else {
-				if info.SubType == "中标" || info.SubType == "成交" {
-					if info.Bidamount > 0 {
-						project.Bidamount = info.Bidamount
-						project.Bidamounttag = 0
-					} else {
-						flag := false
-						if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
-							for _, res := range project.InfoFiled {
-								if res.ContractCode != "" && res.ContractCode == info.ContractCode {
-									flag = true
-									break
-								}
-								if res.Bidamount == project.Bidamount {
-									flag = true
-									break
-								}
-							}
-							if !flag {
-								project.Bidamount = project.Bidamount + info.Bidamount
-								project.Bidamounttag = 0
-							} else {
-								if info.Budget > 0 && project.Bidamount > info.Bidamount {
-									project.Bidamount = info.Bidamount
-									project.Bidamounttag = 0
-								}
-							}
-						}
-					}
-				}
-			}
-		}
-	} else {
-		project.Bidamounttag = 1
-	}
-
-}
+//func CountAmount(project *Project, info *Info, tmp map[string]interface{}) {
+//	if info.HasPackage {
+//		budget := 0.0
+//		for _, v := range project.Package {
+//			v1, _ := v.([]map[string]interface{})
+//			for _, v2 := range v1 {
+//				if v2["budget"] != nil {
+//					b1 := util.Float64All(v2["budget"])
+//					if b1 > 0 {
+//						budget = budget + b1
+//						break
+//					}
+//				} else {
+//					project.Budgettag = 1
+//				}
+//			}
+//		}
+//		if budget > 0 {
+//			project.Budget = budget
+//			project.Budgettag = 0
+//		} else if budget == 0 && info.Budget > 0 {
+//			project.Budget = info.Budget
+//			project.Budgettag = 0
+//		}
+//	} else {
+//		//招标没有多包
+//		k := KeyPackage.FindStringSubmatch(info.ProjectName)
+//		if len(k) > 0 {
+//			//招标是单包
+//			if len(project.Package) > 0 {
+//				//项目有多包
+//				flag := false
+//				for _, v := range project.Package {
+//					v1, _ := v.([]map[string]interface{})
+//					if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
+//						flag = true
+//					}
+//				}
+//				if !flag {
+//					project.Budget = project.Budget + info.Budget
+//					project.Budgettag = 0
+//				}
+//			} else {
+//				//项目没有多包
+//				if info.Budget > 0 {
+//					project.Budget = project.Budget + info.Budget
+//					project.Budgettag = 0
+//				} else if info.Budget == 0 && tmp["budget"] != nil {
+//					project.Budgettag = 0
+//				}
+//			}
+//		} else {
+//			if info.Budget > 0 && project.Budget < info.Budget {
+//				project.Budget = info.Budget
+//				project.Budgettag = 0
+//			}
+//		}
+//	}
+//	if info.SubType == "中标" || info.SubType == "成交" || info.SubType == "合同" {
+//		if info.HasPackage {
+//			bidamount := 0.0
+//			for _, v := range project.Package {
+//				v1, _ := v.([]map[string]interface{})
+//				for _, v2 := range v1 {
+//					b1 := util.Float64All(v2["bidamount"])
+//					if b1 > 0 {
+//						bidamount = bidamount + b1
+//						break
+//					}
+//				}
+//			}
+//			if bidamount > 0 {
+//				project.Bidamount = bidamount
+//				project.Bidamounttag = 0
+//			} else if bidamount == 0 && info.Bidamount > 0 {
+//				project.Bidamount = info.Bidamount
+//				project.Bidamounttag = 0
+//			}
+//		} else {
+//			//招标没有多包
+//			k := KeyPackage.FindStringSubmatch(info.ProjectName)
+//			if len(k) > 0 {
+//				//招标是单包
+//				if len(project.Package) > 0 {
+//					//项目有多包
+//					flag := false
+//					for _, v := range project.Package {
+//						v1, _ := v.([]map[string]interface{})
+//						if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
+//							flag = true
+//						}
+//					}
+//					if !flag {
+//						project.Bidamount = project.Bidamount + info.Bidamount
+//						project.Bidamounttag = 0
+//					}
+//				} else {
+//					//项目没有多包
+//					if info.Bidamount > 0 {
+//						project.Bidamount = project.Bidamount + info.Bidamount
+//						project.Bidamounttag = 0
+//					} else if info.Bidamount == 0 && tmp["bidamount"] != nil {
+//						project.Bidamounttag = 0
+//					} else {
+//						project.Bidamounttag = 1
+//					}
+//				}
+//			} else {
+//				if info.SubType == "中标" || info.SubType == "成交" {
+//					if info.Bidamount > 0 {
+//						project.Bidamount = info.Bidamount
+//						project.Bidamounttag = 0
+//					} else {
+//						flag := false
+//						if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
+//							for _, res := range project.InfoFiled {
+//								if res.ContractCode != "" && res.ContractCode == info.ContractCode {
+//									flag = true
+//									break
+//								}
+//								if res.Bidamount == project.Bidamount {
+//									flag = true
+//									break
+//								}
+//							}
+//							if !flag {
+//								project.Bidamount = project.Bidamount + info.Bidamount
+//								project.Bidamounttag = 0
+//							} else {
+//								if info.Budget > 0 && project.Bidamount > info.Bidamount {
+//									project.Bidamount = info.Bidamount
+//									project.Bidamounttag = 0
+//								}
+//							}
+//						}
+//					}
+//				}
+//			}
+//		}
+//	} else {
+//		project.Bidamounttag = 1
+//	}
+//
+//}
 
 // 结构体转map
-func StructToMap(filed InfoField) map[string]interface{} {
-	//先转json
-	result, err := json.Marshal(filed)
-	if err != nil {
-		return nil
-	}
-	//json转map
-	res := make(map[string]interface{})
-	err = json.Unmarshal(result, &res)
-	return res
-}
+//func StructToMap(filed InfoField) map[string]interface{} {
+//	//先转json
+//	result, err := json.Marshal(filed)
+//	if err != nil {
+//		return nil
+//	}
+//	//json转map
+//	res := make(map[string]interface{})
+//	err = json.Unmarshal(result, &res)
+//	return res
+//}
 
 func ClearData(clearMap, tmp map[string]interface{}) {
 	for k, v := range tmp {

data_project/task.go

@@ -522,8 +522,7 @@ L:
 				if !siteJudge(util.ObjToString(tmp["spidercode"])) {
 					//extracttype -1: 重复，1: 不重复
 					if util.IntAll(tmp["extracttype"]) == 1 {
-						if util.ObjToString(tmp["toptype"]) != "采购意向" && util.ObjToString(tmp["toptype"]) != "产权" {
-
+						if util.ObjToString(tmp["toptype"]) != "采购意向" && util.ObjToString(tmp["toptype"]) != "产权" && util.ObjToString(tmp["toptype"]) != "拟建" {
 							if P_QL.currentType == "ql" {
 								infoPool <- tmp
 							} else if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {

data_project/update.go

@@ -727,16 +727,33 @@ func (p *ProjectTask) updateProFiled(tmp map[string]interface{}, thisinfo *Info,
 		set["package"] = project.Package
 	}
 	//处理多包后，计算预算金额、中标金额
-	CountAmount(project, thisinfo, tmp)
-	if project.Budget >= 0 && project.Budgettag != 1 {
-		pInfo.Budget = project.Budget
-		set["budget"] = pInfo.Budget
-		set["budgettag"] = 0
+	//CountAmount(project, thisinfo, tmp)
+	//if project.Budget >= 0 && project.Budgettag != 1 {
+	//	pInfo.Budget = project.Budget
+	//	set["budget"] = pInfo.Budget
+	//	set["budgettag"] = 0
+	//}
+	//if pInfo.Bidamount >= 0 && project.Bidamounttag != 1 {
+	//	pInfo.Bidamount = project.Bidamount
+	//	set["bidamount"] = pInfo.Bidamount
+	//	set["bidamounttag"] = 0
+	//}
+	//if pInfo.Bidamount > 0 {
+	//	set["sortprice"] = pInfo.Bidamount
+	//} else {
+	//	if pInfo.Budget > 0 {
+	//		set["sortprice"] = pInfo.Budget
+	//	}
+	//}
+	if thisinfo.Budget > 0 {
+		pInfo.Budget = thisinfo.Budget
+		project.Budget = thisinfo.Budget
+		set["budget"] = thisinfo.Budget
 	}
-	if pInfo.Bidamount >= 0 && project.Bidamounttag != 1 {
-		pInfo.Bidamount = project.Bidamount
-		set["bidamount"] = pInfo.Bidamount
-		set["bidamounttag"] = 0
+	if thisinfo.Bidamount > 0 {
+		pInfo.Bidamount = thisinfo.Bidamount
+		project.Bidamount = thisinfo.Bidamount
+		set["bidamount"] = thisinfo.Bidamount
 	}
 	if pInfo.Bidamount > 0 {
 		set["sortprice"] = pInfo.Bidamount
@@ -746,23 +763,23 @@ func (p *ProjectTask) updateProFiled(tmp map[string]interface{}, thisinfo *Info,
 		}
 	}
 
-	infofield := InfoField{
-		Budget:       thisinfo.Budget,
-		Bidamount:    thisinfo.Bidamount,
-		ContractCode: thisinfo.ContractCode,
-		ProjectName:  thisinfo.ProjectName,
-		ProjectCode:  thisinfo.ProjectCode,
-		Bidstatus:    pInfo.Bidstatus,
-	}
-	copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
-	copyMap[thisinfo.Id] = infofield
-	tmpMap := make(map[string]interface{})
-	for k, v := range copyMap {
-		tmpMap[k] = StructToMap(v)
-	}
-	tmpMap[thisinfo.Id] = StructToMap(infofield)
-	project.InfoFiled = copyMap
-	set["infofield"] = tmpMap
+	//infofield := InfoField{
+	//	Budget:       thisinfo.Budget,
+	//	Bidamount:    thisinfo.Bidamount,
+	//	ContractCode: thisinfo.ContractCode,
+	//	ProjectName:  thisinfo.ProjectName,
+	//	ProjectCode:  thisinfo.ProjectCode,
+	//	Bidstatus:    pInfo.Bidstatus,
+	//}
+	//copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
+	//copyMap[thisinfo.Id] = infofield
+	//tmpMap := make(map[string]interface{})
+	//for k, v := range copyMap {
+	//	tmpMap[k] = StructToMap(v)
+	//}
+	//tmpMap[thisinfo.Id] = StructToMap(infofield)
+	//project.InfoFiled = copyMap
+	//set["infofield"] = tmpMap
 
 	set["mpn"] = pInfo.MPN
 	set["mpc"] = pInfo.MPC
@@ -872,25 +889,25 @@ func (p *ProjectTask) newPro(tmp, tmpPro map[string]interface{}, thisinfo *Info)
 		}
 		tmpPro["winnerorder"] = list
 	}
-	infofield := InfoField{
-		Budget:       thisinfo.Budget,
-		Bidamount:    thisinfo.Bidamount,
-		ContractCode: thisinfo.ContractCode,
-		ProjectName:  thisinfo.ProjectName,
-		ProjectCode:  thisinfo.ProjectCode,
-		Bidstatus:    bs,
-	}
-	res := StructToMap(infofield)
-	tmpPro["infofield"] = map[string]interface{}{
-		thisinfo.Id: res,
-	}
-	if tmp["budget"] != nil && tmp["budget"] != "" {
+	//infofield := InfoField{
+	//	Budget:       thisinfo.Budget,
+	//	Bidamount:    thisinfo.Bidamount,
+	//	ContractCode: thisinfo.ContractCode,
+	//	ProjectName:  thisinfo.ProjectName,
+	//	ProjectCode:  thisinfo.ProjectCode,
+	//	Bidstatus:    bs,
+	//}
+	//res := StructToMap(infofield)
+	//tmpPro["infofield"] = map[string]interface{}{
+	//	thisinfo.Id: res,
+	//}
+	if tmp["budget"] != nil {
 		tmpPro["budget"] = thisinfo.Budget
 		tmpPro["budgettag"] = 0
 	} else {
 		tmpPro["budgettag"] = 1
 	}
-	if tmp["bidamount"] != nil && tmp["bidamount"] != "" {
+	if tmp["bidamount"] != nil {
 		tmpPro["bidamount"] = thisinfo.Bidamount
 		tmpPro["bidamounttag"] = 0
 	} else {
@@ -1118,40 +1135,57 @@ func (p *ProjectTask) updateOldProField(pInfo *ProjectCache, thisinfo *Info, tmp
 		tmpPro["package"] = project.Package
 	}
 	//处理多包后，计算预算金额、中标金额
-	CountAmount(project, thisinfo, tmp)
-	if project.Budget >= 0 && project.Budgettag != 1 {
-		pInfo.Budget = project.Budget
-		tmpPro["budget"] = pInfo.Budget
-		tmpPro["budgettag"] = 0
+	//CountAmount(project, thisinfo, tmp)
+	//if project.Budget >= 0 && project.Budgettag != 1 {
+	//	pInfo.Budget = project.Budget
+	//	tmpPro["budget"] = pInfo.Budget
+	//	tmpPro["budgettag"] = 0
+	//}
+	//if project.Bidamount >= 0 && project.Bidamounttag != 1 {
+	//	pInfo.Bidamount = project.Bidamount
+	//	tmpPro["bidamount"] = pInfo.Bidamount
+	//	tmpPro["bidamounttag"] = 0
+	//}
+	//if pInfo.Bidamount >= pInfo.Budget {
+	//	tmpPro["sortprice"] = pInfo.Bidamount
+	//} else if pInfo.Budget >= pInfo.Bidamount {
+	//	tmpPro["sortprice"] = pInfo.Budget
+	//}
+	if thisinfo.Budget > 0 {
+		pInfo.Budget = thisinfo.Budget
+		project.Budget = thisinfo.Budget
+		tmpPro["budget"] = thisinfo.Budget
 	}
-	if project.Bidamount >= 0 && project.Bidamounttag != 1 {
-		pInfo.Bidamount = project.Bidamount
-		tmpPro["bidamount"] = pInfo.Bidamount
-		tmpPro["bidamounttag"] = 0
+	if thisinfo.Bidamount > 0 {
+		pInfo.Bidamount = thisinfo.Bidamount
+		project.Bidamount = thisinfo.Bidamount
+		tmpPro["bidamount"] = thisinfo.Bidamount
 	}
-	if pInfo.Bidamount >= pInfo.Budget {
+	if pInfo.Bidamount > 0 {
 		tmpPro["sortprice"] = pInfo.Bidamount
-	} else if pInfo.Budget >= pInfo.Bidamount {
-		tmpPro["sortprice"] = pInfo.Budget
+	} else {
+		if pInfo.Budget > 0 {
+			tmpPro["sortprice"] = pInfo.Budget
+		}
 	}
 
-	infofield := InfoField{
-		Budget:       thisinfo.Budget,
-		Bidamount:    thisinfo.Bidamount,
-		ContractCode: thisinfo.ContractCode,
-		ProjectName:  thisinfo.ProjectName,
-		ProjectCode:  thisinfo.ProjectCode,
-		Bidstatus:    pInfo.Bidstatus,
-	}
-	copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
-	copyMap[thisinfo.Id] = infofield
-	tmpMap := make(map[string]interface{})
-	for k, v := range copyMap {
-		tmpMap[k] = StructToMap(v)
-	}
-	tmpMap[thisinfo.Id] = StructToMap(infofield)
-	project.InfoFiled = copyMap
-	tmpPro["infofield"] = tmpMap
+	//infofield := InfoField{
+	//	Budget:       thisinfo.Budget,
+	//	Bidamount:    thisinfo.Bidamount,
+	//	ContractCode: thisinfo.ContractCode,
+	//	ProjectName:  thisinfo.ProjectName,
+	//	ProjectCode:  thisinfo.ProjectCode,
+	//	Bidstatus:    pInfo.Bidstatus,
+	//}
+	//copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
+	//copyMap[thisinfo.Id] = infofield
+	//tmpMap := make(map[string]interface{})
+	//for k, v := range copyMap {
+	//	tmpMap[k] = StructToMap(v)
+	//}
+	//tmpMap[thisinfo.Id] = StructToMap(infofield)
+	//project.InfoFiled = copyMap
+	//tmpPro["infofield"] = tmpMap
 	tmpPro["mpn"] = pInfo.MPN
 	tmpPro["mpc"] = pInfo.MPC
 	tmpPro["updatetime"] = p.pici
@@ -1496,18 +1530,18 @@ func (p *ProjectTask) createPro(tmp map[string]interface{}, thisinfo *Info, pid
 		FirstCooperation(set, p1.Buyer, thisinfo.Winners, thisinfo.EntIdList)
 	}
 
-	infofield := InfoField{
-		Budget:       thisinfo.Budget,
-		Bidamount:    thisinfo.Bidamount,
-		ContractCode: thisinfo.ContractCode,
-		ProjectName:  thisinfo.ProjectName,
-		ProjectCode:  thisinfo.ProjectCode,
-		Bidstatus:    bs,
-	}
-	res := StructToMap(infofield)
-	set["infofield"] = map[string]interface{}{
-		thisinfo.Id: res,
-	}
+	//infofield := InfoField{
+	//	Budget:       thisinfo.Budget,
+	//	Bidamount:    thisinfo.Bidamount,
+	//	ContractCode: thisinfo.ContractCode,
+	//	ProjectName:  thisinfo.ProjectName,
+	//	ProjectCode:  thisinfo.ProjectCode,
+	//	Bidstatus:    bs,
+	//}
+	//res := StructToMap(infofield)
+	//set["infofield"] = map[string]interface{}{
+	//	thisinfo.Id: res,
+	//}
 	if tmp["budget"] != nil && tmp["budget"] != "" {
 		set["budget"] = thisinfo.Budget
 		set["budgettag"] = 0
@@ -1880,16 +1914,33 @@ func (p *ProjectTask) updatePro(set, tmp map[string]interface{}, pInfo *ProjectC
 		set["package"] = project.Package
 	}
 	//处理多包后，计算预算金额、中标金额
-	CountAmount(project, thisinfo, tmp)
-	if project.Budget >= 0 && project.Budgettag != 1 {
-		pInfo.Budget = project.Budget
-		set["budget"] = project.Budget
-		set["budgettag"] = 0
+	//CountAmount(project, thisinfo, tmp)
+	//if project.Budget >= 0 && project.Budgettag != 1 {
+	//	pInfo.Budget = project.Budget
+	//	set["budget"] = project.Budget
+	//	set["budgettag"] = 0
+	//}
+	//if project.Bidamount >= 0 && project.Bidamounttag != 1 {
+	//	pInfo.Bidamount = project.Bidamount
+	//	set["bidamount"] = pInfo.Bidamount
+	//	set["bidamounttag"] = 0
+	//}
+	//if pInfo.Bidamount > 0 {
+	//	set["sortprice"] = pInfo.Bidamount
+	//} else {
+	//	if pInfo.Budget > 0 {
+	//		set["sortprice"] = pInfo.Budget
+	//	}
+	//}
+	if thisinfo.Budget > 0 {
+		pInfo.Budget = thisinfo.Budget
+		project.Budget = thisinfo.Budget
+		set["budget"] = thisinfo.Budget
 	}
-	if project.Bidamount >= 0 && project.Bidamounttag != 1 {
-		pInfo.Bidamount = project.Bidamount
-		set["bidamount"] = pInfo.Bidamount
-		set["bidamounttag"] = 0
+	if thisinfo.Bidamount > 0 {
+		pInfo.Bidamount = thisinfo.Bidamount
+		project.Bidamount = thisinfo.Bidamount
+		set["bidamount"] = thisinfo.Bidamount
 	}
 	if pInfo.Bidamount > 0 {
 		set["sortprice"] = pInfo.Bidamount
@@ -1899,21 +1950,20 @@ func (p *ProjectTask) updatePro(set, tmp map[string]interface{}, pInfo *ProjectC
 		}
 	}
 
-	infofield := InfoField{
-		Budget:       thisinfo.Budget,
-		Bidamount:    thisinfo.Bidamount,
-		ContractCode: thisinfo.ContractCode,
-		ProjectName:  thisinfo.ProjectName,
-		ProjectCode:  thisinfo.ProjectCode,
-		Bidstatus:    bs,
-	}
-
-	tmpMap := set["infofield"].(map[string]interface{})
-	tmpMap[thisinfo.Id] = StructToMap(infofield)
-	set["infofield"] = tmpMap
-	copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
-	copyMap[thisinfo.Id] = infofield
-	project.InfoFiled = copyMap
+	//infofield := InfoField{
+	//	Budget:       thisinfo.Budget,
+	//	Bidamount:    thisinfo.Bidamount,
+	//	ContractCode: thisinfo.ContractCode,
+	//	ProjectName:  thisinfo.ProjectName,
+	//	ProjectCode:  thisinfo.ProjectCode,
+	//	Bidstatus:    bs,
+	//}
+	//tmpMap := set["infofield"].(map[string]interface{})
+	//tmpMap[thisinfo.Id] = StructToMap(infofield)
+	//set["infofield"] = tmpMap
+	//copyMap := util.DeepCopy(project.InfoFiled).(map[string]InfoField)
+	//copyMap[thisinfo.Id] = infofield
+	//project.InfoFiled = copyMap
 
 	set["mpn"] = pInfo.MPN
 	set["mpc"] = pInfo.MPC

data_tidb/bidding.go

@@ -51,13 +51,13 @@ func doBiddingTask(gtid, lteid string, mapInfo map[string]interface{}) {
 			}
 			taskBase(tmp)
 			taskTags(tmp)
-			//taskExpand(tmp)
-			//taskAtts(tmp)
-			//taskInfoformat(tmp)
-			//taskIntent(tmp)
-			//taskWinner(tmp)
-			//taskPackage(tmp)
-			//taskPur(tmp)
+			taskExpand(tmp)
+			taskAtts(tmp)
+			taskInfoformat(tmp)
+			taskIntent(tmp)
+			taskWinner(tmp)
+			taskPackage(tmp)
+			taskPur(tmp)
 
 		}(tmp)
 		tmp = make(map[string]interface{})

data_tidb/go.mod

@@ -8,5 +8,5 @@ require (
 	github.com/spf13/cobra v1.5.0
 	go.mongodb.org/mongo-driver v1.10.3
 	go.uber.org/zap v1.23.0
-	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061 // indirect
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061
 )

data_tidb/go.sum

@@ -1,5 +1,3 @@
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
@@ -51,9 +49,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -144,7 +142,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
@@ -209,7 +206,6 @@ gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

data_tidb/main.go

@@ -35,9 +35,16 @@ func init() {
 
 func main() {
 
-	//go SaveFunc()
-	//go SaveTagFunc()
-	//go saveErrMethod()
+	go SaveFunc()
+	go SaveTagFunc()
+	go SaveExpandFunc()
+	go SaveAttrFunc()
+	go SaveImfFunc()
+	go SaveIntentFunc()
+	go SaveWinnerFunc()
+	go SavePackageFunc()
+	go SavePurFunc()
+	go saveErrMethod()
 
 	//rootCmd := &cobra.Command{Use: "my cmd"}
 	//rootCmd.AddCommand(bidding())
@@ -46,11 +53,11 @@ func main() {
 	//	fmt.Println("rootCmd.Execute failed", err.Error())
 	//}
 
-	taskMysql()
+	//taskMysql()
 
-	//UdpClient = udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
-	//UdpClient.Listen(processUdpMsg)
-	//log.Info("Udp服务监听", zap.String("port:", config.Conf.Udp.LocPort))
+	UdpClient = udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
+	UdpClient.Listen(processUdpMsg)
+	log.Info("Udp服务监听", zap.String("port:", config.Conf.Udp.LocPort))
 
 	c := make(chan bool, 1)
 	<-c
@@ -66,7 +73,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 		log.Info("processUdpMsg", zap.Any("mapInfo:", mapInfo))
 		gtid, _ := mapInfo["gtid"].(string)
 		lteid, _ := mapInfo["lteid"].(string)
-		if err != nil || gtid == "" || lteid == "" {
+		if err != nil {
 			UdpClient.WriteUdp([]byte("tidb udp error"), udp.OP_NOOP, ra) //udp失败回写
 		} else {
 			//udp成功回写

field_sync/main.go

@@ -52,8 +52,8 @@ func init() {
 }
 
 func main() {
-	go checkMapJob()
-	go nsqMethod()
+	//go checkMapJob()
+	//go nsqMethod()
 
 	go UpdateBidding()
 	go UpdateExtract()

field_sync/task.go

@@ -5,7 +5,6 @@ import (
 	"field_sync/config"
 	"field_sync/oss"
 	"fmt"
-	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
@@ -29,15 +28,15 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 	defer util.Catch()
 
 	stype := util.ObjToString(mapInfo["stype"])
-	if stype == "bidding" {
-		uq := bson.M{"gtid": bson.M{"$gte": util.ObjToString(mapInfo["gtid"])},
-			"lteid": bson.M{"$lte": util.ObjToString(mapInfo["lteid"])}}
-		MgoB.Update("bidding_processing_ids", uq, bson.M{"$set": bson.M{"dataprocess": 7, "updatetime": time.Now().Unix()}}, false, true)
-	}
+	//if stype == "bidding" {
+	//	uq := bson.M{"gtid": bson.M{"$gte": util.ObjToString(mapInfo["gtid"])},
+	//		"lteid": bson.M{"$lte": util.ObjToString(mapInfo["lteid"])}}
+	//	MgoB.Update("bidding_processing_ids", uq, bson.M{"$set": bson.M{"dataprocess": 7, "updatetime": time.Now().Unix()}}, false, true)
+	//}
 	// 领域标签处理的数据 id段
-	if stype == "bidding_history" {
-		MgoB.Save("field_data_record", map[string]interface{}{"gtid": mapInfo["gtid"], "lteid": mapInfo["lteid"], "status": 0})
-	}
+	//if stype == "bidding_history" {
+	//	MgoB.Save("field_data_record", map[string]interface{}{"gtid": mapInfo["gtid"], "lteid": mapInfo["lteid"], "status": 0})
+	//}
 
 	q, _ := mapInfo["query"].(map[string]interface{})
 	bkey, _ := mapInfo["bkey"].(string)
@@ -90,14 +89,14 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 		MgoB.DestoryMongoConn(biddingConn)
 	}
 	log.Info("bidding sync...over", zap.Int64("all", count), zap.Int("extract sync", c))
-	NextNode(mapInfo, stype)
-	NextNodePro(mapInfo, stype)
-	NextNodeTidb(mapInfo, stype)
-	if stype == "bidding_history" {
-		NextNodeBidData(mapInfo)  // bidding-data数据
-		NextNodeTidbQyxy(mapInfo) // tidb-企业数据
-		NextNodeHn(mapInfo)
-	}
+	//NextNode(mapInfo, stype)
+	//NextNodePro(mapInfo, stype)
+	//NextNodeTidb(mapInfo, stype)
+	//if stype == "bidding_history" {
+	//	NextNodeBidData(mapInfo)  // bidding-data数据
+	//	NextNodeTidbQyxy(mapInfo) // tidb-企业数据
+	//	NextNodeHn(mapInfo)
+	//}
 }
 
 func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
@@ -328,9 +327,9 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		//------------------对比结束
 
 		//处理key descript
-		//if bkey == "" {
-		//	DealInfo(&tmp, &update)
-		//}
+		if bkey == "" {
+			DealInfo(&tmp, &update)
+		}
 		// entidlist
 		extractMap := make(map[string]interface{})
 		if update["s_winner"] != "" {

monitor/common.toml

@@ -11,6 +11,13 @@ size = 5
 user = ""
 password = ""
 
+[db.mongoP]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+coll = "bidding"
+size = 5
+user = ""
+password = ""
 
 [mail]
 send = false

monitor/config/conf.go

@@ -65,12 +65,14 @@ type log struct {
 
 type db struct {
 	MongoB mgo
+	MongoP mgo
 	Es     es
 }
 
 type mgo struct {
 	Addr     string
 	Dbname   string
+	Coll     string
 	Size     int
 	User     string
 	Password string

monitor/init.go

@@ -9,7 +9,7 @@ import (
 )
 
 var (
-	MgoB *mongodb.MongodbSim
+	MgoB, MgoP *mongodb.MongodbSim
 )
 
 // InitLog @Description
@@ -42,4 +42,12 @@ func InitMgo() {
 	}
 	MgoB.InitPool()
 
+	MgoP = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoP.Addr,
+		DbName:      config.Conf.DB.MongoP.Dbname,
+		Size:        config.Conf.DB.MongoP.Size,
+		UserName:    config.Conf.DB.MongoP.User,
+		Password:    config.Conf.DB.MongoP.Password,
+	}
+	MgoB.InitPool()
 }

monitor/main.go

@@ -13,13 +13,16 @@ import (
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"monitor/config"
 	"net/http"
+	"time"
 )
 
 var (
-	WebUrl = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=45962efc-ca87-4996-9ffa-08bf6608ab7a"
+	WebUrl  = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=45962efc-ca87-4996-9ffa-08bf6608ab7a"
+	WebUrl1 = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=1594c58d-c279-4fdc-8d44-c201160e3731"
 
 	WarningStr  = "数据采集bidding_file表数据，已累计%d条数据未处理"
 	WarningStr1 = "标的物等字段识别提醒，bidding表数据已积累%d条未处理"
+	WarningStr2 = "项目合并表数据，超过一个1小时(查询时间点: %d)未有合并新数据"
 )
 
 func init() {
@@ -46,11 +49,16 @@ func biddingFile() *cobra.Command {
 			_ = crn.AddFunc("@hourly", func() {
 				taskFile()
 			})
-			crn.Start()
 
 			_ = crn.AddFunc("0 */30 * * * ?", func() {
 				taskPy()
 			})
+
+			_ = crn.AddFunc("@hourly", func() {
+				taskProject()
+			})
+			crn.Start()
+
 			c := make(chan bool, 1)
 			<-c
 		},
@@ -80,6 +88,16 @@ func taskPy() {
 	}
 }
 
+func taskProject() {
+	now := time.Now().Unix() - 1*60*60
+	c := MgoP.Count(config.Conf.DB.MongoP.Coll, bson.M{"pici": bson.M{"$gte": now}})
+	if c <= 0 {
+		SendMsg1(fmt.Sprintf(WarningStr2, now))
+	} else {
+		log.Info("projectset", zap.Int("count", c))
+	}
+}
+
 func SendMsg(content string) {
 	client := &http.Client{}
 	data := map[string]interface{}{"msgtype": "text", "text": map[string]interface{}{
@@ -91,3 +109,15 @@ func SendMsg(content string) {
 	body, _ := ioutil.ReadAll(resp.Body)
 	log.Info("SendMsg", zap.String("resp", string(body)))
 }
+
+func SendMsg1(content string) {
+	client := &http.Client{}
+	data := map[string]interface{}{"msgtype": "text", "text": map[string]interface{}{
+		"content": content, "mentioned_mobile_list": []string{"15090279371", "15639297172"},
+	}}
+	bytesData, _ := json.Marshal(data)
+	req, _ := http.NewRequest("POST", WebUrl1, bytes.NewReader(bytesData))
+	resp, _ := client.Do(req)
+	body, _ := ioutil.ReadAll(resp.Body)
+	log.Info("SendMsg", zap.String("resp", string(body)))
+}