no message

fieldproject_com/main.go

@@ -44,7 +44,7 @@ func init() {
 	InitLog()
 	InitMgo()
 	InitMysql()
-	//InitEs()
+	InitEs()
 
 	ChEs = make(chan bool, 10)
 	saveSize = 200

proposed_project/common.toml

@@ -41,11 +41,15 @@ password = ""
 
 [redis]
 addr = "192.168.3.207:2679"
-pcode = "project"
-db = 1
+project = "project"
+proposed = "proposed"
+dbT = 1
+dbD = 2
 
 [db.es]
-addr = "http://127.0.0.1:19800"
+addr = "http://192.168.3.149:9200"
+user = ""
+password = ""
 size = 5
 indexP = "proposed_v1"
 # minimum_should_match

proposed_project/config/conf.go

@@ -42,10 +42,12 @@ type udp struct {
 }
 
 type es struct {
-	Addr    string
-	Size    int
-	IndexP  string
-	MinSdMh string
+	Addr     string
+	User     string
+	Password string
+	Size     int
+	IndexP   string
+	MinSdMh  string
 }
 
 type mail struct {
@@ -85,9 +87,11 @@ type mgo struct {
 }
 
 type redis struct {
-	Addr  string
-	Pcode string
-	Db    int
+	Addr     string
+	Project  string
+	Proposed string
+	Dbt      int
+	Dbd      int
 }
 
 type mysql struct {

proposed_project/go.mod

@@ -3,7 +3,7 @@ module proposed_project
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20230308013947-0ef15d576c68
+	app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061
 	github.com/BurntSushi/toml v1.2.0
 	github.com/go-ego/gse v0.80.2
 	github.com/robfig/cron v1.2.0

proposed_project/go.sum

@@ -1,5 +1,7 @@
 app.yhyue.com/data_processing/common_utils v0.0.0-20230308013947-0ef15d576c68 h1:560oa2np5fPKQOGvsOYMV9vNQ8Y6y9O+WlUoNbnZ0UU=
 app.yhyue.com/data_processing/common_utils v0.0.0-20230308013947-0ef15d576c68/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
+app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061 h1:UaE1gor8fIhSBAXcZsNOQDMI5sD9LxGECXFh7Ik4rk4=
+app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=

proposed_project/init.go

@@ -111,8 +111,10 @@ func InitMysql() {
 
 func InitEs() {
 	Es = &elastic.Elastic{
-		S_esurl: config.Conf.DB.Es.Addr,
-		I_size:  config.Conf.DB.Es.Size,
+		S_esurl:  config.Conf.DB.Es.Addr,
+		I_size:   config.Conf.DB.Es.Size,
+		Username: config.Conf.DB.Es.User,
+		Password: config.Conf.DB.Es.Password,
 	}
 	Es.InitElasticSize()
 }

proposed_project/load_data.go

@@ -14,7 +14,7 @@ func loadProject() {
 	sess := MgoPro.GetMgoConn()
 	defer MgoPro.DestoryMongoConn(sess)
 	f := map[string]interface{}{"list": 0}
-	it := sess.DB(MgoPro.DbName).C(config.Conf.Serve.ProColl).Find(nil).Select(f).Iter()
+	it := sess.DB(MgoPro.DbName).C(config.Conf.DB.MongoP.ProposedColl).Find(nil).Select(f).Iter()
 	count := 0
 
 	ch := make(chan bool, 3)

proposed_project/main.go

@@ -291,10 +291,11 @@ func projectComb() *cobra.Command {
 			initSeg()
 			InitEs()
 
-			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Pcode, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Db)
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Project, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbt)
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Proposed, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbd)
 
 			go SavePpMethod()
-			taskC()
+			taskComb()
 			c := make(chan bool, 1)
 			<-c
 		},
@@ -313,7 +314,9 @@ func projectCombTidb() *cobra.Command {
 			InitMysql()
 			initStage()
 
-			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Pcode, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Db)
+			redis.InitRedis1("ent_id=172.17.4.189:8379", 6) // name_id
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Project, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbt)
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Proposed, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbd)
 			go SaveEntFunc1("dwd_f_nzj_ent", EntField)
 			taskD()
 			c := make(chan bool, 1)
@@ -331,17 +334,23 @@ func projectCombAdd() *cobra.Command {
 		Short: "Start processing combined project add",
 		Run: func(cmd *cobra.Command, args []string) {
 
+			initSeg()
+			InitEs()
 			InitMysql()
 			initStage()
 
-			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Pcode, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Db)
+			redis.InitRedis1("ent_id=172.17.4.189:8379", 6) // name_id
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Project, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbt)
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Proposed, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbd)
 			go SaveEntFunc1("dwd_f_nzj_ent", EntField)
 
+			go taskAA()
+			go taskBB()
+
 			crn := cron.New()
-			cronstr := "0 */30 * * * ?" // 每30min执行一次
+			cronstr := "0 0 22 * * ?" // 每天10点执行一次
 			_ = crn.AddFunc(cronstr, func() {
 				if TaskSingle {
-					TaskSingle = false
 					go taskAA()
 					go taskBB()
 				} else {
@@ -479,7 +488,7 @@ func esDisp() {
 		q = bson.M{"pici": bson.M{"$gt": pici}}
 	}
 
-	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.Serve.ProColl).Find(q).Select(SelectF).Iter()
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(q).Select(SelectF).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
 		if count%200 == 0 {

proposed_project/proTask.go

@@ -27,7 +27,7 @@ func updateAllQueue() {
 					defer func() {
 						<-updateSp
 					}()
-					MgoPro.UpSertBulk(config.Conf.Serve.ProColl, arru...)
+					MgoPro.UpSertBulk(config.Conf.DB.MongoP.ProposedColl, arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, saveSize)
 				indexu = 0
@@ -39,7 +39,7 @@ func updateAllQueue() {
 					defer func() {
 						<-updateSp
 					}()
-					MgoPro.UpSertBulk(config.Conf.Serve.ProColl, arru...)
+					MgoPro.UpSertBulk(config.Conf.DB.MongoP.ProposedColl, arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, saveSize)
 				indexu = 0

proposed_project/projectTask.go

@@ -32,7 +32,9 @@ var (
 	reg1    = regexp.MustCompile("分布式光伏发电|自然人|出让|国有建设用地使用权")
 
 	field  = []string{"projectname.pname"}
-	sField = []string{"projectname", "bidstatus", "firsttime", "_id", "area", "city", "district"}
+	sField = []string{"projectname", "bidstatus", "firsttime", "_id", "area", "city", "district", "ids"}
+
+	esQ = `{"query": {"bool": {"must": [{"multi_match": {"query": "%s","type": "phrase","fields": ["projectname"]}}]}},"size": 100}`
 
 	savePpPool = make(chan map[string]interface{}, 5000)
 	savePpSp   = make(chan bool, 3)
@@ -53,7 +55,9 @@ func initSeg() {
 	//sort.Strings(stopWords)
 }
 
-func taskC() {
+// @Description 关联
+// @Author J 2023/4/25 10:26
+func taskComb() {
 	sess := MgoPro.GetMgoConn()
 	defer MgoPro.DestoryMongoConn(sess)
 
@@ -68,6 +72,7 @@ func taskC() {
 		"area":          1,
 		"city":          1,
 		"district":      1,
+		"ids":           1,
 	}
 
 	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(nil).Select(f).Iter()
@@ -106,11 +111,11 @@ func taskC() {
 
 			wds, q := Method1(util.ObjToString(tmp["projectname"]))
 			if q != "" {
-				binfo := Es.Get("projectset_v1", q)
+				binfo := Es.Get("projectset", q)
 				if binfo != nil && len(*binfo) > 0 {
 					n2 = len(*binfo)
 					for _, m := range *binfo {
-						if b, _ := redis.Exists(config.Conf.DB.Redis.Pcode, util.ObjToString(m["_id"])); b {
+						if b, _ := redis.Exists(config.Conf.DB.Redis.Project, util.ObjToString(m["_id"])); b {
 							continue
 						}
 						if util.ObjToString(m["bidstatus"]) == "拟建" {
@@ -122,7 +127,9 @@ func taskC() {
 							continue
 						}
 
-						redis.PutCKV(config.Conf.DB.Redis.Pcode, util.ObjToString(m["_id"]), mongodb.BsonIdToSId(tmp["_id"]))
+						size := len(m["ids"].([]interface{}))
+						redis.PutCKV(config.Conf.DB.Redis.Project, util.ObjToString(m["_id"]), fmt.Sprintf("%s-%d", mongodb.BsonIdToSId(tmp["_id"]), size))
+						redis.PutCKV(config.Conf.DB.Redis.Proposed, mongodb.BsonIdToSId(tmp["_id"]), len(tmp["ids"].([]interface{})))
 						mArr = append(mArr, map[string]interface{}{"pid": util.ObjToString(m["_id"]), "projectname": m["projectname"], "source": 2})
 
 					}
@@ -480,7 +487,7 @@ func taskD() {
 			}()
 
 			if ids, ok := tmp["ids"].([]interface{}); ok {
-				//id := mongodb.BsonIdToSId(tmp["_id"])
+				id := mongodb.BsonIdToSId(tmp["_id"])
 
 				for _, p := range ids {
 					p1 := p.(map[string]interface{})
@@ -498,67 +505,67 @@ func taskD() {
 							m["infoid"] = util.ObjToString(l1["infoid"])
 							m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(l1["infoid"])))
 							m["createtime"] = time.Now().Format(util.Date_Full_Layout)
-							MgoPro.Save("projectset_comb_temp1", m)
-							//MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+							//MgoPro.Save("projectset_comb_temp1", m)
+							MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+						}
+					}
+					if buyer := util.ObjToString((*info)["buyer"]); buyer != "" {
+						s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": buyer})
+						if s <= 0 {
+							saveEnt := make(map[string]interface{})
+							saveEnt["proposed_id"] = id
+							saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+							saveEnt["name"] = buyer
+							if eid := redis.GetStr("ent_id", buyer); eid != "" {
+								arr := strings.Split(eid, "_")
+								saveEnt["name_id"] = arr[0]
+								if len(arr) == 2 {
+									saveEnt["area_code"] = arr[1]
+								} else if len(arr) == 3 {
+									saveEnt["city_code"] = arr[2]
+								}
+								info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
+								if info != nil && len(*info) > 0 {
+									saveEnt["address"] = (*info)[0]["address"]
+								}
+							}
+							saveEnt["identity_type"] = 2
+							saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+							saveEntPool1 <- saveEnt
+						}
+					}
+					if winner := util.ObjToString((*info)["buyer"]); winner != "" {
+						for _, w := range strings.Split(winner, ",") {
+							s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": w})
+							if s <= 0 {
+								saveEnt := make(map[string]interface{})
+								saveEnt["proposed_id"] = id
+								saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+								saveEnt["name"] = w
+								if eid := redis.GetStr("ent_id", w); eid != "" {
+									arr := strings.Split(eid, "_")
+									saveEnt["name_id"] = arr[0]
+									if len(arr) == 2 {
+										saveEnt["area_code"] = arr[1]
+									} else if len(arr) == 3 {
+										saveEnt["city_code"] = arr[2]
+									}
+									info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
+									if info != nil && len(*info) > 0 {
+										saveEnt["address"] = (*info)[0]["address"]
+									}
+								}
+								saveEnt["identity_type"] = 3
+								saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+								saveEntPool1 <- saveEnt
+							}
 						}
 					}
-					//if buyer := util.ObjToString((*info)["buyer"]); buyer != "" {
-					//	s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": buyer})
-					//	if s <= 0 {
-					//		saveEnt := make(map[string]interface{})
-					//		saveEnt["proposed_id"] = id
-					//		saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
-					//		saveEnt["name"] = buyer
-					//		if eid := redis.GetStr("ent_id", buyer); eid != "" {
-					//			arr := strings.Split(eid, "_")
-					//			saveEnt["name_id"] = arr[0]
-					//			if len(arr) == 2 {
-					//				saveEnt["area_code"] = arr[1]
-					//			} else if len(arr) == 3 {
-					//				saveEnt["city_code"] = arr[2]
-					//			}
-					//			info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
-					//			if info != nil && len(*info) > 0 {
-					//				saveEnt["address"] = (*info)[0]["address"]
-					//			}
-					//		}
-					//		saveEnt["identity_type"] = 2
-					//		saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
-					//		saveEntPool1 <- saveEnt
-					//	}
-					//}
-					//if winner := util.ObjToString((*info)["buyer"]); winner != "" {
-					//	for _, w := range strings.Split(winner, ",") {
-					//		s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": w})
-					//		if s <= 0 {
-					//			saveEnt := make(map[string]interface{})
-					//			saveEnt["proposed_id"] = id
-					//			saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
-					//			saveEnt["name"] = w
-					//			if eid := redis.GetStr("ent_id", w); eid != "" {
-					//				arr := strings.Split(eid, "_")
-					//				saveEnt["name_id"] = arr[0]
-					//				if len(arr) == 2 {
-					//					saveEnt["area_code"] = arr[1]
-					//				} else if len(arr) == 3 {
-					//					saveEnt["city_code"] = arr[2]
-					//				}
-					//				info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
-					//				if info != nil && len(*info) > 0 {
-					//					saveEnt["address"] = (*info)[0]["address"]
-					//				}
-					//			}
-					//			saveEnt["identity_type"] = 3
-					//			saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
-					//			saveEntPool1 <- saveEnt
-					//		}
-					//	}
-					//}
 				}
 
-				//size := MysqlTool.Count("dwd_f_nzj_follw_record", bson.M{"proposed_id": id})
-				//info := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"proposed_id": id}, "project_stage_code", "publishtime desc")
-				//MysqlTool.Update("dwd_f_nzj_baseinfo", bson.M{"proposed_id": id}, bson.M{"follow_num": size, "project_stage_code": (*info)["project_stage_code"], "updatetime": time.Now().Format(util.Date_Full_Layout)})
+				size := MysqlTool.Count("dwd_f_nzj_follw_record", bson.M{"proposed_id": id})
+				info := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"proposed_id": id}, "project_stage_code", "publishtime desc")
+				MysqlTool.Update("dwd_f_nzj_baseinfo", bson.M{"proposed_id": id}, bson.M{"follow_num": size, "project_stage_code": (*info)["project_stage_code"], "updatetime": time.Now().Format(util.Date_Full_Layout)})
 			}
 
 		}(tmp)
@@ -595,7 +602,7 @@ func taskAA() {
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
 		if count%2000 == 0 {
-			log.Info(fmt.Sprintf("current --- %d", count))
+			log.Info(fmt.Sprintf("project current --- %d", count))
 		}
 		if pc := util.Int64All(tmp["pici"]); pc > config.Conf.Serve.Pici {
 			config.Conf.Serve.Pici = pc
@@ -607,24 +614,54 @@ func taskAA() {
 				<-ch
 				wg.Done()
 			}()
+			bUpdate := false
 			id := mongodb.BsonIdToSId(tmp["_id"])
-			if str := redis.GetStr(config.Conf.DB.Redis.Pcode, id); str != "" {
+			if str := redis.GetStr(config.Conf.DB.Redis.Project, id); str != "" {
 				strs := strings.Split(str, "-")
-				if len(tmp["list"].([]interface{})) != util.IntAll(strs[1]) {
-					for _, info := range tmp["list"].([]interface{}) {
-						info1 := info.(map[string]interface{})
-
+				size := len(tmp["list"].([]interface{}))
+				if size != util.IntAll(strs[1]) {
+					list := tmp["list"].([]interface{})
+					for k := range list {
+						info1 := list[size-k-1].(map[string]interface{}) //倒序
+						infoid := util.ObjToString(info1["infoid"])
+						binfo := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"infoid": infoid}, "", "")
+						if binfo != nil && len(*binfo) > 0 {
+							break
+						} else {
+							m := make(map[string]interface{})
+							m["project_stage_code"] = tagFunc(info1)
+							m["proposed_id"] = strs[0]
+							m["title"] = util.ObjToString(info1["title"])
+							if t := util.Int64All(info1["publishtime"]); t > 0 {
+								m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+							}
+							m["infoid"] = util.ObjToString(info1["infoid"])
+							m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(info1["infoid"])))
+							m["createtime"] = time.Now().Format(util.Date_Full_Layout)
+							MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+							bUpdate = true
+						}
 					}
+					redis.PutCKV(config.Conf.DB.Redis.Project, id, fmt.Sprintf("%s-%d", strs[0], size))
 				}
 			} else {
-
+				// 新项目
+				// 新项目是否都有必要进行关联(拟建项目)
+				bUpdate = Method3(tmp)
+			}
+			// 更新拟在建基本信息表
+			if bUpdate {
+				size := MysqlTool.Count("dwd_f_nzj_follw_record", bson.M{"proposed_id": id})
+				info := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"proposed_id": id}, "project_stage_code", "publishtime desc")
+				if info != nil && len(*info) > 0 {
+					MysqlTool.Update("dwd_f_nzj_baseinfo", bson.M{"proposed_id": id}, bson.M{"follow_num": size, "project_stage_code": (*info)["project_stage_code"], "updatetime": time.Now().Format(util.Date_Full_Layout)})
+				}
 			}
-
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
 	wg.Wait()
-	log.Info(fmt.Sprintf("over --- %d, pici ---%d", count, config.Conf.Serve.Pici))
+	log.Info(fmt.Sprintf("project over --- %d, pici ---%d", count, config.Conf.Serve.Pici))
 }
 
 // @Description 拟建项目增量
@@ -636,14 +673,16 @@ func taskBB() {
 	ch := make(chan bool, config.Conf.Serve.Thread)
 	wg := &sync.WaitGroup{}
 
-	//q := bson.M{"_id": mongodb.StringTOBsonId("60a2995b8a2adb30a57172ec")}
-	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.CombColl).Find(nil).Iter()
+	q := bson.M{"pici": bson.M{"$gte": config.Conf.Serve.Pici}}
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(q).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
 		if count%2000 == 0 {
-			log.Info(fmt.Sprintf("current --- %d", count))
+			log.Info(fmt.Sprintf("proposed current --- %d", count))
+		}
+		if pc := util.Int64All(tmp["pici"]); pc > config.Conf.Serve.Pici {
+			config.Conf.Serve.Pici = pc
 		}
-
 		ch <- true
 		wg.Add(1)
 		go func(tmp map[string]interface{}) {
@@ -651,94 +690,169 @@ func taskBB() {
 				<-ch
 				wg.Done()
 			}()
-
-			if ids, ok := tmp["ids"].([]interface{}); ok {
-				//id := mongodb.BsonIdToSId(tmp["_id"])
-
-				for _, p := range ids {
-					p1 := p.(map[string]interface{})
-					info, _ := MgoPro.FindById(config.Conf.DB.MongoP.ProjectColl, util.ObjToString(p1["pid"]), nil)
-					if list, ok1 := (*info)["list"].([]interface{}); ok1 {
-						for _, l := range list {
-							l1 := l.(map[string]interface{})
+			bUpdate := false
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			if num := redis.GetInt(config.Conf.DB.Redis.Proposed, id); num > 0 {
+				size := len(tmp["list"].([]interface{}))
+				list := tmp["list"].([]interface{})
+				if num != size {
+					for k := range list {
+						info1 := list[size-k-1].(map[string]interface{})
+						infoid := util.ObjToString(info1["infoid"])
+						binfo := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"infoid": infoid}, "", "")
+						if binfo != nil && len(*binfo) > 0 {
+							break
+						} else {
 							m := make(map[string]interface{})
-							m["project_stage_code"] = tagFunc(l1)
-							m["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
-							m["title"] = util.ObjToString(l1["title"])
-							if t := util.Int64All(l1["publishtime"]); t > 0 {
+							m["project_stage_code"] = tagFunc(info1)
+							m["proposed_id"] = id
+							m["title"] = util.ObjToString(info1["title"])
+							if t := util.Int64All(info1["publishtime"]); t > 0 {
 								m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
 							}
-							m["infoid"] = util.ObjToString(l1["infoid"])
-							m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(l1["infoid"])))
+							m["infoid"] = util.ObjToString(info1["infoid"])
+							m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(info1["infoid"])))
 							m["createtime"] = time.Now().Format(util.Date_Full_Layout)
-							MgoPro.Save("projectset_comb_temp1", m)
-							//MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+							MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+							bUpdate = true
 						}
 					}
-					//if buyer := util.ObjToString((*info)["buyer"]); buyer != "" {
-					//	s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": buyer})
-					//	if s <= 0 {
-					//		saveEnt := make(map[string]interface{})
-					//		saveEnt["proposed_id"] = id
-					//		saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
-					//		saveEnt["name"] = buyer
-					//		if eid := redis.GetStr("ent_id", buyer); eid != "" {
-					//			arr := strings.Split(eid, "_")
-					//			saveEnt["name_id"] = arr[0]
-					//			if len(arr) == 2 {
-					//				saveEnt["area_code"] = arr[1]
-					//			} else if len(arr) == 3 {
-					//				saveEnt["city_code"] = arr[2]
-					//			}
-					//			info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
-					//			if info != nil && len(*info) > 0 {
-					//				saveEnt["address"] = (*info)[0]["address"]
-					//			}
-					//		}
-					//		saveEnt["identity_type"] = 2
-					//		saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
-					//		saveEntPool1 <- saveEnt
-					//	}
-					//}
-					//if winner := util.ObjToString((*info)["buyer"]); winner != "" {
-					//	for _, w := range strings.Split(winner, ",") {
-					//		s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": w})
-					//		if s <= 0 {
-					//			saveEnt := make(map[string]interface{})
-					//			saveEnt["proposed_id"] = id
-					//			saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
-					//			saveEnt["name"] = w
-					//			if eid := redis.GetStr("ent_id", w); eid != "" {
-					//				arr := strings.Split(eid, "_")
-					//				saveEnt["name_id"] = arr[0]
-					//				if len(arr) == 2 {
-					//					saveEnt["area_code"] = arr[1]
-					//				} else if len(arr) == 3 {
-					//					saveEnt["city_code"] = arr[2]
-					//				}
-					//				info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
-					//				if info != nil && len(*info) > 0 {
-					//					saveEnt["address"] = (*info)[0]["address"]
-					//				}
-					//			}
-					//			saveEnt["identity_type"] = 3
-					//			saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
-					//			saveEntPool1 <- saveEnt
-					//		}
-					//	}
-					//}
+					redis.PutCKV(config.Conf.DB.Redis.Proposed, id, size)
 				}
+			} else {
+				// 新项目
+				// 1、关联
+				wds, q := Method1(util.ObjToString(tmp["projectname"]))
+				if q != "" {
+					binfo := Es.Get("projectset", q)
+					if binfo != nil && len(*binfo) > 0 {
+						for _, m := range *binfo {
+							if b, _ := redis.Exists(config.Conf.DB.Redis.Project, util.ObjToString(m["_id"])); b {
+								continue
+							}
+							if util.ObjToString(m["bidstatus"]) == "拟建" {
+								continue
+							}
+							if judgeArea(tmp, m, wds) {
+								continue
+							}
 
-				//size := MysqlTool.Count("dwd_f_nzj_follw_record", bson.M{"proposed_id": id})
-				//info := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"proposed_id": id}, "project_stage_code", "publishtime desc")
-				//MysqlTool.Update("dwd_f_nzj_baseinfo", bson.M{"proposed_id": id}, bson.M{"follow_num": size, "project_stage_code": (*info)["project_stage_code"], "updatetime": time.Now().Format(util.Date_Full_Layout)})
+							size := len(m["ids"].([]interface{}))
+							redis.PutCKV(config.Conf.DB.Redis.Project, util.ObjToString(m["_id"]), fmt.Sprintf("%s-%d", mongodb.BsonIdToSId(tmp["_id"]), size))
+							redis.PutCKV(config.Conf.DB.Redis.Proposed, mongodb.BsonIdToSId(tmp["_id"]), len(tmp["ids"].([]interface{})))
+
+							bUpdate = true
+							// 2、保存信息到tidb
+							if list, ok1 := m["list"].([]interface{}); ok1 {
+								for _, l := range list {
+									l1 := l.(map[string]interface{})
+									m := make(map[string]interface{})
+									m["project_stage_code"] = tagFunc(l1)
+									m["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+									m["title"] = util.ObjToString(l1["title"])
+									if t := util.Int64All(l1["publishtime"]); t > 0 {
+										m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+									}
+									m["infoid"] = util.ObjToString(l1["infoid"])
+									m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(l1["infoid"])))
+									m["createtime"] = time.Now().Format(util.Date_Full_Layout)
+									MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+								}
+							}
+							if buyer := util.ObjToString(m["buyer"]); buyer != "" {
+								s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": buyer})
+								if s <= 0 {
+									saveEnt := make(map[string]interface{})
+									saveEnt["proposed_id"] = id
+									saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+									saveEnt["name"] = buyer
+									if eid := redis.GetStr("ent_id", buyer); eid != "" {
+										arr := strings.Split(eid, "_")
+										saveEnt["name_id"] = arr[0]
+										if len(arr) == 2 {
+											saveEnt["area_code"] = arr[1]
+										} else if len(arr) == 3 {
+											saveEnt["city_code"] = arr[2]
+										}
+										info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
+										if info != nil && len(*info) > 0 {
+											saveEnt["address"] = (*info)[0]["address"]
+										}
+									}
+									saveEnt["identity_type"] = 2
+									saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+									saveEntPool1 <- saveEnt
+								}
+							}
+							if winner := util.ObjToString(m["buyer"]); winner != "" {
+								for _, w := range strings.Split(winner, ",") {
+									s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": w})
+									if s <= 0 {
+										saveEnt := make(map[string]interface{})
+										saveEnt["proposed_id"] = id
+										saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+										saveEnt["name"] = w
+										if eid := redis.GetStr("ent_id", w); eid != "" {
+											arr := strings.Split(eid, "_")
+											saveEnt["name_id"] = arr[0]
+											if len(arr) == 2 {
+												saveEnt["area_code"] = arr[1]
+											} else if len(arr) == 3 {
+												saveEnt["city_code"] = arr[2]
+											}
+											info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
+											if info != nil && len(*info) > 0 {
+												saveEnt["address"] = (*info)[0]["address"]
+											}
+										}
+										saveEnt["identity_type"] = 3
+										saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+										saveEntPool1 <- saveEnt
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+			// 更新拟在建基本信息表
+			if bUpdate {
+				size := MysqlTool.Count("dwd_f_nzj_follw_record", bson.M{"proposed_id": id})
+				info := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"proposed_id": id}, "project_stage_code", "publishtime desc")
+				MysqlTool.Update("dwd_f_nzj_baseinfo", bson.M{"proposed_id": id}, bson.M{"follow_num": size, "project_stage_code": (*info)["project_stage_code"], "updatetime": time.Now().Format(util.Date_Full_Layout)})
 			}
-
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
 	wg.Wait()
-	log.Info(fmt.Sprintf("over --- %d", count))
+	log.Info(fmt.Sprintf("proposed over --- %d, pici ---%d", count, config.Conf.Serve.Pici))
+}
+
+func Method3(tmp map[string]interface{}) bool {
+	pname := util.ObjToString(tmp["projectname"])
+	pname = strings.ReplaceAll(pname, "\"", "'")
+	binfo := Es.Get("proposed", fmt.Sprintf(esQ, pname))
+	if binfo != nil && len(*binfo) > 0 {
+		if list, ok1 := (*binfo)[0]["list"].([]interface{}); ok1 {
+			for _, l := range list {
+				l1 := l.(map[string]interface{})
+				m := make(map[string]interface{})
+				m["project_stage_code"] = tagFunc(l1)
+				m["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+				m["title"] = util.ObjToString(l1["title"])
+				if t := util.Int64All(l1["publishtime"]); t > 0 {
+					m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+				}
+				m["infoid"] = util.ObjToString(l1["infoid"])
+				m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(l1["infoid"])))
+				m["createtime"] = time.Now().Format(util.Date_Full_Layout)
+				MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+			}
+			redis.PutCKV(config.Conf.DB.Redis.Project, mongodb.BsonIdToSId(tmp["_id"]), fmt.Sprintf("%s-%d", util.ObjToString((*binfo)[0]["_id"]), len(list)))
+			return true
+		}
+	}
+	return false
 }
 
 var saveEntPool1 = make(chan map[string]interface{}, 5000)

proposed_project/tidbTask.go

@@ -91,9 +91,9 @@ func taskTidb(q map[string]interface{}) {
 	wg := &sync.WaitGroup{}
 	var query *mongodb.MgoIter
 	if q != nil && len(q) > 0 {
-		query = sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.Serve.ProColl).Find(q).Select(SelectF).Iter()
+		query = sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(q).Select(SelectF).Iter()
 	} else {
-		query = sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.Serve.ProColl).Find(nil).Select(SelectF).Iter()
+		query = sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(nil).Select(SelectF).Iter()
 	}
 
 	count := 0
@@ -269,7 +269,7 @@ func taskTidb_add(q map[string]interface{}) {
 	wg := &sync.WaitGroup{}
 
 	log.Info("taskTidb_add", zap.Any("q: ", q))
-	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.Serve.ProColl).Find(q).Select(nil).Iter()
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(q).Select(nil).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
 		if count%200 == 0 {