no message

fullproject/src_v1/config.json

@@ -76,9 +76,10 @@
         "blacklist": ["有问题", "个县公司", "T及分公司", "大厦\\d+室", "住宅楼", "购物中心", "盖法人章", "东侧路面拓宽", "、技术研发中心", "地下室", "车库", "二公司", "二类居住", "钢芯铝绞线", "已仔细"]
     },
     "agency": {
-        "pre_regexp": ["<[^>]*>", "(\\(|（)?[a-zA-Z](\\)|）)?", "(\\(|（)?\\d{0,8}.?\\d+(\\)|）)?", "#(~|、)?", "#\\d{1,};?", "(\\+|\\.|、|-|#|%|:|：)+"],
-        "back_regexp": ["为其所$"],
-        "back_rep_regexp": ["(有限公$)#有限公司", "(有限责任公$)#有限责任公司", "(公司公司&)#公司", "(公司等&)#公司", "([a-z]限公司$)#有限公司", "(有公司$)#有限公司", "(招标代理有$)#招标代理有限公司"],
-        "blacklist": []
+        "pre_regexp": ["发布人:", "<[^>]*>", "(\\(|（)?[a-zA-Z](\\)|）)?", "(\\(|（)?\\d{0,8}.?\\d+(\\)|）)?", "#(~|、)?", "#\\d{1,};?", "(\\+|\\.|、|-|#|%|:|：)+"],
+        "back_regexp": ["为其所$", "（简称$", ":我公司$"],
+        "back_rep_regexp": ["(有限公$)#有限公司", "(有限公同$)#有限公司", "(有限责任公$)#有限责任公司", "(公司公司&)#公司", "(公司等&)#公司", "([a-z]限公司$)#有限公司", "(有公司$)#有限公司", "(招标代理有$)#招标代理有限公司"],
+        "blacklist": ["承包项目", "（二标段", "举行了", "提交一份公司", "驱动更新", "邱海丽", "集中代理采购", "公司组织的", "利用项目", "地址:", "交易网", "年龄:", "性别:", "竞争性谈判", "null", "停车场", "招标人", "制造项目", "(中|招)标公告",
+            "“组组通”工程", "备案单位（章）", "小时", "梁工"]
     }
 }

fullproject/src_v1/init.go

@@ -76,7 +76,7 @@ func init() {
 	udpclient.Listen(processUdpMsg)
 	log.Println("Udp服务监听", udpport)
 
-	BlackList = Sysconfig["blacklist"].([]interface{})
+	BlackList = Sysconfig["rp_blacklist"].([]interface{})
 	BlaskListMap = make(map[string]bool)
 	for _, v := range BlackList {
 		BlaskListMap[util.ObjToString(v)] = true

fullproject/src_v1/project.go

@@ -455,11 +455,6 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 			set[f] = tmp[f]
 		}
 	}
-	//保存补充过后的area、city
-	if qu.ObjToString(set["area"]) != thisinfo.Area {
-		set["area"] = thisinfo.Area
-		set["city"] = thisinfo.City
-	}
 	bidopentime := qu.Int64All(tmp["bidopentime"])
 	if bidopentime > 0 {
 		set["bidopentime"] = bidopentime
@@ -808,24 +803,36 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		set["exception"] = ex
 	}
 	//3\4\5--省、市、县
+	//if thisinfo.Area != "全国" {
+	//	if pInfo.Area == "全国" {
+	//		pInfo.Area = thisinfo.Area
+	//		set["area"] = thisinfo.Area
+	//	} else if pInfo.Area != thisinfo.Area {
+	//		//xt = false
+	//	}
+	//	if pInfo.City == "" && thisinfo.City != "" {
+	//		pInfo.City = thisinfo.City
+	//		set["city"] = thisinfo.City
+	//	} else if pInfo.City != thisinfo.City {
+	//		//xt = false
+	//	}
+	//	if thisinfo.District != "" && pInfo.District == "" {
+	//		pInfo.District = thisinfo.District
+	//		set["district"] = thisinfo.District
+	//	}
+	//}
+	//相同城市的公告才会合并到一起(全国列外)
 	if thisinfo.Area != "全国" {
-		if pInfo.Area == "全国" {
-			pInfo.Area = thisinfo.Area
-			set["area"] = thisinfo.Area
-		} else if pInfo.Area != thisinfo.Area {
-			//xt = false
-		}
-		if pInfo.City == "" && thisinfo.City != "" {
-			pInfo.City = thisinfo.City
-			set["city"] = thisinfo.City
-		} else if pInfo.City != thisinfo.City {
-			//xt = false
-		}
-		if thisinfo.District != "" && pInfo.District == "" {
+		pInfo.Area = thisinfo.Area
+		set["area"] = thisinfo.Area
+		pInfo.City = thisinfo.City
+		set["city"] = thisinfo.City
+		if thisinfo.District != "" {
 			pInfo.District = thisinfo.District
 			set["district"] = thisinfo.District
 		}
 	}
+	set["district"] = thisinfo.District
 	//6--项目名称
 	if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
 		pInfo.ProjectName = thisinfo.ProjectName

fullproject/src_v1/task.go

@@ -769,7 +769,7 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 		thisinfo.ReviewExperts = ClearRp(thisinfo.ReviewExperts)
 	}
 	//winners整理、清理
-	winner := WinnerAndBuyerFilter(tmp["winner"].(string), "winner")
+	winner := QyFilter(util.ObjToString(tmp["winner"]), "winner")
 	tmp["winner"] = winner
 	m1 := map[string]bool{}
 	winners := []string{}
@@ -782,7 +782,7 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 		thisinfo.HasPackage = true
 		for _, p := range packageM {
 			pm, _ := p.(map[string]interface{})
-			pw := WinnerAndBuyerFilter(pm["winner"].(string), "winner")
+			pw := QyFilter(util.ObjToString(pm["winner"]), "winner")
 			if pw != "" && !m1[pw] {
 				m1[pw] = true
 				winners = append(winners, pw)
@@ -793,7 +793,7 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	//清理winnerorder
 	var wins []map[string]interface{}
 	for _, v := range thisinfo.WinnerOrder {
-		w := WinnerAndBuyerFilter(util.ObjToString(v["entname"]), "winner")
+		w := QyFilter(util.ObjToString(v["entname"]), "winner")
 		if w != "" {
 			v["entname"] = w
 			wins = append(wins, v)
@@ -801,7 +801,7 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	}
 	thisinfo.WinnerOrder = wins
 	//清理buyer
-	buyer := WinnerAndBuyerFilter(util.ObjToString(tmp["tmp"]), "buyer")
+	buyer := QyFilter(util.ObjToString(tmp["tmp"]), "buyer")
 	tmp["buyer"] = buyer
 	thisinfo.Buyer = buyer
 
@@ -941,12 +941,15 @@ func (p *ProjectTask) fillInPlace(tmp map[string]interface{}) {
 				return
 			} else {
 				if site.City != "" {
-					tmp["area"] = site.City
+					tmp["area"] = site.Area
+					tmp["city"] = site.City
+					tmp["district"] = site.District
 				}
 			}
 		} else {
 			tmp["area"] = site.Area
 			tmp["city"] = site.City
+			tmp["district"] = site.District
 			return
 		}
 	}
@@ -985,7 +988,7 @@ func ClearRp(tmp []string) []string {
 	return arrTmp
 }
 
-func WinnerAndBuyerFilter(name, stype string) string {
+func QyFilter(name, stype string) string {
 	name = strings.ReplaceAll(name, " ", "")
 
 	preReg := PreRegexp[stype]

qyxy_change/qy_baidu/config.json

@@ -1,18 +1,22 @@
 {
-  "mongodbServer": "192.168.3.207:27092",
-  "mongodbPoolSize": 5,
-  "mongodbName": "mixdata",
-  "coll_baidu": "baidu_enterprise",
-  "coll_qy": "qyxy",
-  "coll_change": "qyxy_change_1",
-  "coll_back": "qyxy_change_back",
+  "baidu": {
+    "dbServer": "192.168.3.207:27092",
+    "dbName": "mixdata",
+    "dbSize": 5
+  },
   "pingan": {
     "dbServer": "192.168.3.207:27092",
     "dbName": "mixdata",
-    "dbColl": ""
+    "dbSize": 5
   },
-  "lasttime": 1600246800,
-  "tasktime": 1,
+  "coll_baidu": "baidu_enterprise",
+  "coll_pingan": "qyxy_tmp",
+  "coll_qy": "qyxy",
+  "coll_change": "qyxy_change",
+  "coll_back": "qyxy_change_back",
+  "lasttime": 0,
+  "baidu_tasktime": 2,
+  "pingan_tasktime": "WED",
   "changeType": [
     {
       "change_code": "100001",

qyxy_change/qy_baidu/main.go

@@ -7,40 +7,44 @@ import (
 
 var (
 	Sysconfig                                  map[string]interface{}
-	MongoTool                                  *mongodb.MongodbSim
-	MongoPaAdd                                 *mongodb.MongodbSim //凭安增量
-	Dbname                                     string
+	MgoBd                                 	   *mongodb.MongodbSim
+	MgoMix                                 	   *mongodb.MongodbSim		//凭安
+	Dbname_bd,	Dbname_pa                      string
 	CollBd, CollQy, CollSave, CollBack, CollPa string
 	LastTime                                   int64
-	TaskTime                                   int
+	BdTaskTime                                 int
+	PaTaskTime								   string
 	ChangeMap                                  []map[string]interface{}
 )
 
 func init() {
 	util.ReadConfig(&Sysconfig)
 
-	Dbname = Sysconfig["mongodbName"].(string)
-	MongoTool = &mongodb.MongodbSim{
-		MongodbAddr: Sysconfig["mongodbServer"].(string),
-		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
-		DbName:      Dbname,
+	baidu := util.ObjToMap(Sysconfig["baidu"])
+	Dbname_bd = util.ObjToString((*baidu)["dbName"])
+	MgoBd = &mongodb.MongodbSim{
+		MongodbAddr: util.ObjToString((*baidu)["dbServer"]),
+		Size:        util.IntAll((*baidu)["dbSize"]),
+		DbName:      Dbname_bd,
 	}
-	MongoTool.InitPool()
-	paDb := util.ObjToMap(Sysconfig["pingan"])
-	MongoPaAdd = &mongodb.MongodbSim{
-		MongodbAddr: (*paDb)["dbServer"].(string),
-		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
-		DbName:      (*paDb)["dbName"].(string),
+	MgoBd.InitPool()
+	pingan := util.ObjToMap(Sysconfig["pingan"])
+	Dbname_pa = util.ObjToString((*pingan)["dbName"])
+	MgoMix = &mongodb.MongodbSim{
+		MongodbAddr: util.ObjToString((*pingan)["dbServer"]),
+		Size:        util.IntAll((*pingan)["dbSize"]),
+		DbName:      Dbname_pa,
 	}
-	MongoPaAdd.InitPool()
+	MgoMix.InitPool()
 
-	CollPa = (*paDb)["dbColl"].(string)
+	CollPa = Sysconfig["coll_pingan"].(string)
 	CollBd = Sysconfig["coll_baidu"].(string)
 	CollQy = Sysconfig["coll_qy"].(string)
 	CollSave = Sysconfig["coll_change"].(string)
 	CollBack = Sysconfig["coll_back"].(string)
 	LastTime = util.Int64All(Sysconfig["lasttime"])
-	TaskTime = util.IntAll(Sysconfig["tasktime"])
+	BdTaskTime = util.IntAll(Sysconfig["baidu_tasktime"])
+	PaTaskTime = util.ObjToString(Sysconfig["pingan_tasktime"])
 
 	ChangeMap = util.ObjArrToMapArr(Sysconfig["changeType"].([]interface{}))
 	initChangeMap()
@@ -64,6 +68,9 @@ func initChangeMap() {
 
 func main() {
 	go SaveData()
-	//go TimeTask()
-	GetBdData()
+	go TimeTask()
+	//GetBdData()
+	//GetPaData()
+	ch := make(chan bool, 1)
+	<-ch
 }

qyxy_change/qy_baidu/task.go

@@ -14,24 +14,23 @@ import (
 
 func TimeTask() {
 	c := cron.New()
-	//cronstr := "0 */" + fmt.Sprint(TaskTime) + " * * * ?"
-	cronstr := "0 0 */" + fmt.Sprint(TaskTime) + " * * ?" //每TaskTime小时执行一次
-	_ = c.AddFunc(cronstr, func() { StartTask() })
+	cronstrBd := "0 0 */" + fmt.Sprint(BdTaskTime) + " * * ?" 		//每TaskTime小时执行一次
+	//cronstr := "0 0 " + fmt.Sprint(TaskTime) + " * * ?"			//每天TaskTime跑一次
+	cronstrPa := "0 0 15 ? * " + fmt.Sprint(PaTaskTime) 			//凭安增量数据每周三跑一次
+	util.Debug(cronstrPa)
+	_ = c.AddFunc(cronstrBd, func() { GetBdData() })
+	_ = c.AddFunc(cronstrPa, func() { GetPaData() })
 	c.Start()
 }
-func StartTask() {
-	GetBdData()		//百度数据
-	GetPaData()		//凭安数据
-}
 
 func GetBdData()  {
 	count := 0
 	lastid := ""
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
+	sess := MgoBd.GetMgoConn()
+	defer MgoBd.DestoryMongoConn(sess)
 	fields := map[string]interface{}{"data": 1, "down_time": 1}
 	q := bson.M{"down_time": bson.M{"$gte": LastTime}}
-	query := sess.DB(Dbname).C(CollBd).Find(q).Select(fields).Iter()
+	query := sess.DB(Dbname_bd).C(CollBd).Find(q).Select(fields).Iter()
 	tmp := make(map[string]interface{})
 	for query.Next(&tmp) {
 		lastid = mongodb.BsonIdToSId(tmp["_id"])
@@ -41,15 +40,16 @@ func GetBdData()  {
 		findEnt(tmp)
 		count++
 	}
+	util.Debug("baidu 处理", count, "条数据")
 }
 
 func GetPaData() {
 	count := 0
 	lastid := ""
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
+	sess := MgoMix.GetMgoConn()
+	defer MgoMix.DestoryMongoConn(sess)
 	fields := map[string]interface{}{"changes": 1, "company_id": 1, "company_name": 1}
-	query := sess.DB(Dbname).C(CollPa).Find(nil).Select(fields).Iter()
+	query := sess.DB(Dbname_pa).C(CollPa).Find(nil).Select(fields).Iter()
 	tmp := make(map[string]interface{})
 	for query.Next(&tmp) {
 		lastid = mongodb.BsonIdToSId(tmp["_id"])
@@ -59,9 +59,8 @@ func GetPaData() {
 		if tmp["changes"] != nil && len(tmp["changes"].([]interface{})) > 0 {
 			currentTime := time.Now().Unix()
 			q := bson.M{"company_name": tmp["company_name"]}
-			changeEnt, _ := MongoTool.FindOne(CollSave, q)
+			changeEnt, _ := MgoMix.FindOne(CollSave, q)
 			if changeEnt != nil && len(*changeEnt) > 0 {
-				util.Debug("凭安数据---更新企业信息-----ID----", tmp["_id"])
 				changeList := tmp["changes"].([]interface{})
 				if len(changeList) < len((*changeEnt)["changes"].([]interface{})) {
 					tmp["changes"] = (*changeEnt)["changes"].([]interface{})
@@ -73,7 +72,6 @@ func GetPaData() {
 					tmp["updatetime"] = currentTime
 				}
 			}else {
-				util.Debug("凭安数据---新增企业信息-----ID----", tmp["_id"])
 				infoList := tmp["changes"].([]interface{})
 				for _, item := range infoList {
 					item1 := item.(map[string]interface{})
@@ -95,6 +93,7 @@ func GetPaData() {
 			count++
 		}
 	}
+	util.Debug("pingan 处理", count, "条数据")
 }
 
 func findEnt(tmp map[string]interface{}) {
@@ -107,13 +106,11 @@ func findEnt(tmp map[string]interface{}) {
 	infoList := (*changeData)["list"].([]interface{})
 	currentTime := time.Now().Unix()
 	q := bson.M{"company_name": (*ent)["entName"]}
-	changeEnt, _ := MongoTool.FindOne(CollSave, q)
-	util.Debug(*changeEnt)
+	changeEnt, _ := MgoMix.FindOne(CollSave, q)
 	update := map[string]interface{}{}
 	if changeEnt != nil && len(*changeEnt) > 0 {
 		//1、企业变更库有该企业信息
 		if (*changeEnt)["changes"] != nil{
-			util.Debug("百度----更新企业信息-----ID----", (*changeEnt)["_id"])
 			(*changeEnt)["updatetime"] = currentTime
 			if len(infoList) > len(tmp["changes"].([]interface{})) {
 				mapArr := setChangeInfo(infoList)
@@ -133,11 +130,10 @@ func findEnt(tmp map[string]interface{}) {
 		}
 	} else {
 		//2、企业变更库没有该企业信息
-		paEnt, _ := MongoTool.FindOne(CollQy, q)
+		paEnt, _ := MgoMix.FindOne(CollQy, q)
 		saveEnt := map[string]interface{}{}
 		if saveEnt != nil && len(*paEnt) > 0 {
 			//3、企业库有该企业信息
-			util.Debug("百度----新增企业信息----ID-----", (*paEnt)["_id"])
 			saveEnt["_id"] = primitive.NewObjectID()
 			saveEnt["company_id"] = (*paEnt)["company_id"]
 			saveEnt["company_name"] = (*ent)["entName"]
@@ -167,7 +163,7 @@ func findEnt(tmp map[string]interface{}) {
 			saveEnt["company_name"] = (*ent)["entName"]
 			saveEnt["createtime"] = currentTime
 			saveEnt["changes"] = setChangeInfo(infoList)
-			MongoTool.Save(CollBack, saveEnt)
+			MgoMix.Save(CollBack, saveEnt)
 		}
 	}
 }
@@ -219,7 +215,7 @@ func SaveData() {
 					defer func() {
 						<-SP
 					}()
-					MongoTool.UpSertBulk(CollSave, arru...)
+					MgoMix.UpSertBulk(CollSave, arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0
@@ -231,7 +227,7 @@ func SaveData() {
 					defer func() {
 						<-SP
 					}()
-					MongoTool.UpSertBulk(CollSave, arru...)
+					MgoMix.UpSertBulk(CollSave, arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0