no message

forecast/es/config.json

@@ -8,8 +8,8 @@
   "tasktime": 0,
   "updateid": "",
   "elastic": {
-    "addr": "http://192.168.3.206:9800",
-    "index": "forecast_v1",
+    "addr": "http://127.0.0.1:9800",
+    "index": "forecast_v2",
     "itype": "forecast",
     "pool": 12,
     "esfields": [

forecast/es/task.go

@@ -13,7 +13,7 @@ import (
 
 //定时任务
 func TimeTask() {
-	go SaveAdd()
+	//go SaveAdd()
 	c := cron.New()
 	cronstr := "0 0 2 * * ?" //每天2点执行
 	//cronstr := "0 */" + fmt.Sprint(TaskTime) + " * * * ?" //每TaskTime小时执行一次
@@ -76,9 +76,14 @@ func SaveAdd() {
 				var mpArr []map[string]interface{}
 				for _, v := range mp {
 					v1 := v.(map[string]interface{})
+					tmp := make(map[string]interface{})
 					if v1["purchasing"] != nil {
-						mpArr = append(mpArr, map[string]interface{}{"purchasing": v1["purchasing"]})
+						tmp["purchasing"] = v1["purchasing"]
 					}
+					if v1["p_projects"] != nil {
+						tmp["p_projects"] = v1["p_projects"]
+					}
+					mpArr = append(mpArr, tmp)
 				}
 				if len(mpArr) > 0 {
 					esMap["results"] = mpArr
@@ -100,7 +105,7 @@ func SaveAll() {
 
 	pool := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
-	//q := bson.M{"_id": mongodb.StringTOBsonId("6194a3c105180be8dae19cbb")}
+	//q := bson.M{"_id": mongodb.StringTOBsonId("6227a7b18f0c45f21aabe120")}
 	it := sess.DB(Dbname).C(Dbcoll).Find(nil).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
@@ -135,9 +140,14 @@ func SaveAll() {
 				var mpArr []map[string]interface{}
 				for _, v := range mp {
 					v1 := v.(map[string]interface{})
+					tmp := make(map[string]interface{})
 					if v1["purchasing"] != nil {
-						mpArr = append(mpArr, map[string]interface{}{"purchasing": v1["purchasing"]})
+						tmp["purchasing"] = v1["purchasing"]
+					}
+					if v1["p_projects"] != nil {
+						tmp["p_projects"] = v1["p_projects"]
 					}
+					mpArr = append(mpArr, tmp)
 				}
 				if len(mpArr) > 0 {
 					esMap["results"] = mpArr

fullproject/src_v1/init.go

@@ -19,7 +19,7 @@ var (
 	MongoTool, MgoBidding, MgoSpider               *MongodbSim            //mongodb连接
 	ExtractColl, ProjectColl, BackupColl, SiteColl string                 //抽取表、项目表、项目快照表、站点表
 	ExtractColl1                                   string
-	Thread                                         int                    //配置项线程数
+	Thread                                         int //配置项线程数
 	BlackList                                      []interface{}
 	BlaskListMap                                   map[string]bool
 )
@@ -462,36 +462,36 @@ func CosineSimilar(srcWords1, dstWords1 string) float64 {
 
 func initWinnerRegexp() {
 	winRegMap := Sysconfig["winner"].(map[string]interface{})
-	preRegexps := winRegMap["pre_regexp"].([]interface{})
-	backRegexps := winRegMap["back_regexp"].([]interface{})
-	backRepRegexps := winRegMap["back_rep_regexp"].([]interface{})
+	//preRegexps := winRegMap["pre_regexp"].([]interface{})
+	//backRegexps := winRegMap["back_regexp"].([]interface{})
+	//backRepRegexps := winRegMap["back_rep_regexp"].([]interface{})
 	backBlack := winRegMap["blacklist"].([]interface{})
-	var winPreRegexps []*regexp.Regexp
-	for _, v := range preRegexps {
-		reg := regexp.MustCompile("^" + v.(string))
-		winPreRegexps = append(winPreRegexps, reg)
-	}
-	PreRegexp["winner"] = winPreRegexps
-
-	var winBackRegexps []*regexp.Regexp
-	for _, v := range backRegexps {
-		reg := regexp.MustCompile(v.(string))
-		winBackRegexps = append(winBackRegexps, reg)
-	}
-	BackRegexp["winner"] = winBackRegexps
-
-	var winBackRepRegexps []RegexpInfo
-	for _, v := range backRepRegexps {
-		reps := strings.Split(v.(string), "#")
-		if len(reps) > 1 {
-			reg := RegexpInfo{
-				regs:   regexp.MustCompile(reps[0]),
-				repstr: reps[1],
-			}
-			winBackRepRegexps = append(winBackRepRegexps, reg)
-		}
-	}
-	BackRepRegexp["winner"] = winBackRepRegexps
+	//var winPreRegexps []*regexp.Regexp
+	//for _, v := range preRegexps {
+	//	reg := regexp.MustCompile("^" + v.(string))
+	//	winPreRegexps = append(winPreRegexps, reg)
+	//}
+	//PreRegexp["winner"] = winPreRegexps
+
+	//var winBackRegexps []*regexp.Regexp
+	//for _, v := range backRegexps {
+	//	reg := regexp.MustCompile(v.(string))
+	//	winBackRegexps = append(winBackRegexps, reg)
+	//}
+	//BackRegexp["winner"] = winBackRegexps
+
+	//var winBackRepRegexps []RegexpInfo
+	//for _, v := range backRepRegexps {
+	//	reps := strings.Split(v.(string), "#")
+	//	if len(reps) > 1 {
+	//		reg := RegexpInfo{
+	//			regs:   regexp.MustCompile(reps[0]),
+	//			repstr: reps[1],
+	//		}
+	//		winBackRepRegexps = append(winBackRepRegexps, reg)
+	//	}
+	//}
+	//BackRepRegexp["winner"] = winBackRepRegexps
 
 	var winBlackRegexps []*regexp.Regexp
 	for _, v := range backBlack {
@@ -503,36 +503,36 @@ func initWinnerRegexp() {
 
 func initBuyerRegexp() {
 	buyRegMap := Sysconfig["buyer"].(map[string]interface{})
-	preRegexps := buyRegMap["pre_regexp"].([]interface{})
-	backRegexps := buyRegMap["back_regexp"].([]interface{})
-	backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
+	//preRegexps := buyRegMap["pre_regexp"].([]interface{})
+	//backRegexps := buyRegMap["back_regexp"].([]interface{})
+	//backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
 	backBlack := buyRegMap["blacklist"].([]interface{})
-	var winPreRegexps []*regexp.Regexp
-	for _, v := range preRegexps {
-		reg := regexp.MustCompile("^" + v.(string))
-		winPreRegexps = append(winPreRegexps, reg)
-	}
-	PreRegexp["buyer"] = winPreRegexps
-
-	var winBackRegexps []*regexp.Regexp
-	for _, v := range backRegexps {
-		reg := regexp.MustCompile(v.(string))
-		winBackRegexps = append(winBackRegexps, reg)
-	}
-	BackRegexp["buyer"] = winBackRegexps
-
-	var winBackRepRegexps []RegexpInfo
-	for _, v := range backRepRegexps {
-		reps := strings.Split(v.(string), "#")
-		if len(reps) > 1 {
-			reg := RegexpInfo{
-				regs:   regexp.MustCompile(reps[0]),
-				repstr: reps[1],
-			}
-			winBackRepRegexps = append(winBackRepRegexps, reg)
-		}
-	}
-	BackRepRegexp["buyer"] = winBackRepRegexps
+	//var winPreRegexps []*regexp.Regexp
+	//for _, v := range preRegexps {
+	//	reg := regexp.MustCompile("^" + v.(string))
+	//	winPreRegexps = append(winPreRegexps, reg)
+	//}
+	//PreRegexp["buyer"] = winPreRegexps
+
+	//var winBackRegexps []*regexp.Regexp
+	//for _, v := range backRegexps {
+	//	reg := regexp.MustCompile(v.(string))
+	//	winBackRegexps = append(winBackRegexps, reg)
+	//}
+	//BackRegexp["buyer"] = winBackRegexps
+
+	//var winBackRepRegexps []RegexpInfo
+	//for _, v := range backRepRegexps {
+	//	reps := strings.Split(v.(string), "#")
+	//	if len(reps) > 1 {
+	//		reg := RegexpInfo{
+	//			regs:   regexp.MustCompile(reps[0]),
+	//			repstr: reps[1],
+	//		}
+	//		winBackRepRegexps = append(winBackRepRegexps, reg)
+	//	}
+	//}
+	//BackRepRegexp["buyer"] = winBackRepRegexps
 
 	var winBlackRegexps []*regexp.Regexp
 	for _, v := range backBlack {
@@ -544,36 +544,36 @@ func initBuyerRegexp() {
 
 func initAgencyRegexp() {
 	buyRegMap := Sysconfig["agency"].(map[string]interface{})
-	preRegexps := buyRegMap["pre_regexp"].([]interface{})
-	backRegexps := buyRegMap["back_regexp"].([]interface{})
-	backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
+	//preRegexps := buyRegMap["pre_regexp"].([]interface{})
+	//backRegexps := buyRegMap["back_regexp"].([]interface{})
+	//backRepRegexps := buyRegMap["back_rep_regexp"].([]interface{})
 	backBlack := buyRegMap["blacklist"].([]interface{})
-	var winPreRegexps []*regexp.Regexp
-	for _, v := range preRegexps {
-		reg := regexp.MustCompile("^" + v.(string))
-		winPreRegexps = append(winPreRegexps, reg)
-	}
-	PreRegexp["agency"] = winPreRegexps
-
-	var winBackRegexps []*regexp.Regexp
-	for _, v := range backRegexps {
-		reg := regexp.MustCompile(v.(string))
-		winBackRegexps = append(winBackRegexps, reg)
-	}
-	BackRegexp["agency"] = winBackRegexps
-
-	var winBackRepRegexps []RegexpInfo
-	for _, v := range backRepRegexps {
-		reps := strings.Split(v.(string), "#")
-		if len(reps) > 1 {
-			reg := RegexpInfo{
-				regs:   regexp.MustCompile(reps[0]),
-				repstr: reps[1],
-			}
-			winBackRepRegexps = append(winBackRepRegexps, reg)
-		}
-	}
-	BackRepRegexp["agency"] = winBackRepRegexps
+	//var winPreRegexps []*regexp.Regexp
+	//for _, v := range preRegexps {
+	//	reg := regexp.MustCompile("^" + v.(string))
+	//	winPreRegexps = append(winPreRegexps, reg)
+	//}
+	//PreRegexp["agency"] = winPreRegexps
+
+	//var winBackRegexps []*regexp.Regexp
+	//for _, v := range backRegexps {
+	//	reg := regexp.MustCompile(v.(string))
+	//	winBackRegexps = append(winBackRegexps, reg)
+	//}
+	//BackRegexp["agency"] = winBackRegexps
+
+	//var winBackRepRegexps []RegexpInfo
+	//for _, v := range backRepRegexps {
+	//	reps := strings.Split(v.(string), "#")
+	//	if len(reps) > 1 {
+	//		reg := RegexpInfo{
+	//			regs:   regexp.MustCompile(reps[0]),
+	//			repstr: reps[1],
+	//		}
+	//		winBackRepRegexps = append(winBackRepRegexps, reg)
+	//	}
+	//}
+	//BackRepRegexp["agency"] = winBackRepRegexps
 
 	var winBlackRegexps []*regexp.Regexp
 	for _, v := range backBlack {

fullproject/src_v1/project.go

@@ -208,7 +208,6 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 									resArr[0].MPN = append(resArr[0].MPN, info.ProjectName)
 								}
 							}
-
 						} else if k2 < 3 {
 							if resArr[0].ProjectCode == "" {
 								resArr[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
@@ -832,13 +831,15 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 			set["district"] = thisinfo.District
 		}
 	}
-	//6--项目名称
-	if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+	//项目名称
+	//if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+	if pInfo.ProjectName == "" && thisinfo.ProjectName != "" {
 		pInfo.ProjectName = thisinfo.ProjectName
 		set["projectname"] = thisinfo.ProjectName
 	}
 	//7--项目编号
-	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+	//if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+	if pInfo.ProjectCode == "" && thisinfo.ProjectCode != "" {
 		pInfo.ProjectCode = thisinfo.ProjectCode
 		set["projectcode"] = thisinfo.ProjectCode
 	}

fullproject/src_v1/project_tool.go

@@ -1,5 +1,7 @@
 package main
 
+var DateTimeSelect = []string{"bidopentime", "signaturedate", "project_completedate", "comeintime"}
+
 //项目中的字段
 var FIELDS = []string{
 	"area",
@@ -93,4 +95,3 @@ var bidtype = map[string]string{
 	"竞价": "竞价",
 	"竞谈": "竞谈",
 }
-

fullproject/src_v1/task.go

@@ -26,9 +26,9 @@ import (
 转换成info对象
 **/
 
-var PreRegexp = map[string][]*regexp.Regexp{}
-var BackRegexp = map[string][]*regexp.Regexp{}
-var BackRepRegexp = map[string][]RegexpInfo{}
+//var PreRegexp = map[string][]*regexp.Regexp{}
+//var BackRegexp = map[string][]*regexp.Regexp{}
+//var BackRepRegexp = map[string][]RegexpInfo{}
 var BlackRegexp = map[string][]*regexp.Regexp{}
 
 var (
@@ -529,12 +529,14 @@ L:
 					}
 				}
 				if util.IntAll(tmp["repeat"]) == 0 {
-					if P_QL.currentType == "ql" {
-						infoPool <- tmp
-					} else if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
-						infoPool <- tmp
-					} else {
-						util.Debug("增量   dataging == 1 ", tmp["_id"])
+					if util.ObjToString(tmp["toptype"]) != "采购意向" {
+						if P_QL.currentType == "ql" {
+							infoPool <- tmp
+						} else if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
+							infoPool <- tmp
+						} else {
+							util.Debug("增量   dataging == 1 ", tmp["_id"])
+						}
 					}
 				} else {
 					countRepeat++
@@ -602,6 +604,16 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 		return nil
 	}
 
+	// 处理publishtime为空
+	if thisinfo.Publishtime <= 0 {
+		for _, d := range DateTimeSelect {
+			if tmp[d] != nil {
+				thisinfo.Publishtime = util.Int64All(tmp[d])
+				tmp["publishtime"] = tmp[d]
+			}
+		}
+	}
+
 	if len(thisinfo.Topscopeclass) == 0 {
 		thisinfo.Topscopeclass = []string{}
 	}
@@ -647,9 +659,7 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 				thisinfo.PTC = StrOrNum.ReplaceAllString(thisinfo.PTC, "")
 			}
 		}
-		if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
-			thisinfo.pnbval++
-		}
+		thisinfo.pnbval++
 	}
 	if thisinfo.ProjectCode == thisinfo.PTC || strings.Index(thisinfo.ProjectCode, thisinfo.PTC) > -1 {
 		thisinfo.PTC = ""
@@ -904,18 +914,18 @@ func ClearRp(tmp []string) []string {
 func QyFilter(name, stype string) string {
 	name = strings.ReplaceAll(name, " ", "")
 
-	preReg := PreRegexp[stype]
-	for _, v := range preReg {
-		name = v.ReplaceAllString(name, "")
-	}
-	backReg := BackRegexp[stype]
-	for _, v := range backReg {
-		name = v.ReplaceAllString(name, "")
-	}
-	backRepReg := BackRepRegexp[stype]
-	for _, v := range backRepReg {
-		name = v.regs.ReplaceAllString(name, v.repstr)
-	}
+	//preReg := PreRegexp[stype]
+	//for _, v := range preReg {
+	//	name = v.ReplaceAllString(name, "")
+	//}
+	//backReg := BackRegexp[stype]
+	//for _, v := range backReg {
+	//	name = v.ReplaceAllString(name, "")
+	//}
+	//backRepReg := BackRepRegexp[stype]
+	//for _, v := range backRepReg {
+	//	name = v.regs.ReplaceAllString(name, v.repstr)
+	//}
 
 	blackReg := BlackRegexp[stype]
 	for _, v := range blackReg {

fullproject/src_v1/update.go

@@ -16,8 +16,8 @@ import (
 )
 
 /**
-	直接修改保存项目字段信息
- */
+直接修改保存项目字段信息
+*/
 func (p *ProjectTask) modifyUpdate(pid string, index, position int, tmpPro, modifyProMap map[string]interface{}) {
 	// 1-0 备份
 	tmpPro["reason"] = "直接在原有项目上修改字段信息"
@@ -69,7 +69,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 			// 修改内存
 			p.modifyMem(tmpPro)
 		}
-	}else if index == 0 {
+	} else if index == 0 {
 		// 招标公告信息在原有项目中位于第1个
 		merge := p.ReMerge(info, tmp, tmpPro)
 		if merge {
@@ -83,8 +83,8 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 			Id := p.AllIdsMap[pInfoId]
 			p.AllIdsMapLock.Unlock()
 			proMem := Id.P
-			flag := true		// 标记是否需要删除原有项目信息
-			for _, v := range proList{
+			flag := true // 标记是否需要删除原有项目信息
+			for _, v := range proList {
 				v1 := v.(map[string]interface{})
 				temp := MongoTool.FindById(ExtractColl, qu.ObjToString(v1["infoid"]))
 				if len(temp) == 0 {
@@ -105,7 +105,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 			}
 			if flag {
 				delOldPro(pInfoId)
-			}else {
+			} else {
 				//合并到原有项目中， 不用继续重新合并， 原有项目内部合并即可
 				p.innerMerge(proList, tmpPro)
 				bol := MongoTool.UpdateById(ProjectColl, pInfoId, map[string]interface{}{"$set": tmpPro})
@@ -128,13 +128,13 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 				}
 				p.AllIdsMapLock.Unlock()
 			}
-		}else {
+		} else {
 			//没有合并到新项目中，更新list字段，有条件更新项目外围字段
 			p.modifyUpdate(pInfoId, index, position, tmpPro, modifyMap)
 			// 修改内存
 			p.modifyMem(tmpPro)
 		}
-	}else if index == 1 {
+	} else if index == 1 {
 		// 招标公告信息在原有项目中处于中间某一条
 		merge := p.ReMerge(info, tmp, tmpPro)
 		if merge {
@@ -155,13 +155,13 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 				qu.Debug(string(by))
 				_ = udpclient.WriteUdp(by, mu.OP_TYPE_DATA, toaddr[1])
 			}
-		}else {
+		} else {
 			// 未合并到新项目中
 			p.modifyUpdate(pInfoId, index, position, tmpPro, modifyMap)
 			// 修改内存
 			p.modifyMem(tmpPro)
 		}
-	}else if index == 2 {
+	} else if index == 2 {
 		// 招标公告信息在原有项目中位于最后一条
 		merge := p.ReMerge(info, tmp, tmpPro)
 		if merge {
@@ -184,7 +184,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 				qu.Debug(string(by))
 				_ = udpclient.WriteUdp(by, mu.OP_TYPE_DATA, toaddr[1])
 			}
-		}else {
+		} else {
 			// 未合并到新项目中
 			p.modifyUpdate(pInfoId, index, position, tmpPro, modifyMap)
 			// 修改内存
@@ -209,14 +209,14 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 		}
 		return
 	}
-	var index = -1		//0：第一个，1：中间，2：最后一个
+	var index = -1 //0：第一个，1：中间，2：最后一个
 	for i, v := range ids {
 		if qu.ObjToString(v) == infoid {
 			if i == 0 {
 				index = 0
-			}else if i == len(ids) - 1 {
+			} else if i == len(ids)-1 {
 				index = 2
-			}else {
+			} else {
 				index = 1
 			}
 		}
@@ -227,8 +227,8 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 		delete(tmpPro, "reason")
 		proList = deleteSlice1(proList, proList[0])
 		var pro *ProjectInfo
-		flag := true		// 标记是否需要删除原有项目信息
-		for _, v := range proList{
+		flag := true // 标记是否需要删除原有项目信息
+		for _, v := range proList {
 			v1 := v.(map[string]interface{})
 			temp := MongoTool.FindById(ExtractColl, qu.ObjToString(v1["infoid"]))
 			if len(temp) == 0 {
@@ -249,7 +249,7 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 		}
 		if flag {
 			delOldPro(pid)
-		}else {
+		} else {
 			tmpPro, pro = p.innerMerge(proList, tmpPro)
 			bol := MongoTool.UpdateById(ProjectColl, pid, map[string]interface{}{"$set": tmpPro})
 			if bol {
@@ -271,7 +271,7 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 			}
 			p.AllIdsMapLock.Unlock()
 		}
-	}else if index == 1 {
+	} else if index == 1 {
 		tmpPro["reason"] = "删除list中间某一条招标公告信息"
 		backupPro(tmpPro)
 		delete(tmpPro, "reason")
@@ -296,7 +296,7 @@ func (p *ProjectTask) delJudge(infoid, pid string) {
 			v.P = pro
 		}
 		p.AllIdsMapLock.Unlock()
-	}else if index == 2 {
+	} else if index == 2 {
 		tmpPro["reason"] = "删除list中最后一条招标公告信息"
 		backupPro(tmpPro)
 		delete(tmpPro, "reason")
@@ -457,7 +457,7 @@ func (p *ProjectTask) innerMerge(infoList []interface{}, tmpPro map[string]inter
 	newP := make(map[string]interface{})
 	newP["_id"] = tmpPro["_id"]
 	var p1 *ProjectInfo
-	for k, m := range infoList{
+	for k, m := range infoList {
 		m1 := m.(map[string]interface{})
 		temp := MongoTool.FindById(ExtractColl, qu.ObjToString(m1["infoid"]))
 		if len(temp) == 0 {
@@ -470,7 +470,7 @@ func (p *ProjectTask) innerMerge(infoList []interface{}, tmpPro map[string]inter
 		tempInfo := ParseInfo(temp)
 		if k == 0 {
 			p1 = p.newPro(temp, newP, tempInfo)
-		}else {
+		} else {
 			p.updateOldProField(p1, tempInfo, newP, temp, m1)
 		}
 	}
@@ -481,7 +481,7 @@ func (p *ProjectTask) innerMerge1(infoList []interface{}, infoid string, tmpPro
 	newP := make(map[string]interface{})
 	newP["_id"] = tmpPro["_id"]
 	var p1 *ProjectInfo
-	for k, m := range infoList{
+	for k, m := range infoList {
 		m1 := m.(map[string]interface{})
 		if m1["infoid"] == infoid {
 			continue
@@ -497,7 +497,7 @@ func (p *ProjectTask) innerMerge1(infoList []interface{}, infoid string, tmpPro
 		tempInfo := ParseInfo(temp)
 		if k == 0 {
 			p1 = p.newPro(temp, newP, tempInfo)
-		}else {
+		} else {
 			p.updateOldProField(p1, tempInfo, newP, temp, m1)
 		}
 	}
@@ -710,10 +710,12 @@ func (p *ProjectTask) updateProFiled(tmp map[string]interface{}, thisinfo *Info,
 		set["bidamount"] = pInfo.Bidamount
 		set["bidamounttag"] = 0
 	}
-	if pInfo.Bidamount >= pInfo.Budget {
+	if pInfo.Bidamount > 0 {
 		set["sortprice"] = pInfo.Bidamount
-	} else if pInfo.Budget >= pInfo.Bidamount {
-		set["sortprice"] = pInfo.Budget
+	} else {
+		if pInfo.Budget > 0 {
+			set["sortprice"] = pInfo.Budget
+		}
 	}
 
 	infofield := InfoField{
@@ -889,6 +891,7 @@ func (p *ProjectTask) newPro(tmp, tmpPro map[string]interface{}, thisinfo *Info)
 	}
 	return &p1
 }
+
 // 合并字段到老项目中
 func (p *ProjectTask) updateOldProField(pInfo *ProjectInfo, thisinfo *Info, tmpPro, tmp, m1 map[string]interface{}) {
 	if len(pInfo.Ids) > 30 {
@@ -1150,7 +1153,7 @@ func UpdateValue(proMap map[string]interface{}, index, position int, modifyMap m
 				updataSet[k] = modifyMap[k]
 				proMap[k] = modifyMap[k]
 				tempMap[k] = modifyMap[k]
-			}else {
+			} else {
 				updataSet[k] = modifyMap[k]
 				proMap[k] = modifyMap[k]
 				tempMap[k] = modifyMap[k]
@@ -1178,7 +1181,7 @@ func UpdateValue(proMap map[string]interface{}, index, position int, modifyMap m
 //备份(快照)
 func backupPro(tmp map[string]interface{}) {
 	tmp1 := make(map[string]interface{})
-	for k, v := range tmp{
+	for k, v := range tmp {
 		tmp1[k] = v
 	}
 	if Sysconfig["backupFlag"].(bool) {
@@ -1191,7 +1194,7 @@ func backupPro(tmp map[string]interface{}) {
 // 删除原有项目数据
 func delOldPro(pid string) {
 	t := MongoTool.Delete(ProjectColl, pid)
-	if t >=0 {
+	if t >= 0 {
 		client := Es.GetEsConn()
 		defer Es.DestoryEsConn(client)
 		Es.DelById(Index, Itype, pid)
@@ -1199,7 +1202,7 @@ func delOldPro(pid string) {
 }
 
 // 修改内存中的数据
-func (p *ProjectTask) modifyMem(tmpPro map[string]interface{})  {
+func (p *ProjectTask) modifyMem(tmpPro map[string]interface{}) {
 	pid := qu.ObjToString(tmpPro["_id"])
 	var pro ProjectInfo
 	err := mapstructure.Decode(tmpPro, &pro)
@@ -1231,4 +1234,4 @@ func (p *ProjectTask) printMemPro(pid string) {
 		qu.Debug("mem pro ----------", *v.P)
 	}
 	p.AllIdsMapLock.Unlock()
-}
+}

qyxy/qyxy_change/main.go

@@ -62,7 +62,7 @@ func initChangeMap() {
 
 func main() {
 	//go SaveData()
-	go updateMethod()
+	//go updateMethod()
 
 	//go TimeTask()
 	go GetData()

qyxy/qyxy_change/task.go

@@ -64,11 +64,9 @@ func GetData() {
 					setMark(item) //change_name_new
 					//update["changes"] = changes
 					update["update_time"] = currentTime
-					saveInfo := []map[string]interface{}{
-						{"company_id": tmp["company_id"]},
-						{"$set": update, "$push": map[string]interface{}{"changes": item}},
-					}
-					updatePool <- saveInfo
+					saveInfo := map[string]interface{}{"$set": update, "$push": map[string]interface{}{"changes": item}}
+					//updatePool <- saveInfo
+					MgoMix.UpdateById(CollSave, tmp["company_id"], saveInfo)
 				}
 			} else {
 				query := bson.M{"_id": tmp["company_id"]}
@@ -90,11 +88,9 @@ func GetData() {
 					save["create_time"] = currentTime
 					save["update_time"] = currentTime
 					//save["_id"] = primitive.NewObjectID()
-					saveInfo := []map[string]interface{}{
-						{"company_id": tmp["company_id"]},
-						{"$set": save},
-					}
-					updatePool <- saveInfo
+					saveInfo := map[string]interface{}{"$set": save}
+					//updatePool <- saveInfo
+					MgoMix.Save(CollSave, saveInfo)
 				}
 			}
 		}(tmp)

udpcreateindex/src/biddingall.go

@@ -5,7 +5,6 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"mongodb"
 	"qfw/util/elastic"
-	"qfw/util/redis"
 	"regexp"
 	"time"
 	//"fmt"
@@ -15,7 +14,6 @@ import (
 	"reflect"
 	"strings"
 	"sync"
-
 )
 
 //对字段处理 bidamount  budget
@@ -77,7 +75,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 	query := session.DB(db).C(c).Find(q).Select(bson.M{
 		"projectinfo.attachment": 0,
 		"contenthtml":            0,
-		"publishdept":			  0,		// 6.30		迭代报错，字段值乱码
+		"publishdept":            0, // 6.30		迭代报错，字段值乱码
 	}).Sort("_id").Iter()
 	//查询抽取结果
 	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
@@ -230,7 +228,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				//					}
 				//					update["s_winner"] = strings.Join(winnerarr, ",")
 				//				}
-			}else {
+			} else {
 				area := qutil.ObjToString(tmp["area"])
 				city := qutil.ObjToString(tmp["city"])
 				district := qutil.ObjToString(tmp["district"])
@@ -249,28 +247,28 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				tmp[tk] = tv
 			}
 			if tmp["s_winner"] != "" {
-				sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
+				//sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
 				var cid []string
-				for _, w := range sWinnerarr {
-					if w != "" {
-						id := redis.GetStr("qyxy_id", w)
-						if id == "" {
-							ents, _ := mgostandard.Find("qyxy_std", map[string]interface{}{"company_name": w}, map[string]interface{}{"updatetime": -1}, nil, false, -1, -1)
-							if len(*ents) > 0 {
-								id = qutil.ObjToString((*ents)[0]["_id"])
-							}else {
-								ent, _ := qyxydb.FindOne("company_history_name", map[string]interface{}{"history_name": w})
-								if len(*ent) > 0 {
-									id = qutil.ObjToString((*ent)["company_id"])
-								}
-							}
-						}
-						if id == "" {
-							id = "-"
-						}
-						cid = append(cid, id)
-					}
-				}
+				//for _, w := range sWinnerarr {
+				//	if w != "" {
+				//		id := redis.GetStr("qyxy_id", w)
+				//		if id == "" {
+				//			ents, _ := mgostandard.Find("qyxy_std", map[string]interface{}{"company_name": w}, map[string]interface{}{"updatetime": -1}, nil, false, -1, -1)
+				//			if len(*ents) > 0 {
+				//				id = qutil.ObjToString((*ents)[0]["_id"])
+				//			}else {
+				//				ent, _ := qyxydb.FindOne("company_history_name", map[string]interface{}{"history_name": w})
+				//				if len(*ent) > 0 {
+				//					id = qutil.ObjToString((*ent)["company_id"])
+				//				}
+				//			}
+				//		}
+				//		if id == "" {
+				//			id = "-"
+				//		}
+				//		cid = append(cid, id)
+				//	}
+				//}
 				if len(cid) > 0 {
 					tmp["entidlist"] = cid
 					update["entidlist"] = cid
@@ -394,7 +392,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							}
 						} else if field == "review_experts" {
 							// 评审专家
-							if arr, ok :=tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
+							if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
 								arr1 := qutil.ObjArrToStringArr(arr)
 								newTmp[field] = strings.Join(arr1, ",")
 							}
@@ -404,10 +402,10 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							if tmp[field] != nil && tmp["bidendtime"] == nil {
 								newTmp["bidendtime"] = tmp[field]
 								newTmp[field] = tmp["bidendtime"]
-							}else if tmp[field] == nil && tmp["bidendtime"] != nil {
+							} else if tmp[field] == nil && tmp["bidendtime"] != nil {
 								newTmp["bidendtime"] = tmp[field]
 								newTmp[field] = tmp["bidendtime"]
-							}else {
+							} else {
 								if tmp["bidopentime"] != nil {
 									newTmp[field] = tmp["bidopentime"]
 								}
@@ -540,21 +538,21 @@ func UpdateExtract() {
 }
 
 //城市标准校验
-func standardCheckCity(area string,city string,district string) map[string]string{
+func standardCheckCity(area string, city string, district string) map[string]string {
 	rdata := make(map[string]string)
-	if area=="香港"||area=="澳门"||area=="台湾" || (area=="全国"&&(city==""&&district=="")) {
+	if area == "香港" || area == "澳门" || area == "台湾" || (area == "全国" && (city == "" && district == "")) {
 		return rdata
 	}
 	//第一步：区校验
-	if district!="" {
+	if district != "" {
 		districtArr := DistrictDict[district]
-		if districtArr==nil {//涉及了 个别别名相关的数据
-			trim_arr := aliasDataDistrict(district)//拆分后缀
-			if len(trim_arr)>0 {
-				for _,alias_district := range trim_arr {
+		if districtArr == nil { //涉及了 个别别名相关的数据
+			trim_arr := aliasDataDistrict(district) //拆分后缀
+			if len(trim_arr) > 0 {
+				for _, alias_district := range trim_arr {
 					alias_districtArr := DistrictDict[alias_district]
-					for _,v:=range alias_districtArr{
-						if  city == v.C_Name && area == v.P_Name {
+					for _, v := range alias_districtArr {
+						if city == v.C_Name && area == v.P_Name {
 							rdata["district"] = alias_district
 							return rdata
 						}
@@ -562,23 +560,23 @@ func standardCheckCity(area string,city string,district string) map[string]strin
 				}
 			}
 			rdata["district"] = ""
-		}else {
+		} else {
 			isTrue := false
-			for _,v:=range districtArr{
-				if  city == v.C_Name && area == v.P_Name {
+			for _, v := range districtArr {
+				if city == v.C_Name && area == v.P_Name {
 					isTrue = true
 					break
 				}
 			}
 			if isTrue { //完全匹配
 				return rdata
-			}else { //未完全匹配
-				if len(districtArr)==1 {
+			} else { //未完全匹配
+				if len(districtArr) == 1 {
 					rdata["area"] = districtArr[0].P_Name
 					rdata["city"] = districtArr[0].C_Name
 					rdata["district"] = districtArr[0].D_Name
 					return rdata
-				}else {
+				} else {
 					rdata["district"] = ""
 				}
 			}
@@ -588,11 +586,11 @@ func standardCheckCity(area string,city string,district string) map[string]strin
 	//第二步：区校验-失败   市-校验
 	if city != "" {
 		cityArr := CityDict[city]
-		if cityArr==nil {
+		if cityArr == nil {
 			//把市当成区，匹配三级   - 存在优化空间- city:郑州  别名
 			districtArr := DistrictDict[city]
-			for _,v:=range districtArr{
-				if  city == v.C_Name && area == v.P_Name {
+			for _, v := range districtArr {
+				if city == v.C_Name && area == v.P_Name {
 					rdata["area"] = districtArr[0].P_Name
 					rdata["city"] = districtArr[0].C_Name
 					rdata["district"] = districtArr[0].D_Name
@@ -600,23 +598,23 @@ func standardCheckCity(area string,city string,district string) map[string]strin
 				}
 			}
 			rdata["city"] = ""
-		}else {
+		} else {
 			isTrue := false
-			for _,v:=range cityArr{
-				if  area == v.P_Name {
+			for _, v := range cityArr {
+				if area == v.P_Name {
 					isTrue = true
 					break
 				}
 			}
 			if isTrue { //完全匹配
 				return rdata
-			}else { //未完全匹配
-				if len(cityArr)==1 {
+			} else { //未完全匹配
+				if len(cityArr) == 1 {
 					rdata["area"] = cityArr[0].P_Name
 					rdata["city"] = cityArr[0].C_Name
 					rdata["district"] = ""
 					return rdata
-				}else {
+				} else {
 					rdata["city"] = ""
 				}
 			}
@@ -624,7 +622,7 @@ func standardCheckCity(area string,city string,district string) map[string]strin
 	}
 
 	//第三步：省份校验
-	if ProvinceDict[area]==nil {
+	if ProvinceDict[area] == nil {
 		rdata["area"] = "全国"
 		rdata["city"] = ""
 		rdata["district"] = ""
@@ -634,29 +632,29 @@ func standardCheckCity(area string,city string,district string) map[string]strin
 }
 
 var cityEndReg = regexp.MustCompile("(区|县|市)$")
+
 //拆分三级县
 func aliasDataDistrict(district string) []string {
-	arr :=[]string{}
+	arr := []string{}
 	if cityEndReg.MatchString(district) {
 		str := cityEndReg.FindString(district)
 		strings.TrimRight(district, str)
-		if str=="县"{
-			arr = append(arr,fmt.Sprintf("%s区",strings.TrimRight(district, str)))
-			arr = append(arr,fmt.Sprintf("%s市",strings.TrimRight(district, str)))
-		}else if str=="区"{
-			arr = append(arr,fmt.Sprintf("%s县",strings.TrimRight(district, str)))
-			arr = append(arr,fmt.Sprintf("%s市",strings.TrimRight(district, str)))
-		} else if str=="市"{
-			arr = append(arr,fmt.Sprintf("%s县",strings.TrimRight(district, str)))
-			arr = append(arr,fmt.Sprintf("%s区",strings.TrimRight(district, str)))
-		}else {
+		if str == "县" {
+			arr = append(arr, fmt.Sprintf("%s区", strings.TrimRight(district, str)))
+			arr = append(arr, fmt.Sprintf("%s市", strings.TrimRight(district, str)))
+		} else if str == "区" {
+			arr = append(arr, fmt.Sprintf("%s县", strings.TrimRight(district, str)))
+			arr = append(arr, fmt.Sprintf("%s市", strings.TrimRight(district, str)))
+		} else if str == "市" {
+			arr = append(arr, fmt.Sprintf("%s县", strings.TrimRight(district, str)))
+			arr = append(arr, fmt.Sprintf("%s区", strings.TrimRight(district, str)))
+		} else {
 
 		}
-	}else { //未找到 district- 区县市  例： district : 金水
-		arr = append(arr,fmt.Sprintf("%s区",district))
-		arr = append(arr,fmt.Sprintf("%s县",district))
-		arr = append(arr,fmt.Sprintf("%s市",district))
+	} else { //未找到 district- 区县市  例： district : 金水
+		arr = append(arr, fmt.Sprintf("%s区", district))
+		arr = append(arr, fmt.Sprintf("%s县", district))
+		arr = append(arr, fmt.Sprintf("%s市", district))
 	}
 	return arr
 }
-

udpcreateindex/src/biddingindex.go

@@ -23,7 +23,7 @@ var date1 = regexp.MustCompile("20[0-2][0-9][年|\\-\\/|.][0-9]{1,2}[月|\\-|\\/
 //对字段处理 bidamount  budget
 //招标数据表和抽取表一一对应开始更新
 
-func biddingTask(data []byte, mapInfo map[string]interface{}) {
+func biddingTask(data []byte, mapInfo map[string]interface{}, tasktype string) {
 	defer qutil.Catch()
 	q, _ := mapInfo["query"].(map[string]interface{})
 	bkey, _ := mapInfo["bkey"].(string)
@@ -87,7 +87,7 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 			}
 			udpclient.WriteUdp(data, mu.OP_TYPE_DATA, toadd)
 		}
-		n1, n2 = doIndex(res, eMap, index, itype, db, c, bkey)
+		n1, n2 = doIndex(res, eMap, index, itype, db, c, bkey, tasktype)
 		//if int64(n1 + n2) != count {
 		//	log.Println("任务错误，结果不一致")
 		//}
@@ -95,52 +95,12 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 		log.Println("数据量太大，放弃！", count)
 		mgo.DestoryMongoConn(session)
 	}
-	log.Println(mapInfo, "create bidding index...over", "all:", count, "n1:", n1, "n2:", n2)
+	log.Println(mapInfo, "create bidding index...over", "all:", count, "bidding size:", n1, "，es size:", n2)
 
-	//go delEs(mapInfo, index, itype, db, c) //删除索引
 }
 
-//删除索引
-// func delEs(mapInfo map[string]interface{}, index, itype, db, c string) {
-// 	defer qutil.Catch()
-// 	other_delete := false
-// 	if other_index != "" && other_itype != "" {
-// 		other_delete = true
-// 	}
-// 	ids := qutil.ObjToString(mapInfo["ids"])
-// 	idsarr := strings.Split(ids, ",")
-// 	log.Println("delete ids count:", len(idsarr))
-// 	n1 := 0
-// 	update := [][]map[string]interface{}{} //将bidding表中的extracttype改为-1
-// 	set := map[string]interface{}{
-// 		"$set": map[string]interface{}{"extracttype": -1},
-// 	}
-// 	for _, id := range idsarr {
-// 		if id != "" {
-// 			update = append(update, []map[string]interface{}{ //更新
-// 				map[string]interface{}{
-// 					"_id": mongodb.StringTOBsonId(id),
-// 				},
-// 				set,
-// 			})
-// 			if elastic.DelById(index, itype, id) {
-// 				n1++
-// 			}
-// 			if other_delete {
-// 				bidding_other_es.DelById(other_index, other_itype, id)
-// 			}
-// 		}
-// 	}
-// 	//更新
-// 	if len(update) > 0 {
-// 		mgo.UpdateBulkAll(db, c, update...)
-// 	}
-
-// 	log.Println("result delete bidding index...over", "all:", n1)
-// }
-
-func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, index, itype, db, c, bkey string) (int, int) {
-	n1, n2 := 0, 0
+func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, index, itype, db, c, bkey, tasktype string) (int, int) {
+	n1, n2 := 0, 0 //bidding数量，索引数量
 	//线程池
 	UpdatesLock := sync.Mutex{}
 	fields := strings.Split(bidding["fields"].(string), ",")
@@ -156,18 +116,30 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 			tmp = make(map[string]interface{})
 			continue
 		}
+		tid := mongodb.BsonIdToSId(tmp["_id"])
+		//loginfo := make(map[string]interface{}) // 日志
 		update := map[string]interface{}{} //要更新的mongo数据
 		//对比方法----------------
-		tid := mongodb.BsonIdToSId(tmp["_id"])
-
 		if eMap[tid] != nil {
 			compare = eMap[tid]
-			if qutil.IntAll(compare["dataging"]) == 1 { //extract中dataging=1不生索引
-				tmp = make(map[string]interface{})
-				compare = nil
-				continue
+			if tasktype == "bidding" {
+				// 增量id段 正常数据
+				if num := qutil.IntAll(compare["dataging"]); num == 1 { //extract中dataging=1跳过
+					tmp = make(map[string]interface{})
+					compare = nil
+					continue
+				}
+				delete(eMap, tid)
+			}
+			if tasktype == "bidding_history" {
+				//增量id段 历史数据
+				if compare["history_updatetime"] == nil { //extract中history_updatetime不存在跳过
+					tmp = make(map[string]interface{})
+					compare = nil
+					continue
+				}
+				delete(eMap, tid)
 			}
-			delete(eMap, tid)
 			//更新bidding表，生成索引;bidding表modifyinfo中的字段不更新
 			modifyinfo := make(map[string]bool)
 			if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
@@ -190,7 +162,6 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 						update[k] = ""
 					}
 				}
-
 			}
 			if qutil.IntAll(compare["repeat"]) == 1 {
 				update["extracttype"] = -1
@@ -236,33 +207,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 				}
 				update["s_topscopeclass"] = strings.Join(newclass, ",")
 			}
-			//处理中标企业
-			//			winner, _ := compare["winner"].(string)
-			//			m1 := map[string]bool{}
-			//			if winner != "" {
-			//				m1[winner] = true
-			//			}
-			//			package1 := compare["package"]
-			//			if package1 != nil {
-			//				packageM, _ := package1.(map[string]interface{})
-			//				for _, p := range packageM {
-			//					pm, _ := p.(map[string]interface{})
-			//					pw, _ := pm["winner"].(string)
-			//					if pw != "" {
-			//						m1[pw] = true
-			//					}
-			//				}
-			//			}
 			compare = nil
-			//			if len(m1) > 0 {
-			//				//str := ","
-			//				winnerarr := []string{}
-			//				for k, _ := range m1 {
-			//					//str += k + ","
-			//					winnerarr = append(winnerarr, k)
-			//				}
-			//				update["s_winner"] = strings.Join(winnerarr, ",")
-			//			}
 		} else {
 			area := qutil.ObjToString(tmp["area"])
 			city := qutil.ObjToString(tmp["city"])
@@ -361,6 +306,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		//		}
 		go IS.Add("bidding")
 		if qutil.IntAll(update["extracttype"]) != -1 {
+			n2++
 			newTmp := map[string]interface{}{}                //最终生索引的数据
 			for field, ftype := range biddingIndexFieldsMap { //
 				if tmp[field] != nil { //
@@ -477,17 +423,15 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 					}
 				}
 			}
-
-			YuceEndtime(newTmp) // 预测结果时间
+			YuceEndtime(newTmp)                      // 预测结果时间
+			newTmp["createtime"] = time.Now().Unix() // es库数据创建时间，只有增量数据有
 			arrEs = append(arrEs, newTmp)
 		}
 		if len(update) > 0 {
 			delete(update, "winnerorder") //winnerorder不需要更新到bindding表，删除
 			arr = append(arr, []map[string]interface{}{
-				{
-					"_id": tmp["_id"]},
-				{
-					"$set": update},
+				{"_id": tmp["_id"]},
+				{"$set": update},
 			})
 		}
 		if len(arr) >= BulkSize-1 {

udpcreateindex/src/biddingindexback.go

@@ -51,7 +51,7 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 	query := session.DB(db).C(c).Find(q).Select(bson.M{
 		"projectinfo.attachment": 0,
 		"contenthtml":            0,
-		"publishdept":			  0,
+		"publishdept":            0,
 	}).Sort("_id").Iter()
 	//查询抽取结果
 	n := 0
@@ -203,7 +203,7 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 						}
 					} else if field == "review_experts" {
 						// 评审专家
-						if arr, ok :=tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
+						if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
 							arr1 := qutil.ObjArrToStringArr(arr)
 							newTmp[field] = strings.Join(arr1, ",")
 						}
@@ -223,7 +223,7 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 						}
 						if strings.Contains(detail, qutil.ObjToString(tmp["title"])) {
 							newTmp[field] = FilterDetail(detail)
-						}else {
+						} else {
 							newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
 						}
 					} else if field == "_id" || field == "topscopeclass" { //不做处理
@@ -278,7 +278,12 @@ var filterReg = regexp.MustCompile("<[^>]+>")
 var filterSpace = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]")
 
 func FilterDetail(text string) string {
-	return filterReg.ReplaceAllString(text, "")
+	text = filterReg.ReplaceAllString(text, "")
+	for _, s := range FilterKeyword {
+		reg := regexp.MustCompile(s)
+		text = reg.ReplaceAllString(text, "")
+	}
+	return text
 }
 
 func FilterDetailSpace(text string) string {
@@ -287,6 +292,6 @@ func FilterDetailSpace(text string) string {
 
 // 正则判断是否包含
 func checkContains(s, sub string) bool {
-	reg := regexp.MustCompile(`(?i)(^|([\s\t\n]+))(` +sub+`)($|([\s\t\n]+))`)
+	reg := regexp.MustCompile(`(?i)(^|([\s\t\n]+))(` + sub + `)($|([\s\t\n]+))`)
 	return reg.MatchString(s)
 }

udpcreateindex/src/config.json

@@ -4,9 +4,9 @@
   "uname": "root",
   "upwd": "root",
   "mongodb": {
-    "addr": "192.168.3.207:27092",
+    "addr": "192.168.3.207:27001",
     "pool": 10,
-    "db": "wjh"
+    "db": "qfw_data"
   },
   "savedb": {
     "addr": "192.168.3.207:27092",
@@ -30,15 +30,15 @@
     "type": "buyer"
   },
   "biddingback": {
-    "db": "wjh",
+    "db": "qfw_data",
     "collect": "bidding",
-    "index": "bidding",
+    "index": "bidding_v2",
     "type": "bidding"
   },
   "bidding": {
-    "db": "wjh",
+    "db": "qfw_data",
     "collect": "bidding",
-    "index": "bidding_v1",
+    "index": "bidding_v2",
     "type": "bidding",
     "extractdb": "wjh",
     "extractcollect": "extract",
@@ -66,9 +66,10 @@
     "projectinfomap": {
       "approvecode": "string", "approvecontent": "string", "approvestatus": "string", "approvetime": "string", "approvedept": "string", "approvenumber": "string", "projecttype": "string", "approvecity": "string"
     },
-    "purchasinglist": "itemname,brandname,model,unitname,number,unitprice,totalprice",
+    "purchasinglist": "itemname,brandname,model,unitname,number,unitprice,totalprice,projectname,buyer,item,projectscope,expurasingtime,reserved_amount",
     "purchasinglistmap": {
-      "itemname": "string", "brandname": "string", "model": "string", "unitname": "string", "number": "float64", "unitprice": "float64", "totalprice": "float64"
+      "itemname": "string", "brandname": "string", "model": "string", "unitname": "string", "number": "float64", "unitprice": "float64", "totalprice": "float64", "projectname": "string", "buyer": "string", "item": "string",
+      "projectscope": "string", "expurasingtime": "string", "reserved_amount": "string"
     },
     "winnerorder": "sort,sortstr,entname",
     "winnerordermap": {
@@ -124,5 +125,12 @@
     "addr": "http://192.168.3.206:9800",
     "pool": 12,
     "node": "4q7v7e6mQ5aeCwjUgM6HcA"
-  }
+  },
+  "filter-keyword": ["(招标网|千里马|采招网|招标采购导航网|招标与采购网|中国招投标网|中国采购与招标网|优质采)[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}(qianlima|zhaobiao|okcis|zbytb|infobidding|bidcenter|youzhicai|chinabidding)[a-z0-9.\\/\\/]{0,40}",
+    "招标网[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}zhaobiao[a-z0-9.\\/\\/]{0,40}",
+    "千里马[\\w\\W]{0,15}[a-z0-9:\\/\\/.]{0,20}qianlima[a-z0-9.\\/\\/]{0,10}",
+    "[\\(（]?(网址)?[:：；;]?(http|https|htpps)*[:：]?(//)?www.(zhaobiao|chinabidding|infobidding|zbytb|okcis|qianlima|youzhicai).(com|cn)+/?[\\)）]?",
+    "[\\(（]?(网址)?[:：][http|https|htpps]+([:：])//www.bidcenter.com.cn/?(qita/webseo80.shtml)?[\\)）]?",
+    "千里马(平台|网站)+", "[“\"]?优质采(电子交易平台|云采购平台|交易平台)?[”\"]?", "《?(中国采购与|中国)?招(投)?标(与采购|采购导航)?网》?",
+    "《?元博网(采购与招标网)?》?", "《?(中国)?招标采购导航网》?", "中国采招\\W*网[(（](bidcenter)?.*[)）]"]
 }

udpcreateindex/src/main.go

@@ -12,7 +12,6 @@ import (
 	_ "net/http/pprof"
 	"qfw/util"
 	elastic "qfw/util/elastic"
-	"qfw/util/redis"
 	"strings"
 	"time"
 	u "util"
@@ -61,23 +60,26 @@ var (
 	esAddr string
 	esNode string
 
-	ProvinceDict map[string][]Province //省份-map
-	CityDict     map[string][]City     //城市-map
-	DistrictDict map[string][]District //区县-map
+	FilterKeyword []string              //正文竟品关键词过滤
+	ProvinceDict  map[string][]Province //省份-map
+	CityDict      map[string][]City     //城市-map
+	DistrictDict  map[string][]District //区县-map
 
 	winner, bidding, biddingback, project, project2, buyer, standard, qyxy_ent map[string]interface{}
 )
 var UpdataMgoCache = make(chan []map[string]interface{}, 1000)
 var SP = make(chan bool, 5)
+var SaveLogChan = make(chan []map[string]interface{}, 1000)
+var SaveSp = make(chan bool, 5)
 
 var StopFlag = false // 程序生索引停止标志
 
 func init() {
 	util.ReadConfig(&Sysconfig)
 	// company_id
-	redis.InitRedis1("qyxy_id=172.17.4.189:8379", 4)
-	inits()
-	go checkMapJob()
+	//redis.InitRedis1("qyxy_id=172.17.4.189:8379", 4)
+	//inits()
+	//go checkMapJob()
 	detailLength = util.IntAllDef(Sysconfig["detaillength"], 50000)
 	fileLength = util.IntAllDef(Sysconfig["filelength"], 50000)
 	updport, _ = Sysconfig["updport"].(string)
@@ -214,6 +216,8 @@ func init() {
 	log.Println(purchasinglistFields)
 
 	initCheckCity()
+
+	FilterKeyword = util.ObjArrToStringArr(Sysconfig["filter-keyword"].([]interface{}))
 	//初始化oss
 	u.InitOss()
 
@@ -221,8 +225,8 @@ func init() {
 
 func main() {
 
-	go inspectQuery()
-	go task_index()
+	//go inspectQuery()
+	//go task_index()
 	go UpdateExtract() //抽取表中新增entidlist字段
 	updport := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
@@ -302,7 +306,15 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					defer func() {
 						<-pool
 					}()
-					biddingTask(data, mapInfo)
+					biddingTask(data, mapInfo, tasktype)
+				}()
+			case "bidding_history": //增量id段历史数据
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					biddingTask(data, mapInfo, tasktype)
 				}()
 			case "project":
 				pool <- true
@@ -469,3 +481,38 @@ func initCheckCity() {
 	}
 	util.Debug(fmt.Sprintf("城市配置加载完毕...省~%d 市~%d 区~%d", len(ProvinceDict), len(CityDict), len(DistrictDict)))
 }
+
+func saveLog() {
+	arru := make([][]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-SaveLogChan:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				SaveSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-SaveSp
+					}()
+					extractmgo.UpSertBulk("createIndex_log", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				SaveSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-SaveSp
+					}()
+					extractmgo.UpSertBulk("createIndex_log", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}

udpcreateindex/src/util/ossclient.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	ossEndpoint        = "https://oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
+	ossEndpoint        = "oss-cn-beijing.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
 	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
 	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
 	ossBucketName      = "topjy"