Browse Source

Merge branch 'loc3.4.1' into dev3.4

* loc3.4.1:
  no message
  no message
  no message

# Conflicts:
#	udpcreateindex/src/config.json
Jianghan 4 years ago
parent
commit
45ebf16fb3

+ 9 - 9
fullproject/src_v1/config.json

@@ -1,5 +1,5 @@
 {
-    "loadStart": 1,
+    "loadStart": 0,
 	"validdays":150,
     "statusdays": 15,
 	"mongodbServers": "192.168.3.207:27092",
@@ -8,16 +8,15 @@
 	"hints":"publishtime_1",
     "extractColl": "jh_info",
     "projectColl": "jh_project",
-    "updateColl": "jh_info",
-    "backupFlag": false,
+    "backupFlag": true,
     "siteColl": "site",
     "thread": 1,
     "jkmail": {
         "to": "wangjianghan@topnet.net.cn",
-        "api": "http://10.171.112.160:19281/_send/_mail"
+        "api": "http://172.17.145.179:19281/_send/_mail"
     },
     "es": {
-        "addr": "http://127.0.0.1:9800",
+        "addr": "http://192.168.3.128:9800",
         "index": "projectset",
         "itype": "projectset",
         "pool": 10
@@ -31,7 +30,7 @@
         },
         {
             "addr": "127.0.0.1",
-            "port": 14833,
+            "port": 1483,
             "memo": "修改项目创建new"
         }
     ],
@@ -54,15 +53,16 @@
         "中班椅", "书架", "书柜", "休息台", "不争辩", "不准确", "不压缩", "不处于", "不挑剔", "不留密码", "中标候选", "中级", "义务", "不存", "东山", "东莞", "中山", "佛山", "南方基地", "不处", "中材", "中美合资", "五效", "万元整", "不漏项", "东进西移",
         "业务", "中级职称", "水土", "成都", "税率为", "合肥", "天津", "保养等", "东湖", "万张图片", "仪表", "中标人", "中标内容", "书桌", "不卡顿", "不跳帧", "不完整", "不组织", "之中", "东建业", "东北特钢", "业主", "不锈钢", "中转",
         "凭借栏杆", "不含", "不含斜", "中杯", "光分纤箱", "光交箱","宜宾", "合柜", "性质", "不及格等", "中央空调", "中检", "乌鲁木齐", "五个环", "乙方", "保密", "保证", "人民币", "无效", "以活动", "从化", "休息室", "位置", "储入", "中间件", "公众",
-        "关键", "化学", "家禽", "水产等", "羊肉", "运行", "金属", "定标", "银行", "本科", "理由", "第五章", "计算机", "通信", "项目管理", "评标", "利益", "香港", "在家", "年度", "不得", "不提供", "中职教材", "以及", "业务", "种类", "从业人员"],
+        "关键", "化学", "家禽", "水产等", "羊肉", "运行", "金属", "定标", "银行", "本科", "理由", "第五章", "计算机", "通信", "项目管理", "评标", "利益", "香港", "在家", "年度", "不得", "不提供", "中职教材", "以及", "业务", "种类", "从业人员", "公告于"],
     "winner": {
         "pre_regexp": ["及报价\\d拟中标人第一:", "中标金额(元)", "第\\d中标候单位:", "第\\d中标侯选人:", "及报价\\d", "成交金额(元)", "成交金额", "成交金额包\\d", "成交金额(元)", "成交金额", "中标供应商为", "</tr><tr><tdcolspan=\"8\">",
             "中标供应商如下:", "中标供应商为", "中标供应商联系地址中标金额(万元)\\d", "基本情况名称:", "基本情况中标候选人第\\d+名:", "______", "第一中标侯选人名称: \\d", "第一中标侯选单位:", "第一中标供应商-", "第一中标排序人:", "第一中标排序人",
             "第一成交侯选人:", "第一成交供应商-", "第一成交候选供应商及报价:", "拟定供应商名称:", "名单及其排序名次第一名单位名称", "名单及其排序名\\d次第一名单位名称", "名单及其排序推荐中标人单位名称", "名单及其排序", "名称:", "是:", "\\(成交\\)", "\\)¥41100.00",
             "\\)为:", "预成交单位:", "最终报价第(二|三)低的", "同意推荐最终报价最低的", "单位名称", "的供应商", "推荐", "第一名", "\\([A-Z]\\)", "([A-Z])", "<[^>]*>", "[^a-zA-Z\\p{Han}]{1,}", "[A-Z](-|、)", "A\\d+", "B\\d+", "?[a-zA-Z]?[包|段]",
-            "①"],
+            "①", "投标人被人民法院[,]{0,}"],
         "back_regexp": ["为中标人$", "为预中标供应商$", "为成交候选人$", "为该项目的预成交单位$", "为成交单位$", "为本次招标项目的中标单位$", "(公司$", "工期:0质量要求:null保证金金额:null$", ",报价:182391224.2900工期$", "786000元$", "预成交人地址",
-            "为该项目的预成交单位$", "为成交单位$", "预成交单位$", ",?(投标报价|报价)\\d{1,}\\.\\d{1,}(万?)元$", ",?(投标报价|报价)\\d+.?\\d+(万?)元$", ",?(投标报价|报价)\\d+.?\\d+(万?)元/平方米$", ",\\[慧通主要产品报价点击下载\\]$", ",$", ",?中标金额\\d{1,}(万?)元$", ",?中标金额\\d{1,}\\.\\d{1,}(万?)元$",
+            "为该项目的预成交单位$", "为成交单位$", "预成交单位$", ",投标人被人民法院$",
+            ",?(投标报价|报价)\\d{1,}\\.\\d{1,}(万?)元$", ",?(投标报价|报价)\\d+.?\\d+(万?)元$", ",?(投标报价|报价)\\d+.?\\d+(万?)元/平方米$", ",\\[慧通主要产品报价点击下载\\]$", ",$", ",?中标金额\\d{1,}(万?)元$",",?中标金额\\d{1,}\\.\\d{1,}(万?)元$",
             ",_$", "单位名称$", ",为本项目.*$", "[^a-zA-Z\\p{Han}]{1,}$"],
         "back_rep_regexp": ["(有限公$)#有限公司", "(有限责任公$)#有限责任公司", "(公司公司&)#公司", "(公司等&)#公司"],
         "blacklist": ["项目废标", "标项内容", "单位名称", "null", "不足3家", "中标单位", "公告时间", "(或印鉴)", "中标金额", "法定代表人", "员会名单", "含全部内", "工期", "报价", "名称得分", "元)备注", "的全部内", "地址", "\\.\\.", "法定家数", "的投标人",

+ 0 - 2
fullproject/src_v1/init.go

@@ -21,7 +21,6 @@ var (
 	Sysconfig                                      map[string]interface{} //读取配置文件
 	MongoTool                                      *MongodbSim            //mongodb连接
 	ExtractColl, ProjectColl, BackupColl, SiteColl string                 //抽取表、项目表、项目快照表、站点表
-	UpdateColl                                     string                 // 金额修改数据表
 	Thread                                         int                    //配置项线程数
 	//NextNode                 []interface{}
 	BlackList    								   []interface{}
@@ -66,7 +65,6 @@ func init() {
 
 	ExtractColl = Sysconfig["extractColl"].(string)
 	ProjectColl = Sysconfig["projectColl"].(string)
-	UpdateColl = Sysconfig["updateColl"].(string)
 	BackupColl = Sysconfig["projectColl"].(string) + "_back"
 	SiteColl = Sysconfig["siteColl"].(string)
 	Thread = util.IntAll(Sysconfig["thread"])

+ 1 - 1
fullproject/src_v1/load_data.go

@@ -18,7 +18,7 @@ func (p *ProjectTask) loadData(starttime int64) {
 	sess := MongoTool.GetMgoConn()
 	defer MongoTool.DestoryMongoConn(sess)
 	q := map[string]interface{}{
-		"lasttime": map[string]interface{}{"$gt": starttime},
+		"lasttime": map[string]interface{}{"$gte": starttime},
 	}
 	it := sess.DB(MongoTool.DbName).C(p.coll).Find(&q).Iter()
 	n := 0

+ 5 - 27
fullproject/src_v1/main.go

@@ -36,7 +36,6 @@ func init() {
 			Port: util.IntAll(m["port"]),
 		})
 	}
-
 	es := Sysconfig["es"].(map[string]interface{})
 	Es = &elastic.Elastic{
 		S_esurl: util.ObjToString(es["addr"]),
@@ -121,28 +120,6 @@ func mainT() {
 	time.Sleep(99999 * time.Hour)
 }
 
-func mainS() {
-	id := "5987e5e85d11e1c745d36c4c"
-	mapinfo := map[string]interface{}{}
-	mapinfo["id"] = id
-	mapinfo["stype"] = "updateMoney"
-	mapinfo["budget"] = "12345"
-	mapinfo["ip"] = "127.0.0.1"
-	mapinfo["port"] = Sysconfig["udpport"]
-	if Sysconfig["loadStart"] != nil {
-		loadStart := util.Int64All(Sysconfig["loadStart"])
-		if loadStart > -1 {
-			P_QL.loadData(loadStart)
-		}
-	}
-	P_QL.loadSite()
-	P_QL.currentType = mapinfo["stype"].(string)
-	P_QL.pici = time.Now().Unix()
-	P_QL.taskUpdateMoney(mapinfo)
-	P_QL.Brun = true
-	time.Sleep(20 * time.Second)
-}
-
 //udp调用信号
 func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 	switch act {
@@ -189,24 +166,25 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					P_QL.pici = time.Now().Unix()
 					P_QL.taskUpdateInfo(mapInfo)
 				}()
-			case "updateMoney": //修改金额
+			case "updatePro": //修改项目外围字段(只修改外围字段值)
 				go func() {
 					defer func() {
 						<-SingleThread
 					}()
 					P_QL.currentType = tasktype
 					P_QL.pici = time.Now().Unix()
-					P_QL.taskUpdateMoney(mapInfo)
+					P_QL.taskUpdatePro(mapInfo)
 				}()
-			case "updateMoneyMgo": //修改金额
+			case "deleteInfo":	// 删除招标公告
 				go func() {
 					defer func() {
 						<-SingleThread
 					}()
 					P_QL.currentType = tasktype
 					P_QL.pici = time.Now().Unix()
-					P_QL.taskQuery()
+					P_QL.delInfoPro(mapInfo)
 				}()
+
 			case "history": //历史数据合并,暂时不写
 				go func() {
 					defer func() {

+ 46 - 21
fullproject/src_v1/project.go

@@ -426,24 +426,6 @@ var FIELDS = []string{
 	"package",
 }
 
-var bidtype = map[string]string{
-	"招标": "招标",
-	"邀标": "邀标",
-	"询价": "询价",
-	"单一": "单一",
-	"竞价": "竞价",
-	"竞谈": "竞谈",
-}
-
-var bidstatus = map[string]string{
-	"预告": "预告",
-	"中标": "中标",
-	"成交": "成交",
-	"废标": "废标",
-	"流标": "流标",
-	"合同": "合同",
-}
-
 //招标时间zbtime、中标时间jgtime、项目状态bidstatus、招标类型bidtype、最后发布时间lasttime、首次发布时间firsttime
 
 func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *ProjectInfo) {
@@ -556,7 +538,9 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	if len(thisinfo.WinnerOrder) > 0 {
 		var list = []string{}
 		for _, v := range thisinfo.WinnerOrder {
-			list = append(list, qu.ObjToString(v["entname"]))
+			if BinarySearch(list, qu.ObjToString(v["entname"])) == -1 {
+				list = append(list, qu.ObjToString(v["entname"]))
+			}
 		}
 		set["winnerorder"] = list
 		p1.Winnerorder = list
@@ -828,7 +812,6 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 			set["district"] = thisinfo.District
 		}
 	}
-	set["district"] = thisinfo.District
 	//6--项目名称
 	if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
 		pInfo.ProjectName = thisinfo.ProjectName
@@ -912,7 +895,9 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	if len(thisinfo.WinnerOrder) > 0 {
 		var list = []string{}
 		for _, v := range thisinfo.WinnerOrder {
-			list = append(list, qu.ObjToString(v["entname"]))
+			if BinarySearch(list, qu.ObjToString(v["entname"])) == -1 {
+				list = append(list, qu.ObjToString(v["entname"]))
+			}
 		}
 		set["winnerorder"] = list
 		pInfo.Winnerorder = list
@@ -1342,3 +1327,43 @@ func IsCreatePro(info *Info) (bol bool) {
 	}
 	return bol
 }
+
+var bidtype = map[string]string{
+	"招标": "招标",
+	"邀标": "邀标",
+	"询价": "询价",
+	"单一": "单一",
+	"竞价": "竞价",
+	"竞谈": "竞谈",
+}
+
+var bidstatus = map[string]string{
+	"预告": "预告",
+	"中标": "中标",
+	"成交": "成交",
+	"废标": "废标",
+	"流标": "流标",
+	"合同": "合同",
+}
+
+func GetBidTypeAndBidStatus(info *Info) (string, string) {
+	typeStr := bidtype[info.TopType]
+	statusStr := bidstatus[info.SubType]
+	if info.Infoformat == 2 || info.SubType == "拟建" {
+		statusStr = "拟建"
+		typeStr = ""
+	} else {
+		if bidtype[typeStr] == "" {
+			typeStr = "招标"
+		}
+		if typeStr == "招标" {
+			statusStr = typeStr
+		} else {
+			if statusStr == "" {
+				statusStr = "其它"
+			}
+		}
+	}
+	return typeStr, statusStr
+
+}

+ 165 - 271
fullproject/src_v1/task.go

@@ -6,6 +6,7 @@ import (
 	"gopkg.in/mgo.v2/bson"
 	"log"
 	mu "mfw/util"
+	"mgoutil/mongodb"
 	"qfw/util"
 	"regexp"
 	"strings"
@@ -13,6 +14,7 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/goinggo/mapstructure"
 	"github.com/robfig/cron"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
@@ -29,12 +31,26 @@ var BackRegexp = map[string][]*regexp.Regexp{}
 var BackRepRegexp = map[string][]RegexpInfo{}
 var BlackRegexp = map[string][]*regexp.Regexp{}
 
+var (
+	//从标题获取项目编号
+	titleGetPc  = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)")
+	titleGetPc1 = regexp.MustCompile("[\\[【((](.{0,6}(编号|编码|项号|包号|代码|标段?号)[::为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()()][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()()][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】))]")
+	titleGetPc2 = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$")
+	//项目编号过滤
+	pcReplace = regexp.MustCompile("([\\[【((〖〔《{﹝{](重|第?[二三四再]次.{0,4})[\\]】))〗〕》}﹞}])$|[\\[\\]【】()()〖〗〔〕《》{}﹝﹞-;{}–  ]+|(号|重|第?[二三四五再]次(招标)?)$|[ __]+|((采购)?项目|采购(项目)?)$")
+	//项目编号只是数字或只是字母4个以下
+	StrOrNum = regexp.MustCompile("^[0-9_-]{1,4}$|^[a-zA-Z_-]{1,4}$")
+	//纯数字或纯字母
+	StrOrNum2 = regexp.MustCompile("^[0-9_-]+$|^[a-zA-Z_-]+$")
+	//含分包词,招标未识别分包  合并到一个项目
+	KeyPackage = regexp.MustCompile("[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}(包|段)|(包|段)[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}")
+)
+
 type RegexpInfo struct {
 	regs   *regexp.Regexp
 	repstr string
 }
 
-
 //项目合并对象
 type ProjectTask struct {
 	InitMinTime int64 //最小时间,小于0的处理一次
@@ -322,187 +338,53 @@ func (p *ProjectTask) taskZl(udpInfo map[string]interface{}) {
 //招标字段更新
 func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
 	defer util.Catch()
-	db := MongoTool.DbName
-	coll, _ := udpInfo["coll"].(string)
-	if coll == "" {
-		coll = ExtractColl
-	}
-	thread := util.IntAllDef(Thread, 4)
-	if thread > 0 {
-		p.thread = thread
+	infoid := udpInfo["infoid"].(string)
+	infoMap := MongoTool.FindById(ExtractColl, infoid)
+	if infoMap["modifyinfo"] == nil {
+		util.Debug("does not exist modifyinfo ---,", infoid)
+		return
 	}
-	q, _ := udpInfo["query"].(map[string]interface{})
-	gtid := udpInfo["gtid"].(string)
-	lteid := udpInfo["lteid"].(string)
-	if q == nil {
-		q = map[string]interface{}{
-			"_id": map[string]interface{}{
-				"$gte": StringTOBsonId(gtid),
-				"$lte": StringTOBsonId(lteid),
-			},
-			"is_m": 1,
-		}
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+	esquery := `{"query": {"bool": {"must": [{"match": {"ids": "`+infoid+`"}}]}}}`
+	data := Es.Get(Index, Itype, esquery)
+	if len(*data) > 0 {
+		pid := util.ObjToString(((*data)[0])["_id"])
+		p.updateJudge(infoMap, pid)
+	}else {
+		util.Debug("not find project---,", infoid)
 	}
-	log.Println("查询语句:", q)
-	p.enter(db, coll, q)
 }
 
-func (p *ProjectTask) taskQuery() {
+func (p *ProjectTask) taskUpdatePro(udpInfo map[string]interface{}) {
 	defer util.Catch()
-	count := 0
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
-	fields := map[string]interface{} {"budget": 1, "bidamount": 1, "package": 1}
-	ms := sess.DB(MongoTool.DbName).C(UpdateColl).Find(map[string]interface{}{}).Select(fields)
-	query := ms.Iter()
-L:
-	for {
-		tmp := make(map[string]interface{})
-		if query.Next(&tmp) {
-			lastid := tmp["_id"]
-			tmp["id"] = tmp["_id"].(primitive.ObjectID).Hex();
-			if count%1000 == 0 {
-				log.Println("current modify", count, lastid)
-			}
-			p.taskUpdateMoney(tmp)
-			count++
-		} else {
-			break L
-		}
+	util.Debug(udpInfo)
+	pid := util.ObjToString(udpInfo["pid"])
+	updateMap := util.ObjToMap(udpInfo["updateField"])
+	if pid == "" || len(*updateMap) == 0 {
+		util.Debug("参数有误")
+		return
 	}
-}
-
-//修改公告信息的预算/中标金额
-func (p *ProjectTask) taskUpdateMoney(udpInfo map[string]interface{}) {
-	defer util.Catch()
-	id := udpInfo["id"].(string)
-	budget := util.Float64All(udpInfo["budget"])
-	bidamount := util.Float64All(udpInfo["bidamount"])
-
-	client := Es.GetEsConn()
-	defer Es.DestoryEsConn(client)
-	esquery := `{"query": {"bool": {"must": [{"term": {"list.infoid": "`+id+`"}}]}}}`
-	data := Es.Get(Index, Itype, esquery)
-	if len(*data) > 0 {
-		pid := util.ObjToString((*data)[0]["_id"])
-		pro := MongoTool.FindById(ProjectColl, pid)
-		if len(pro) == 0 {
-			util.Debug("未找到项目, pid=", pid)
-			return
-		}
-		var info *map[string]interface{}
-		for _, v := range []interface{}(pro["list"].(primitive.A)){
-			v1 := v.(map[string]interface{})
-			if util.ObjToString(v1["infoid"]) == id {
-				info = util.ObjToMap(v)
-				infoField := util.ObjToMap(pro["infofield"])
-				if udpInfo["budget"] != nil{
-					util.Debug("update-------", (*info)["infoid"])
-					//if pro["budget"] == (*info)["budget"] {
-					//	pro["budget"] = budget
-					//}
-					//多包中的金额
-					if util.IntAll(pro["multipackage"]) == 1 {
-						if packages, ok := pro["package"].(map[string]interface{}); ok {
-						M :
-							for k, v := range packages{
-								v1 := []interface{}(v.(primitive.A))
-								for _, v2 := range v1{
-									v3 := v2.(map[string]interface{})
-									if util.ObjToString(v3["infoid"]) == id {
-										if v3["budget"] != nil {
-											pkg := udpInfo["package"].(map[string]interface{})
-											tmp := pkg[k].(map[string]interface{})
-											v3["budget"] = tmp["budget"]
-										}
-									}else {
-										break M
-									}
-								}
-							}
-						}
-					}
-					(*info)["budget"] = budget
-					(*util.ObjToMap((*infoField)[id]))["budget"] = budget
-					if pro["sortprice"] == (*info)["budget"] {
-						pro["sortprice"] = budget
-					}
-				}else {
-					delete(*info, "budget")
-				}
-				if udpInfo["bidamount"] != nil{
-					//if pro["bidamount"] == (*info)["bidamount"] {
-					//	pro["bidamount"] = bidamount
-					//}
-					v1["bidamount"] = bidamount
-					if util.IntAll(pro["multipackage"]) == 1 {
-						if packages, ok := pro["package"].(map[string]interface{}); ok {
-							for k, v := range packages{
-								v1 := []interface{}(v.(primitive.A))
-								for _, v2 := range v1{
-									v3 := v2.(map[string]interface{})
-									if util.ObjToString(v3["infoid"]) == id {
-										if v3["bidamount"] != nil {
-											pkg := udpInfo["package"].(map[string]interface{})
-											tmp := pkg[k].(map[string]interface{})
-											v3["bidamount"] = tmp["bidamount"]
-										}
-									}
-								}
-							}
-						}
-					}
-					(*info)["bidamount"] = bidamount
-					(*util.ObjToMap((*infoField)[id]))["bidamount"] = bidamount
-					if pro["sortprice"] == (*info)["bidamount"] {
-						pro["sortprice"] = bidamount
-					}
-				}else {
-					delete(*info, "bidamount")
-				}
-				break
-			}
-		}
-		var project *ProjectInfo
-		var pInfo *Info
-		bys, _ := json.Marshal(pro)
-		_ = json.Unmarshal(bys, &project)
-		bys1, _ := json.Marshal(info)
-		_ = json.Unmarshal(bys1, &pInfo)
-		if len(project.Ids) > 1 {
-			CountAmount(project, pInfo, *info)
-			if project.Budget > 0 {
-				pro["budget"] = project.Budget
-			}
-			if project.Bidamount > 0 {
-				pro["bidamount"] = project.Bidamount
-			}
-		}else {
-			pro["budget"] = budget
-			pro["bidamount"] = bidamount
-			if budget > bidamount {
-				pro["sortprice"] = budget
+	proMap := MongoTool.FindById(ProjectColl, pid)
+	if len(proMap) > 1 {
+		proMap["reason"] = "直接修改项目字段信息"
+		backupPro(proMap)
+		delete(proMap, "reason")
+		updataMap := make(map[string]interface{})
+		modifyInfo := make(map[string]interface{})
+		for k, v := range *updateMap{
+			if strings.Contains(k, "time") {
+				updataMap[k] = util.Int64All(v)
 			}else {
-				pro["sortprice"] = bidamount
+				updataMap[k] = v
 			}
+			modifyInfo[k] = true
 		}
-		set := map[string]interface{}{
-			"$set": pro,
-		}
-		MongoTool.UpdateById(ProjectColl, pid, set)
-
-		loadStart := util.Int64All(Sysconfig["loadStart"])
-		if loadStart > -1 && project.LastTime >loadStart {
-			util.Debug("内存中存在该项目信息", project.Id)
-			p.AllIdsMapLock.Lock()
-			p.AllIdsMap[pid].P = project
-			p.AllIdsMapLock.Unlock()
-		}
-
-		bol := Es.DelById(Index, Itype, pid)
+		updataMap["modifyinfo"] = modifyInfo
+		util.Debug(updataMap)
+		bol := MongoTool.UpdateById(ProjectColl, pid, map[string]interface{}{"$set": updataMap})
 		if bol {
-			util.Debug("删除es索引, pid------", pid)
-			//调udp生索引
+			//es索引
 			by, _ := json.Marshal(map[string]interface{}{
 				"query": map[string]interface{}{
 					"_id": bson.M{
@@ -514,24 +396,40 @@ func (p *ProjectTask) taskUpdateMoney(udpInfo map[string]interface{}) {
 			util.Debug(string(by))
 			_ = udpclient.WriteUdp(by, mu.OP_TYPE_DATA, toaddr[1])
 		}
+		// 内存
+		var pro ProjectInfo
+		err := mapstructure.Decode(proMap, &pro)
+		if err != nil {
+			util.Debug(err)
+		}
+		p.AllIdsMapLock.Lock()
+		if v, ok := p.AllIdsMap[pid]; ok {
+			v.P = &pro
+		}
+		p.AllIdsMapLock.Unlock()
+	}else {
+		util.Debug("Not find project---", pid)
 	}
 }
 
-func FindMoney(key string, project map[string]interface{}) float64 {
-	money := -0.1
-	for i, v := range []interface{}(project["list"].(primitive.A)){
-		v1 := v.(map[string]interface{})
-		if i == 0 {
-			if v1[key] != nil {
-				money = util.Float64All(v1[key])
-			}
-		}else {
-			if v1[key] != nil && util.Float64All(v1[key]) > money {
-				money = util.Float64All(v1[key])
-			}
-		}
+func (p *ProjectTask) delInfoPro(udpInfo map[string]interface{}) {
+	defer util.Catch()
+	util.Debug(udpInfo)
+	infoid := util.ObjToString(udpInfo["infoid"])
+	if infoid == "" {
+		util.Debug("参数有误")
+		return
+	}
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+	esquery := `{"query": {"bool": {"must": [{"match": {"ids": "`+infoid+`"}}]}}}`
+	data := Es.Get(Index, Itype, esquery)
+	if len(*data) > 0 {
+		pid := util.ObjToString(((*data)[0])["_id"])
+		p.delJudge(infoid, pid)
+	}else {
+		util.Debug("not find project---,", infoid)
 	}
-	return money
 }
 
 func StringTOBsonId(id string) primitive.ObjectID {
@@ -588,13 +486,8 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 						p.fillInPlace(tmp)
 						info := ParseInfo(tmp)
 						p.currentTime = info.Publishtime
-						if p.currentType == "updateInfo" {
-							//招标信息更改合并
-							p.updateJudge(tmp, info)
-						} else {
-							//普通合并
-							p.CommonMerge(tmp, info)
-						}
+						//普通合并
+						p.CommonMerge(tmp, info)
 					} else {
 						//信息错误,进行更新
 						util.Debug(tmp["_id"])
@@ -649,21 +542,6 @@ L:
 
 }
 
-var (
-	//从标题获取项目编号
-	titleGetPc  = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)")
-	titleGetPc1 = regexp.MustCompile("[\\[【((](.{0,6}(编号|编码|项号|包号|代码|标段?号)[::为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()()][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()()][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】))]")
-	titleGetPc2 = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$")
-	//项目编号过滤
-	pcReplace = regexp.MustCompile("([\\[【((〖〔《{﹝{](重|第?[二三四再]次.{0,4})[\\]】))〗〕》}﹞}])$|[\\[\\]【】()()〖〗〔〕《》{}﹝﹞-;{}–  ]+|(号|重|第?[二三四五再]次(招标)?)$|[ __]+|((采购)?项目|采购(项目)?)$")
-	//项目编号只是数字或只是字母4个以下
-	StrOrNum = regexp.MustCompile("^[0-9_-]{1,4}$|^[a-zA-Z_-]{1,4}$")
-	//纯数字或纯字母
-	StrOrNum2 = regexp.MustCompile("^[0-9_-]+$|^[a-zA-Z_-]+$")
-	//含分包词,招标未识别分包  合并到一个项目
-	KeyPackage = regexp.MustCompile("[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}(包|段)|(包|段)[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}")
-)
-
 func (p *ProjectTask) CommonMerge(tmp map[string]interface{}, info *Info) {
 	if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
 		if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
@@ -828,92 +706,108 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	return thisinfo
 }
 
-func (p *ProjectTask) updateJudge(tmp map[string]interface{}, info *Info) {
-	index := -1
-	pInfoId := ""
-	p.AllIdsMapLock.Lock()
-	F:
-		for k, ID := range p.AllIdsMap {
-			for i, id := range ID.P.Ids {
-				if info.Id == id {
-					pInfoId = k
-					index = i
-					break F
-				}
+func (p *ProjectTask) updateJudge(infoMap map[string]interface{}, pid string) {
+	tmpPro := MongoTool.FindById(ProjectColl, pid)
+
+	modifyProMap := make(map[string]interface{}) // 修改项目的字段
+	for k := range infoMap{
+		if modifyMap, ok := infoMap["modifyinfo"].(map[string]interface{}); ok {
+			if modifyMap[k] != nil {
+				modifyProMap[k] = infoMap[k]
 			}
 		}
+	}
+	if len(modifyProMap) == 0 {
+		util.Debug("修改招标公告信息不需要修改项目信息字段", infoMap["_id"])
+		return
+	}
+	p.AllIdsMapLock.Lock()
+	_, ok := p.AllIdsMap[pid]
 	p.AllIdsMapLock.Unlock()
-	//未找到招标信息
-	if index == -1 {
-		if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
-			p.currentTime = info.Publishtime
-			p.startProjectMerge(info, tmp)
+	ids := []interface{}(tmpPro["ids"].(primitive.A))
+	index, position := -1, 0		// index 0:第一个,1:中间,2:最后一个   position list中位置
+
+	for i, v := range ids {
+		if util.ObjToString(v) == mongodb.BsonIdToSId(infoMap["_id"]) {
+			position = i
+			if i == 0 {
+				index = 0
+			}else if i == len(ids) - 1 {
+				index = 2
+			}else {
+				index = 1
+			}
 		}
-	} else {
-		tmpPro := MongoTool.FindById(ProjectColl, pInfoId)
-		infoList := []interface{}(tmpPro["list"].(primitive.A))
-		infoMap := infoList[index].(map[string]interface{})
-		modifyMap, f := modifyEle(infoMap, tmp)
+	}
+	if ok {
+		// 周期内
 		//projecthref字段
-		jsonData := tmp["jsondata"].(map[string]interface{})
-		if jsonData != nil && jsonData["projecthref"] != nil {
-			proHref := jsonData["projecthref"].(string)
-			tmp["projecthref"] = proHref
-			p.mapHrefLock.Lock()
-			pid := p.mapHref[proHref]
-			p.mapHrefLock.Unlock()
-			if pid == pInfoId {
-				p.modifyUpdate(pInfoId, index, info, tmp, tmpPro, modifyMap)
-				return
+		if infoMap["jsondata"] != nil {
+			jsonData := infoMap["jsondata"].(map[string]interface{})
+			if proHref, ok := jsonData["projecthref"].(string); ok {
+				p.mapHrefLock.Lock()
+				tempId := p.mapHref[proHref]
+				p.mapHrefLock.Unlock()
+				if pid == tempId {
+					p.modifyUpdate(pid, index, position, tmpPro, modifyProMap)
+				}else {
+					util.Debug("projecthref data id err---pid=" + pid, "---"+tempId)
+				}
+			}else {
+				f := modifyEle(modifyProMap)
+				if f {
+					//合并、修改
+					util.Debug("合并修改更新", "----------------------------")
+					p.mergeAndModify(pid, index, position, infoMap, tmpPro, modifyProMap)
+				} else {
+					//修改
+					util.Debug("修改更新", "----------------------------")
+					p.modifyUpdate(pid, index, position, tmpPro, modifyProMap)
+				}
+			}
+		}else {
+			f := modifyEle(modifyProMap)
+			if f {
+				//合并、修改
+				util.Debug("合并修改更新", "----------------------------")
+				p.mergeAndModify(pid, index, position, infoMap, tmpPro, modifyProMap)
+			} else {
+				//修改
+				util.Debug("修改更新", "----------------------------")
+				p.modifyUpdate(pid, index, position, tmpPro, modifyProMap)
 			}
 		}
-
-		if f {
-			//合并、修改
-			log.Println("合并修改更新", "----------------------------")
-			p.mergeAndModify(pInfoId, index, info, tmp, tmpPro, modifyMap)
-		} else {
-			//修改
-			log.Println("修改更新", "----------------------------")
-			p.modifyUpdate(pInfoId, index, info, tmp, tmpPro, modifyMap)
-		}
+	}else {
+		// 周期外
+		p.modifyUpdate(pid, index, position, tmpPro, infoMap)
 	}
 }
 
 var Elements = []string{
 	"projectname",
 	"projectcode",
+	"buyer",
 	"agency",
-	"budget",
-	"bidamount",
-	"buyerperson",
 	"area",
 	"city",
 	"publishtime",
+	"toptype",
+	"subtype",
 }
 
 /**
-判断修改的字段是否是影响合并流程的要素字段
+	修改的字段
+	修改的字段是否是影响合并流程的要素字段
 */
-func modifyEle(tmpPro map[string]interface{}, tmp map[string]interface{}) (map[string]interface{}, bool) {
-	modifyMap := map[string]interface{}{}
-	for k := range tmpPro {
-		for k1 := range tmp {
-			if k == k1 && tmpPro[k] != tmp[k1] {
-				modifyMap[k] = tmp[k1]
-				break
-			}
-		}
-	}
-	for k := range modifyMap {
-		for _, str := range Elements {
-			if k == str {
-				return modifyMap, true
-			}
+func modifyEle(tmp map[string]interface{}) bool {
+	merge := false
+	for _, str := range Elements {
+		if tmp[str] != nil {
+			merge = true
+			break
 		}
 	}
-	delete(modifyMap, "_id")
-	return modifyMap, false
+	return merge
 }
 
 //补全位置信息

File diff suppressed because it is too large
+ 826 - 215
fullproject/src_v1/update.go


+ 37 - 12
fullproject/udp/src/main.go

@@ -7,22 +7,24 @@ import (
 	mu "mfw/util"
 	"net"
 	"os"
-	"time"
 )
 
-var ip, sid, eid, stype string
+var ip, sid, eid, stype, infoid, pid, udpataField string
 var p int
 
 func main() {
 
 	flag.StringVar(&sid, "sid", "", "开始id")
 	flag.StringVar(&eid, "eid", "", "结束id")
+	flag.StringVar(&infoid, "infoid", "", "修改的公告id")
+	flag.StringVar(&pid, "pid", "", "修改的项目id")
+	flag.StringVar(&udpataField, "field", "", "修改的字段信息")
 	flag.StringVar(&ip, "ip", "127.0.0.1", "ip")
 	flag.IntVar(&p, "p", 1482, "端口")
 	flag.StringVar(&stype, "stype", "", "stype")
 	flag.Parse()
 
-	if ip != "" && p > 0 && sid != "" && eid != "" {
+	if stype != ""{
 		addr := &net.UDPAddr{
 			IP:   net.ParseIP(ip),
 			Port: p,
@@ -35,16 +37,39 @@ func main() {
 				os.Exit(0)
 			}
 		})
-		m1 := map[string]interface{}{
-			"gtid":  sid,
-			"lteid": eid,
-			"stype": stype,
+		if stype == "updatePro" {
+			udpatamap := make(map[string]interface{})
+			err := json.Unmarshal([]byte(udpataField), &udpatamap)
+			if err == nil {
+				m1 := map[string]interface{}{
+					"pid":  pid,
+					"updateField": udpatamap,
+					"stype": stype,
+				}
+				by, _ := json.Marshal(m1)
+				log.Println(string(by))
+				udp.WriteUdp(by, mu.OP_TYPE_DATA, addr)
+			}else {
+				log.Println(err)
+			}
+		}else if stype == "deleteInfo" || stype == "updateInfo" {
+			m1 := map[string]interface{}{
+				"infoid":  infoid,
+				"stype": stype,
+			}
+			by, _ := json.Marshal(m1)
+			log.Println(string(by))
+			udp.WriteUdp(by, mu.OP_TYPE_DATA, addr)
+		}else if sid != "" && eid != "" {
+			m1 := map[string]interface{}{
+				"gtid":  sid,
+				"lteid": eid,
+				"stype": stype,
+			}
+			by, _ := json.Marshal(m1)
+			log.Println(string(by))
+			udp.WriteUdp(by, mu.OP_TYPE_DATA, addr)
 		}
-
-		by, _ := json.Marshal(m1)
-		log.Println(string(by))
-		udp.WriteUdp(by, mu.OP_TYPE_DATA, addr)
-		time.Sleep(30 * time.Second)
 	} else {
 		flag.PrintDefaults()
 		log.Println("参数错误.")

+ 2 - 2
qyxy/src/config.json

@@ -1,11 +1,11 @@
 {
 	"mgodb": "192.168.3.207:27092",
 	"dbsize": 12,
-	"dbname": "mxs",
+	"dbname": "mixdata",
 	"dbcoll": "qyxy",
 	"savecoll": "qyxy_std",
 	"tasktime": 1,
-	"updatetime": 1597202468,
+	"updatetime": 0,
 	"elastic": {
         "addr": "http://192.168.3.11:9800",
         "index": "qyxy_v1",

+ 2 - 2
qyxy/src/main.go

@@ -70,8 +70,8 @@ func init() {
 }
 
 func main() {
-	go TimeTask()
-	//QyxyStandard()
+	//go TimeTask()
+	QyxyStandard()
 	ch := make(chan bool, 1)
 	<-ch
 }

+ 24 - 24
qyxy/src/task.go

@@ -371,31 +371,31 @@ func QyxyStandard() bool {
 
 			}
 			//es数据过滤
-			EsSaveFlag := true
-			company_name := qu.ObjToString(esMap["company_name"])
-			if len([]rune(company_name)) < 8 {
-				EsSaveFlag = false
-			}
-			if EsSaveFlag {
-				company_type := qu.ObjToString(esMap["company_type"])
-				if company_type == "" || company_type == "个体工商户" {
-					EsSaveFlag = false
-				}
-			}
-			if EsSaveFlag {
-				credit_no := strings.TrimSpace(qu.ObjToString(esMap["credit_no"]))
-				company_code := strings.TrimSpace(qu.ObjToString(esMap["company_code"]))
-				if credit_no == "" && company_code == "" {
-					EsSaveFlag = false
-				}
-			}
-			//qu.Debug("esMap---", esMap)
-			// qu.Debug("mgoMap---", mgoMap)
-			// return
+			//EsSaveFlag := true
+			//company_name := qu.ObjToString(esMap["company_name"])
+			//if len([]rune(company_name)) < 8 {
+			//	EsSaveFlag = false
+			//}
+			//if EsSaveFlag {
+			//	company_type := qu.ObjToString(esMap["company_type"])
+			//	if company_type == "" || company_type == "个体工商户" {
+			//		EsSaveFlag = false
+			//	}
+			//}
+			//if EsSaveFlag {
+			//	credit_no := strings.TrimSpace(qu.ObjToString(esMap["credit_no"]))
+			//	company_code := strings.TrimSpace(qu.ObjToString(esMap["company_code"]))
+			//	if credit_no == "" && company_code == "" {
+			//		EsSaveFlag = false
+			//	}
+			//}
+			////qu.Debug("esMap---", esMap)
+			//// qu.Debug("mgoMap---", mgoMap)
+			//// return
 			lock.Lock()
-			if EsSaveFlag {
-				EsSaveCache <- esMap //过滤后数据保存
-			}
+			//if EsSaveFlag {
+			//	EsSaveCache <- esMap //过滤后数据保存
+			//}
 			EsSaveAllCache <- esMap //所有数据保存
 			update = append(update, map[string]interface{}{"$set": mgoMap})
 			if len(update) == 2 {

+ 1 - 1
qyxy_change/qy_baidu/task.go

@@ -18,7 +18,7 @@ func TimeTask() {
 	c := cron.New()
 	cronstrBd := "0 0 */" + fmt.Sprint(BdTaskTime) + " * * ?" 		//每TaskTime小时执行一次
 	//cronstr := "0 0 " + fmt.Sprint(TaskTime) + " * * ?"			//每天TaskTime跑一次
-	cronstrPa := "0 0 15 ? * " + fmt.Sprint(PaTaskTime) 			//凭安增量数据每周跑一次
+	cronstrPa := "0 0 15 ? * " + fmt.Sprint(PaTaskTime) 			//凭安增量数据每周跑一次
 	_ = c.AddFunc(cronstrBd, func() { GetBdData() })
 	_ = c.AddFunc(cronstrPa, func() { GetPaData() })
 	c.Start()

+ 5 - 5
udpcreateindex/src/main.go

@@ -46,8 +46,8 @@ var (
 
 func init() {
 	util.ReadConfig(&Sysconfig)
-	inits()
-	go checkMapJob()
+	//inits()
+	//go checkMapJob()
 	detailLength = util.IntAllDef(Sysconfig["detaillength"], 50000)
 	fileLength = util.IntAllDef(Sysconfig["filelength"], 50000)
 	updport, _ = Sysconfig["updport"].(string)
@@ -65,7 +65,7 @@ func init() {
 		MongodbAddr: mconf["addr"].(string),
 		Size:        util.IntAllDef(mconf["pool"], 5),
 		DbName:      mconf["db"].(string),
-		ReplSet:     "bidding",
+		//ReplSet:     "bidding",
 	}
 	mgo.InitPool()
 
@@ -101,7 +101,7 @@ func init() {
 	}
 	mgostandard = &mongodb.MongodbSim{
 		MongodbAddr: standard["addr"].(string),
-		ReplSet:     "bidding",
+		//ReplSet:     "bidding",
 		Size:        util.IntAllDef(standard["pool"], 5),
 		DbName:      standard["db"].(string),
 	}
@@ -182,7 +182,7 @@ func init() {
 }
 
 func main() {
-	go task_index()
+	//go task_index()
 	updport := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
 	udpclient.Listen(processUdpMsg)

+ 7 - 9
udps/main.go

@@ -10,8 +10,6 @@ import (
 	qutil "qfw/util"
 	"qfw/util/mongodb"
 	"time"
-
-	"gopkg.in/mgo.v2/bson"
 )
 
 var startDate, endDate string
@@ -32,13 +30,13 @@ func main() {
 	flag.StringVar(&q, "q", "", "q查询语句\"{'':''}\",有q就不要gtid,lteid")
 	flag.StringVar(&param, "param", "", "param,生信息发布或其他索引时用双引号套单引号\"{'mgoaddr':'','d':'','c':'','index':'','type':''}\"")
 	flag.Parse()
-	if startDate != "" || endDate != "" {
-		start, _ := time.ParseInLocation(qutil.Date_Short_Layout, startDate, time.Local)
-		end, _ := time.ParseInLocation(qutil.Date_Short_Layout, endDate, time.Local)
-		id1 = qutil.BsonIdToSId(bson.NewObjectIdWithTime(start))
-		id2 = qutil.BsonIdToSId(bson.NewObjectIdWithTime(end))
-		log.Println(id1, id2)
-	}
+	//if startDate != "" || endDate != "" {
+	//	start, _ := time.ParseInLocation(qutil.Date_Short_Layout, startDate, time.Local)
+	//	end, _ := time.ParseInLocation(qutil.Date_Short_Layout, endDate, time.Local)
+	//	id1 = qutil.BsonIdToSId(bson.NewObjectIdWithTime(start))
+	//	id2 = qutil.BsonIdToSId(bson.NewObjectIdWithTime(end))
+	//	log.Println(id1, id2)
+	//}
 	if ip != "" && p > 0 && ((id1 != "" && id2 != "") || (q != "" || tmptime > 0)) {
 		toadd := &net.UDPAddr{
 			IP:   net.ParseIP(ip),

Some files were not shown because too many files changed in this diff