xg

fullproject/src_v1/config.json

@@ -16,6 +16,11 @@
         "to": "wangjianghan@topnet.net.cn",
         "api": "http://172.17.145.179:19281/_send/_mail"
     },
+    "redis": {
+        "dbname": "",
+        "addr": "",
+        "db": ""
+    },
     "bidding": {
         "addr": "192.168.3.207:27092",
         "dbname": "qfw",

fullproject/src_v1/init.go

@@ -95,6 +95,7 @@ func init() {
 	for _, v := range BlackList {
 		BlaskListMap[util.ObjToString(v)] = true
 	}
+
 	initWinnerRegexp()
 	initBuyerRegexp()
 	initAgencyRegexp()

fullproject/src_v1/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"log"
 	mu "mfw/util"
 	"net"
@@ -9,6 +10,7 @@ import (
 	"os/signal"
 	"qfw/util"
 	"qfw/util/elastic"
+	"qfw/util/redis"
 	"syscall"
 	"time"
 )
@@ -22,6 +24,7 @@ var (
 	Es           *elastic.Elastic
 	Index        string
 	Itype        string
+	RedisCode    string
 
 	sid, eid string //测试使用
 )
@@ -44,6 +47,13 @@ func init() {
 	Index = util.ObjToString(es["index"])
 	Itype = util.ObjToString(es["itype"])
 	Es.InitElasticSize()
+
+	redisCon := Sysconfig["redis"].(map[string]interface{})
+	RedisCode = util.ObjToString(redisCon["dbname"])
+	redis_addr := util.ObjToString(redisCon["addr"])
+	redis_db := util.IntAll(redisCon["db"])
+	//"qyxy_buyer=172.17.4.189:8379"
+	redis.InitRedis1(fmt.Sprintf("%s=%s", RedisCode, redis_addr), redis_db) // 采购单位与中标单位初次合作项目
 }
 
 var queryClose = make(chan bool)

fullproject/src_v1/mgotool.go

@@ -292,4 +292,3 @@ func StringTOBsonId(id string) primitive.ObjectID {
 	objectId, _ := primitive.ObjectIDFromHex(id)
 	return objectId
 }
-

fullproject/src_v1/project.go

@@ -2,10 +2,12 @@ package main
 
 import (
 	"encoding/json"
+	"jy/util"
 	"log"
 	"math"
 	"mongodb"
 	qu "qfw/util"
+	"qfw/util/redis"
 	"sort"
 	"strings"
 	"time"
@@ -597,6 +599,11 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	if len(p1.EntIdList) > 0 {
 		set["entidlist"] = p1.EntIdList
 	}
+	// first_cooperation
+	if p1.Buyer != "" && len(thisinfo.Winners) > 0 {
+		FirstCooperation(set, p1.Buyer, thisinfo.Winners, thisinfo.EntIdList)
+	}
+
 	p1.InfoFiled = make(map[string]InfoField)
 	infofield := InfoField{
 		Budget:       thisinfo.Budget,
@@ -824,8 +831,10 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	if thisinfo.Area != "全国" {
 		pInfo.Area = thisinfo.Area
 		set["area"] = thisinfo.Area
-		pInfo.City = thisinfo.City
-		set["city"] = thisinfo.City
+		if thisinfo.City != "" {
+			pInfo.City = thisinfo.City
+			set["city"] = thisinfo.City
+		}
 		if thisinfo.District != "" {
 			pInfo.District = thisinfo.District
 			set["district"] = thisinfo.District
@@ -952,7 +961,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		if len(pInfo.Winners) <= 0 {
 			set["winner"] = qu.ObjToString(tmp["winner"])
 		}
-		sort.Strings(pInfo.Winners)
+		//sort.Strings(pInfo.Winners)
 		for _, k := range thisinfo.Winners {
 			if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
 				if BinarySearch(pInfo.Winners, k) != -1 {
@@ -966,7 +975,8 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 				}
 			}
 		}
-		set["s_winner"] = strings.Join(pInfo.Winners, ",")
+
+		set["s_winner"] = strings.Join(qu.ObjArrToStringArr(Duplicate(pInfo.Winners)), ",")
 	}
 
 	if len(thisinfo.EntIdList) > 0 {
@@ -977,6 +987,10 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		}
 		set["entidlist"] = pInfo.EntIdList
 	}
+	// first_cooperation
+	if pInfo.Buyer != "" && len(pInfo.Winners) > 0 {
+		FirstCooperation(set, pInfo.Buyer, pInfo.Winners, pInfo.EntIdList)
+	}
 
 	//项目规模
 	if len(thisinfo.ProjectScale) > 0 {
@@ -1453,5 +1467,28 @@ func (p *ProjectTask) GetBidTypeAndBidStatus(info *Info) (string, string) {
 		}
 	}
 	return typeStr, statusStr
+}
 
+func FirstCooperation(set map[string]interface{}, b string, winns, entidlist []string) {
+	defer func() {
+		// 处理数组越界异常
+		if r := recover(); r != nil {
+			util.Debug("recover...:", r)
+		}
+	}()
+	pid := mongodb.BsonIdToSId(set["_id"])
+	var eids []string
+	for i, eid := range entidlist {
+		if eid != "-" {
+			text := b + "," + winns[i]
+			ex, _ := redis.Exists(RedisCode, text)
+			if !ex {
+				redis.PutCKV(RedisCode, text, pid)
+				eids = append(eids, eid)
+			}
+		}
+	}
+	if len(eids) > 0 {
+		set["first_cooperation"] = entidlist
+	}
 }

fullproject/src_v1/task.go

@@ -524,7 +524,7 @@ L:
 						log.Println("current", count, lastid)
 					}
 				} else {
-					if count%1000 == 0 {
+					if count%2000 == 0 {
 						log.Println("current", count, lastid)
 					}
 				}

udpcreateindex/src/biddingall.go

@@ -207,17 +207,20 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				//				if winner != "" {
 				//					m1[winner] = true
 				//				}
-				//				package1 := compare["package"]
-				//				if package1 != nil {
-				//					packageM, _ := package1.(map[string]interface{})
-				//					for _, p := range packageM {
-				//						pm, _ := p.(map[string]interface{})
-				//						pw, _ := pm["winner"].(string)
-				//						if pw != "" {
-				//							m1[pw] = true
-				//						}
-				//					}
-				//				}
+				// 多包标记
+				if package1 := compare["package"]; package1 != nil {
+					packageM, _ := package1.(map[string]interface{})
+					for _, p := range packageM {
+						pm, _ := p.(map[string]interface{})
+						if qutil.ObjToString(pm["winner"]) != "" || qutil.Float64All(pm["budget"]) > 0 ||
+							qutil.Float64All(pm["bidamount"]) > 0 {
+							update["multipackage"] = int32(1)
+							break
+						}
+					}
+				} else {
+					update["multipackage"] = int32(0)
+				}
 				compare = nil
 				//				if len(m1) > 0 {
 				//					//str := ","
@@ -246,6 +249,8 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 			for tk, tv := range update {
 				tmp[tk] = tv
 			}
+			tmp_up := []map[string]interface{}{}
+			extupdate := make(map[string]interface{})
 			if tmp["s_winner"] != "" {
 				//sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
 				var cid []string
@@ -272,12 +277,18 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				if len(cid) > 0 {
 					tmp["entidlist"] = cid
 					update["entidlist"] = cid
-					tmp_up := []map[string]interface{}{}
-					tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
-					tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
-					UpdataMgoCache <- tmp_up
+					extupdate["entidlist"] = cid
 				}
 			}
+			// 临时
+			if tmp["purchasing_tag"] != nil {
+				extupdate["purchasing_tag"] = tmp["purchasing_tag"]
+			}
+			if len(extupdate) > 0 {
+				tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
+				tmp_up = append(tmp_up, map[string]interface{}{"$set": extupdate})
+				UpdataMgoCache <- tmp_up
+			}
 			//对projectscope字段的索引处理
 			ps, _ := tmp["projectscope"].(string)
 			if len(ps) > ESLEN {
@@ -401,7 +412,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 						} else if field == "bidopentime" {
 							if tmp[field] != nil && tmp["bidendtime"] == nil {
 								newTmp["bidendtime"] = tmp[field]
-								newTmp[field] = tmp["bidendtime"]
+								newTmp[field] = tmp[field]
 							} else if tmp[field] == nil && tmp["bidendtime"] != nil {
 								newTmp["bidendtime"] = tmp[field]
 								newTmp[field] = tmp["bidendtime"]
@@ -425,6 +436,9 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							}
 						} else if field == "s" {
 							newTmp[field] = tmp[field]
+						} else if field == "purchasing_tag" {
+							// 临时 purchasing_tag bidding表有
+							newTmp[field] = tmp[field]
 						} else { //其它字段判断数据类型，不正确舍弃
 							if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
 								continue

udpcreateindex/src/biddingindex.go

@@ -207,6 +207,19 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 				}
 				update["s_topscopeclass"] = strings.Join(newclass, ",")
 			}
+			if package1 := compare["package"]; package1 != nil {
+				packageM, _ := package1.(map[string]interface{})
+				for _, p := range packageM {
+					pm, _ := p.(map[string]interface{})
+					if qutil.ObjToString(pm["winner"]) != "" || qutil.Float64All(pm["budget"]) > 0 ||
+						qutil.Float64All(pm["bidamount"]) > 0 {
+						update["multipackage"] = 1
+						break
+					}
+				}
+			} else {
+				update["multipackage"] = 0
+			}
 			compare = nil
 		} else {
 			area := qutil.ObjToString(tmp["area"])

udpcreateindex/src/config.json

@@ -48,7 +48,7 @@
       "topscopeclass", "s_topscopeclass", "s_subscopeclass", "area", "city", "district", "s_winner", "_id", "title", "detail", "site", "comeintime", "href", "infoformat", "publishtime",
       "s_sha", "spidercode", "subtype", "toptype", "projectinfo", "purchasing", "purchasinglist", "filetext", "channel", "winnerorder", "project_scale", "project_duration", "project_timeunit",
       "project_startdate", "project_completedate", "payway", "contract_guarantee", "bid_guarantee", "qualifies", "entidlist", "funds", "review_experts", "bidmethod", "bidendtime", "bidopenaddress",
-      "docamount", "agencyrate", "agencyfee", "getdocmethod", "china_bidding", "bidway"
+      "docamount", "agencyrate", "agencyfee", "getdocmethod", "china_bidding", "bidway", "purchasing_tag", "multipackage"
     ],
     "indexfieldsmap": {
       "buyerzipcode": "string", "winnertel": "string", "winnerperson": "string", "contractcode": "string", "winneraddr": "string", "agencyaddr": "string", "buyeraddr": "string", "signaturedate": "int64",
@@ -59,9 +59,9 @@
       "projectinfo": "", "purchasing": "string", "purchasinglist": "", "filetext": "string", "channel": "string", "winnerorder": "", "project_scale": "string", "project_duration": "int32", "project_timeunit": "string",
       "project_startdate": "int64", "project_completedate": "int64", "payway": "string", "contract_guarantee": "bool", "bid_guarantee": "bool", "qualifies": "", "entidlist": "[]string", "funds": "string",
       "review_experts": "", "bidmethod": "string", "bidendtime": "int64", "bidopenaddress": "string", "docamount": "float64", "agencyrate": "float64", "agencyfee": "float64", "bidway": "string",
-      "getdocmethod": "string", "china_bidding": "string"
+      "getdocmethod": "string", "china_bidding": "string", "purchasing_tag": "string", "multipackage": "int32"
     },
-    "fields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,s_topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass,dataging,winnerorder,project_scale,project_duration,project_timeunit,project_startdate,project_completedate, payway,contract_guarantee,bid_guarantee,qualifies,funds,review_experts,bidmethod,bidendtime,bidopenaddress,docamount,bidway,agencyrate,agencyfee,getdocmethod",
+    "fields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,s_topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass,dataging,winnerorder,project_scale,project_duration,project_timeunit,project_startdate,project_completedate, payway,contract_guarantee,bid_guarantee,qualifies,funds,review_experts,bidmethod,bidendtime,bidopenaddress,docamount,bidway,agencyrate,agencyfee,getdocmethod,purchasing_tag",
     "projectinfo": "approvecode,approvecontent,approvestatus,approvetime,approvedept,approvenumber,projecttype,approvecity",
     "projectinfomap": {
       "approvecode": "string", "approvecontent": "string", "approvestatus": "string", "approvetime": "string", "approvedept": "string", "approvenumber": "string", "projecttype": "string", "approvecity": "string"
@@ -126,11 +126,11 @@
     "pool": 12,
     "node": "4q7v7e6mQ5aeCwjUgM6HcA"
   },
-  "filter-keyword": ["(招标网|千里马|采招网|招标采购导航网|招标与采购网|中国招投标网|中国采购与招标网|优质采)[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}(qianlima|zhaobiao|okcis|zbytb|infobidding|bidcenter|youzhicai|chinabidding)[a-z0-9.\\/\\/]{0,40}",
+  "filter-keyword": ["(招标网|千里马|采招网|招标采购导航网|招标与采购网|中国招投标网|中国采购与招标网|中国采购与招标|优质采)[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}(qianlima|zhaobiao|okcis|zbytb|infobidding|bidcenter|youzhicai|chinabidding|Chinabidding｜CHINABIDDING)[a-z0-9.\\/\\/]{0,40}",
     "招标网[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}zhaobiao[a-z0-9.\\/\\/]{0,40}",
     "千里马[\\w\\W]{0,15}[a-z0-9:\\/\\/.]{0,20}qianlima[a-z0-9.\\/\\/]{0,10}",
-    "[\\(（]?(网址)?[:：；;]?(http|https|htpps)*[:：]?(//)?www.(zhaobiao|chinabidding|infobidding|zbytb|okcis|qianlima|youzhicai).(com|cn)+/?[\\)）]?",
-    "[\\(（]?(网址)?[:：][http|https|htpps]+([:：])//www.bidcenter.com.cn/?(qita/webseo80.shtml)?[\\)）]?",
-    "千里马(平台|网站)+", "[“\"]?优质采(电子交易平台|云采购平台|交易平台)?[”\"]?", "《?(中国采购与|中国)?招(投)?标(与采购|采购导航)?网》?",
-    "《?元博网(采购与招标网)?》?", "《?(中国)?招标采购导航网》?", "中国采招\\W*网[(（](bidcenter)?.*[)）]"]
+    "[\\(（]?(网址)?[:：；;]?(http|https|htpps)*[:：]?(\\/\\/)?(www|jinan|WWW)?.(zhaobiao|chinabidding|Chinabidding|CHINABIDDING|infobidding|zbytb|okcis|qianlima|youzhicai).(com|cn|COM|CN)?(.cn|.CN)?\\/?[\\)）]?",
+    "[\\(（]?(网址)?(:：)?(http|https|htpps)*(:|：)?\\/\\/www.bidcenter.com.cn\\/",
+      "千里马(平台|网站)+", "[“\"]?优质采(平台|电子交易平台|云采购平台|交易平台)?[”\"]?", "《?(中国采购与|中国)?招(投)?标(与采购|采购导航)?网》?",
+    "《?元博网(采购与招标网)?》?", "《?(中国)?招标采购导航网》?", "中\\W{0,3}国采\\W{0,3}招\\W{0,3}网\\W*[(（]?(bidcenter.com.cn)?[)）]?", "已方宝", "中国招标与采购"]
 }

udpcreateindex/src/main.go

@@ -12,6 +12,7 @@ import (
 	_ "net/http/pprof"
 	"qfw/util"
 	elastic "qfw/util/elastic"
+	"qfw/util/redis"
 	"strings"
 	"time"
 	u "util"
@@ -77,8 +78,8 @@ var StopFlag = false // 程序生索引停止标志
 func init() {
 	util.ReadConfig(&Sysconfig)
 	// company_id
-	//redis.InitRedis1("qyxy_id=172.17.4.189:8379", 4)
-	//inits()
+	redis.InitRedis1("qyxy_id=172.17.4.189:8379", 4)
+	inits()
 	//go checkMapJob()
 	detailLength = util.IntAllDef(Sysconfig["detaillength"], 50000)
 	fileLength = util.IntAllDef(Sysconfig["filelength"], 50000)