no message

udpcreateindex/src/biddingall.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"regexp"
 	"time"
 	//"fmt"
 	"log"
@@ -20,6 +21,9 @@ import (
 	1、biddingall任务跑历史数据生成索引并更新bidding表
 	2、调用biddingall任务时config.json中indexfields配置要有purchasing、purchasinglist、filetext
 */
+
+var RegSpace = regexp.MustCompile("[\\s\u3000\u2003\u00a0]+")
+
 func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 	defer qutil.Catch()
 	thread := 40
@@ -227,7 +231,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				var cid []string
 				for _, w := range sWinnerarr {
 					if w != "" {
-						ent, _ := mgostandard.FindOne("", map[string]interface{}{"compnay_name": w})
+						ent, _ := mgostandard.FindOne("qyxy_historyname", map[string]interface{}{"company_name": w})
 						if len(*ent) > 0 {
 							cid = append(cid, qutil.ObjToString((*ent)["company_id"]))
 						}
@@ -235,6 +239,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				}
 				if len(cid) > 0 {
 					tmp["entidlist"] = cid
+					update["entidlist"] = cid
 					tmp_up := []map[string]interface{}{}
 					tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
 					tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
@@ -355,12 +360,10 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							}
 						} else if field == "detail" { //过滤
 							detail, _ := tmp[field].(string)
-							if isUtf8([]byte(detail)) || isGBK([]byte(detail)) {
-								if len([]rune(detail)) > detailLength {
-									detail = detail[:detailLength]
-								}
-								newTmp[field] = FilterDetail(detail)
+							if len([]rune(detail)) > detailLength {
+								detail = detail[:detailLength]
 							}
+							newTmp[field] = FilterDetail(detail)
 						} else if field == "_id" || field == "topscopeclass" { //不做处理
 							newTmp[field] = tmp[field]
 						} else if field == "publishtime" || field == "comeintime" {

udpcreateindex/src/biddingindex.go

@@ -127,6 +127,7 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 // }
 
 func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, index, itype, db, c, bkey string) (int, int) {
+	qutil.Debug(len(infos))
 	n1, n2 := 0, 0
 	//线程池
 	UpdatesLock := sync.Mutex{}
@@ -189,6 +190,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		//下面可以多线程跑的--->
 		//处理分类
 		if compare != nil { //extract
+			qutil.Debug("111111111111111111")
 			subscopeclass, _ := compare["subscopeclass"].([]interface{}) //subscopeclass
 			if subscopeclass != nil {
 				//str := ","
@@ -253,10 +255,32 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		if bkey == "" {
 			DealInfo(&tmp, &update)
 		}
+		qutil.Debug(2222222222222222)
 		//同时保存到elastic
 		for tk, tv := range update {
 			tmp[tk] = tv
 		}
+		if tmp["s_winner"] != "" {
+			sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
+			var cid []string
+			for _, w := range sWinnerarr {
+				if w != "" {
+					ent, _ := mgostandard.FindOne("qyxy_historyname", map[string]interface{}{"compnay_name": w})
+					if len(*ent) > 0 {
+						cid = append(cid, qutil.ObjToString((*ent)["company_id"]))
+					}
+				}
+			}
+			qutil.Debug(5555555555555)
+			if len(cid) > 0 {
+				tmp["entidlist"] = cid
+				update["entidlist"] = cid
+				tmp_up := []map[string]interface{}{}
+				tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
+				tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
+				UpdataMgoCache <- tmp_up
+			}
+		}
 		//对projectscope字段的索引处理
 		ps, _ := tmp["projectscope"].(string)
 		if len(ps) > ESLEN {
@@ -291,6 +315,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		//		}
 		go IS.Add("bidding")
 		if qutil.IntAll(update["extracttype"]) != -1 {
+			qutil.Debug(44444444444444)
 			newTmp := map[string]interface{}{}                //最终生索引的数据
 			for field, ftype := range biddingIndexFieldsMap { //
 				if tmp[field] != nil { //
@@ -365,7 +390,6 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 						}
 					} else if field == "detail" { //过滤
 						detail, _ := tmp[field].(string)
-
 						if len([]rune(detail)) > detailLength {
 							detail = detail[:detailLength]
 						}
@@ -476,6 +500,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 			// 	}
 			// }
 			arrEs = append(arrEs, newTmp)
+			qutil.Debug(newTmp)
 		}
 		if len(update) > 0 {
 			delete(update, "winnerorder") //winnerorder不需要更新到bindding表，删除

udpcreateindex/src/config.json

@@ -38,14 +38,14 @@
     "collect": "bidding",
     "index": "bidding",
     "type": "bidding",
-    "extractdb": "mxs",
+    "extractdb": "wjh",
     "extractcollect": "extract",
     "indexfields": [
       "buyerzipcode", "winnertel", "winnerperson", "contractcode", "winneraddr", "agencyaddr", "buyeraddr", "signaturedate", "projectperiod", "projectaddr", "agencytel", "agencyperson",
       "buyerperson", "agency", "projectscope", "projectcode", "bidopentime", "supervisorrate", "buyertel", "bidamount", "winner", "buyer", "budget", "projectname", "bidstatus", "buyerclass",
       "topscopeclass", "s_topscopeclass", "s_subscopeclass", "area", "city", "district", "s_winner", "_id", "title", "detail", "site", "comeintime", "href", "infoformat", "publishtime",
       "s_sha", "spidercode", "subtype", "toptype", "projectinfo", "purchasing", "purchasinglist", "filetext", "channel", "winnerorder", "project_scale", "project_duration", "project_timeunit",
-      "project_startdate", "project_completedate", "payway", "contract_guarantee", "bid_guarantee", "qualifies"
+      "project_startdate", "project_completedate", "payway", "contract_guarantee", "bid_guarantee", "qualifies", "entidlist"
     ],
     "indexfieldsmap": {
       "buyerzipcode": "string", "winnertel": "string", "winnerperson": "string", "contractcode": "string", "winneraddr": "string", "agencyaddr": "string", "buyeraddr": "string", "signaturedate": "int64",
@@ -54,9 +54,9 @@
       "buyerclass": "string", "topscopeclass": "", "s_topscopeclass": "string", "s_subscopeclass": "string", "area": "string", "city": "string", "district": "string", "s_winner": "string", "_id": "", "title": "string",
       "detail": "string", "site": "string", "comeintime": "int64", "href": "string", "infoformat": "int", "publishtime": "int64", "s_sha": "string", "spidercode": "string", "subtype": "string", "toptype": "string",
       "projectinfo": "", "purchasing": "string", "purchasinglist": "", "filetext": "string", "channel": "string", "winnerorder": "", "project_scale": "string", "project_duration": "float64", "project_timeunit": "string",
-      "project_startdate": "int64", "project_completedate": "int64", "payway": "string", "contract_guarantee": "bool", "bid_guarantee": "bool", "qualifies": ""
+      "project_startdate": "int64", "project_completedate": "int64", "payway": "string", "contract_guarantee": "bool", "bid_guarantee": "bool", "qualifies": "", "entidlist": "[]string"
     },
-    "fields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,s_topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass,dataging,winnerorder",
+    "fields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,s_topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass,dataging,winnerorder,project_scale,project_duration,project_timeunit,project_startdate,project_completedate, payway,contract_guarantee,bid_guarantee,qualifies",
     "projectinfo": "approvecode,approvecontent,approvestatus,approvetime,approvedept,approvenumber,projecttype,approvecity",
     "projectinfomap": {
       "approvecode": "string", "approvecontent": "string", "approvestatus": "string", "approvetime": "string", "approvedept": "string", "approvenumber": "string", "projecttype": "string", "approvecity": "string"

udpcreateindex/src/main.go

@@ -48,8 +48,8 @@ var SP = make(chan bool, 5)
 
 func init() {
 	util.ReadConfig(&Sysconfig)
-	//inits()
-	//go checkMapJob()
+	inits()
+	go checkMapJob()
 	detailLength = util.IntAllDef(Sysconfig["detaillength"], 50000)
 	fileLength = util.IntAllDef(Sysconfig["filelength"], 50000)
 	updport, _ = Sysconfig["updport"].(string)
@@ -67,7 +67,7 @@ func init() {
 		MongodbAddr: mconf["addr"].(string),
 		Size:        util.IntAllDef(mconf["pool"], 5),
 		DbName:      mconf["db"].(string),
-		//ReplSet:     "bidding",
+		ReplSet:     "bidding",
 	}
 	mgo.InitPool()
 	project2db = &mongodb.MongodbSim{
@@ -102,7 +102,7 @@ func init() {
 	}
 	mgostandard = &mongodb.MongodbSim{
 		MongodbAddr: standard["addr"].(string),
-		//ReplSet:     "bidding",
+		ReplSet:     "bidding",
 		Size:        util.IntAllDef(standard["pool"], 5),
 		DbName:      standard["db"].(string),
 	}

udpcreateindex/src/util/ossclient.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	ossEndpoint        = "http://oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
+	ossEndpoint        = "oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
 	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
 	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
 	ossBucketName      = "topjy"