xg

fullproject/src_v1/config.json

@@ -4,7 +4,7 @@
     "statusdays": 15,
 	"mongodbServers": "192.168.3.207:27092",
     "mongodbPoolSize": 10,
-    "mongodbName": "qfw_data",
+    "mongodbName": "wjh",
 	"hints":"publishtime_1",
     "extractColl": "bidding",
     "extractColl1": "bidding",

fullproject/src_v1/load_data.go

@@ -35,7 +35,7 @@ func (p *ProjectTask) loadData(starttime int64) {
 					id := tmp.Id.Hex()
 					for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) {
 						if v != "" {
-							v = pcReplace.ReplaceAllString(v, "")
+							//v = pcReplace.ReplaceAllString(v, "")
 							if v != "" {
 								k := p.mapPn[v]
 								if k == nil {
@@ -49,7 +49,7 @@ func (p *ProjectTask) loadData(starttime int64) {
 					}
 					for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) {
 						if v != "" {
-							v = pcReplace.ReplaceAllString(v, "")
+							//v = pcReplace.ReplaceAllString(v, "")
 							if v != "" {
 								k := p.mapPc[v]
 								if k == nil {

fullproject/src_v1/main.go

@@ -70,7 +70,7 @@ func DealSign() {
 	}
 }
 
-func main() {
+func mainT() {
 	//udp跑增量  id段   project
 	//udp跑全量			qlT
 	//udp跑历史数据  信息id1,id2/或id段  ls
@@ -90,9 +90,9 @@ func main() {
 }
 
 //测试组人员使用
-func mainT() {
-	sid = "600f86a17cdc9beb563d9e33"
-	eid = "600f86a17cdc9beb563d9e34"
+func main() {
+	sid = "6062826adeed5af79ca930c9"
+	eid = "6062826adeed5af79ca930ca"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()

fullproject/src_v1/project.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"log"
 	"math"
+	"mongodb"
 	qu "qfw/util"
 	"sort"
 	"strings"
@@ -95,7 +96,7 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 	p.findLock.Lock()
 	defer p.findLock.Unlock()
 	// 3.18 isfow=0数据不参与项目合并
-	code := qu.ObjToString(tmp["spidercode"])
+	code := strings.ReplaceAll(qu.ObjToString(tmp["spidercode"]), " ", "")
 	p.mapSpiderLock.Lock()
 	isflow := p.mapSpider[code]
 	p.mapSpiderLock.Unlock()
@@ -420,28 +421,10 @@ func compareResult(resVal, pjVal, score2 int, comStr, compareBuyer, compareCity,
 	return eqV
 }
 
-//项目中的字段
-var FIELDS = []string{
-	"area",
-	"city",
-	"district",
-	"projectname",
-	"projectcode",
-	"buyer",
-	"buyerclass",
-	"buyerperson",
-	"buyertel",
-	"winner",
-	"agency",
-	"topscopeclass",
-	"subscopeclass",
-	"package",
-}
-
 //招标时间zbtime、中标时间jgtime、项目状态bidstatus、招标类型bidtype、最后发布时间lasttime、首次发布时间firsttime
-
 func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *ProjectInfo) {
-	pId := primitive.NewObjectID() //NewObjectId()
+	//pId := primitive.NewObjectID() //NewObjectId()
+	pId := mongodb.StringTOBsonId(thisinfo.Id)
 	set := map[string]interface{}{}
 	set["_id"] = pId
 	for _, f := range FIELDS {
@@ -515,10 +498,10 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 	set["ids"] = []string{thisinfo.Id}
 	if thisinfo.TopType == "招标" {
-		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
+		//if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
 			set["zbtime"] = tmp["publishtime"]
 			p1.Zbtime = qu.Int64All(tmp["publishtime"])
-		}
+		//}
 	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
 		set["jgtime"] = tmp["publishtime"]
 		p1.Jgtime = thisinfo.Publishtime
@@ -669,49 +652,6 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	return pId.Hex(), &p1
 }
 
-//招标信息字段
-var INFOFIELDS = []string{
-	"projectname",
-	"projectcode",
-	"projectscope",
-	"contractcode",
-	"title",
-	"href",
-	"publishtime",
-	"comeintime",
-	"bidopentime",
-	"toptype",
-	"subtype",
-	"buyer",
-	"buyerclass",
-	"agency",
-	"winner",
-	"budget",
-	"bidamount",
-	"topscopeclass",
-	"subscopclass",
-	"infoformat",
-	"buyerperson",
-	"buyertel",
-	"area",
-	"city",
-	"district",
-	"spidercode",
-	"site",
-	"review_experts",
-	"purchasing",
-	"project_scale",
-	"project_duration",
-	"project_timeunit",
-	"project_startdate",
-	"project_completedate",
-	"payway",
-	"contract_guarantee",
-	"bid_guarantee",
-	"qualifies",
-	"entidlist",
-}
-
 //项目中list的信息
 func (p *ProjectTask) PushListInfo(tmp map[string]interface{}, infoid string) bson.M {
 	res := bson.M{
@@ -761,7 +701,7 @@ func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidty
 	return p1
 }
 
-//更新项目
+//更新项目	全量合并
 func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string, ex int) {
 	if p.currentType != "updateInfo" {
 		if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
@@ -780,7 +720,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	pInfo.LastTime = thisinfo.Publishtime
 	set["lasttime"] = thisinfo.Publishtime
 	if thisinfo.TopType == "招标" {
-		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" && pInfo.Zbtime <= 0 {
+		if pInfo.Zbtime <= 0 {
 			set["zbtime"] = tmp["publishtime"]
 		}
 		if pInfo.Jgtime > 0 {
@@ -790,16 +730,16 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	} else if thisinfo.TopType == "结果" {
 		if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
 			if pInfo.Jgtime > 0 {
-				jg1 := int64(math.Abs(float64(pInfo.Jgtime - thisinfo.Publishtime)))
+				//jg1 := int64(math.Abs(float64(pInfo.Jgtime - thisinfo.Publishtime)))
 				//公告状态和项目状态同样都是中标或者成交，
-				if (thisinfo.SubType == "中标" || thisinfo.SubType == "成交") && (pInfo.Bidstatus == "中标" || pInfo.Bidstatus == "成交") {
-					if jg1 > p.jgTime {
+				if thisinfo.SubType == "成交" && pInfo.Bidstatus == "中标" {
+					if p.jgTime < thisinfo.Publishtime {
 						set["jgtime"] = tmp["publishtime"]
 						pInfo.Jgtime = thisinfo.Publishtime
 					}
 					//公告状态和项目状态同样是流标或者废标
 				} else if (thisinfo.SubType == "流标" || thisinfo.SubType == "废标") && (pInfo.Bidstatus == "流标" || pInfo.Bidstatus == "废标") {
-					if jg1 > p.jgTime {
+					if p.jgTime < thisinfo.Publishtime {
 						set["jgtime"] = tmp["publishtime"]
 						pInfo.Jgtime = thisinfo.Publishtime
 					}
@@ -933,7 +873,12 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		if pInfo.ContractCode == "" {
 			set["contractcode"] = thisinfo.ContractCode
 		}else {
-			set["contractcode"] = pInfo.ContractCode + "," + thisinfo.ContractCode
+			list := strings.Split(pInfo.ContractCode, ",")
+			if BinarySearch(list, thisinfo.ContractCode) == -1 {
+				list = append(list, thisinfo.ContractCode)
+				sort.Strings(list)
+			}
+			set["contractcode"] = strings.Join(list, ",")
 		}
 	}
 
@@ -1179,6 +1124,10 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 }
 
+func (p *ProjectTask) x() {
+	
+}
+
 /**
  *	更新项目时，项目状态的处理
  *	返回是否新增项目，异常标记
@@ -1243,18 +1192,6 @@ func ComparePlace(project *ProjectInfo, info *Info) bool {
 	return true
 }
 
-var PackageEle = []string{
-	"origin",
-	"name",
-	//"text",
-	"budget",
-	"winner",
-	"bidamount",
-	"bidamounttype",
-	"currency",
-	"bidstatus",
-}
-
 func packageEle(map1 map[string]interface{}, id string) map[string]interface{} {
 	p2 := map[string]interface{}{}
 	for _, k := range PackageEle {
@@ -1494,24 +1431,6 @@ func IsCreatePro(info *Info) (bol bool) {
 	return bol
 }
 
-var bidtype = map[string]string{
-	"招标": "招标",
-	"邀标": "邀标",
-	"询价": "询价",
-	"单一": "单一",
-	"竞价": "竞价",
-	"竞谈": "竞谈",
-}
-
-var bidstatus = map[string]string{
-	"预告": "预告",
-	"中标": "中标",
-	"成交": "成交",
-	"废标": "废标",
-	"流标": "流标",
-	"合同": "合同",
-}
-
 func (p *ProjectTask) GetBidTypeAndBidStatus(info *Info) (string, string) {
 	p.mapBidLock.Lock()
 	defer p.mapBidLock.Unlock()

fullproject/src_v1/project_tool.go

@@ -0,0 +1,94 @@
+package main
+
+//项目中的字段
+var FIELDS = []string{
+	"area",
+	"city",
+	"district",
+	"projectname",
+	"projectcode",
+	"buyer",
+	"buyerclass",
+	"buyerperson",
+	"buyertel",
+	"winner",
+	"agency",
+	"topscopeclass",
+	"subscopeclass",
+	"package",
+}
+
+//招标信息字段
+var INFOFIELDS = []string{
+	"projectname",
+	"projectcode",
+	"projectscope",
+	"contractcode",
+	"title",
+	"href",
+	"publishtime",
+	"comeintime",
+	"bidopentime",
+	"toptype",
+	"subtype",
+	"buyer",
+	"buyerclass",
+	"agency",
+	"winner",
+	"budget",
+	"bidamount",
+	"topscopeclass",
+	"subscopclass",
+	"infoformat",
+	"buyerperson",
+	"buyertel",
+	"area",
+	"city",
+	"district",
+	"spidercode",
+	"site",
+	"review_experts",
+	"purchasing",
+	"project_scale",
+	"project_duration",
+	"project_timeunit",
+	"project_startdate",
+	"project_completedate",
+	"payway",
+	"contract_guarantee",
+	"bid_guarantee",
+	"qualifies",
+	"entidlist",
+}
+
+// 包信息
+var PackageEle = []string{
+	"origin",
+	"name",
+	//"text",
+	"budget",
+	"winner",
+	"bidamount",
+	"bidamounttype",
+	"currency",
+	"bidstatus",
+}
+
+var bidstatus = map[string]string{
+	"预告": "预告",
+	"中标": "中标",
+	"成交": "成交",
+	"废标": "废标",
+	"流标": "流标",
+	"合同": "合同",
+}
+
+var bidtype = map[string]string{
+	"招标": "招标",
+	"邀标": "邀标",
+	"询价": "询价",
+	"单一": "单一",
+	"竞价": "竞价",
+	"竞谈": "竞谈",
+}
+

fullproject/src_v1/task.go

@@ -472,6 +472,8 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 					}()
 					p.fillInPlace(tmp)
 					info := ParseInfo(tmp)
+					util.Debug(tmp["projectname"])
+					util.Debug(info.ProjectName)
 					p.currentTime = info.Publishtime
 					//普通合并
 					p.CommonMerge(tmp, info)
@@ -517,16 +519,18 @@ L:
 					}
 				}
 				if util.IntAll(tmp["repeat"]) == 0 {
-					if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
+					if P_QL.currentType == "ql" {
+						infoPool <- tmp
+					}else if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
 						infoPool <- tmp
 					}else {
 						util.Debug("增量   dataging == 1 ", tmp["_id"])
 					}
 				}else {
 					countRepeat++
-					if P_QL.currentType == "project" {
-						util.Debug("repeat err---", tmp["_id"])
-					}
+					//if P_QL.currentType == "project" {
+					//	util.Debug("repeat err---", tmp["_id"])
+					//}
 				}
 				count++
 			} else {
@@ -615,20 +619,20 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	}
 
 	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
-		thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
-		if thisinfo.ProjectName != "" {
+		//thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
+		//if thisinfo.ProjectName != "" {
 			thisinfo.pnbval++
-		}
+		//}
 	}
 
 	if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
 		if thisinfo.ProjectCode != "" {
-			thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
+			//thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
 			if thisinfo.pnbval == 0 && len([]rune(thisinfo.ProjectCode)) < 5 {
 				thisinfo.ProjectCode = StrOrNum.ReplaceAllString(thisinfo.ProjectCode, "")
 			}
 		} else {
-			thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
+			//thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
 			if thisinfo.pnbval == 0 && len([]rune(thisinfo.PTC)) < 5 {
 				thisinfo.PTC = StrOrNum.ReplaceAllString(thisinfo.PTC, "")
 			}

monitor/config.json

@@ -20,6 +20,6 @@
   },
   "jkmail": {
     "to": "wangjianghan@topnet.net.cn",
-    "api": "http://10.171.112.160:19281/_send/_mail"
+    "api": "http://172.17.145.179:19281/_send/_mail"
   }
 }

qyxy_change/qy_baidu/task.go

@@ -24,7 +24,7 @@ func TimeTask() {
 	c.Start()
 }
 
-func GetBdData()  {
+func GetBdData() {
 	count := 0
 	lastid := ""
 	sess := MgoBd.GetMgoConn()

udpcreateindex/src/biddingall.go

@@ -232,26 +232,26 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 			for tk, tv := range update {
 				tmp[tk] = tv
 			}
-			if tmp["s_winner"] != "" {
-				sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
-				var cid []string
-				for _, w := range sWinnerarr {
-					if w != "" {
-						ent, _ := mgostandard.FindOne("qyxy_historyname", map[string]interface{}{"company_name": w})
-						if len(*ent) > 0 {
-							cid = append(cid, qutil.ObjToString((*ent)["company_id"]))
-						}
-					}
-				}
-				if len(cid) > 0 {
-					tmp["entidlist"] = cid
-					update["entidlist"] = cid
-					tmp_up := []map[string]interface{}{}
-					tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
-					tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
-					UpdataMgoCache <- tmp_up
-				}
-			}
+			//if tmp["s_winner"] != "" {
+			//	sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
+			//	var cid []string
+			//	for _, w := range sWinnerarr {
+			//		if w != "" {
+			//			ent, _ := mgostandard.FindOne("qyxy_historyname", map[string]interface{}{"company_name": w})
+			//			if len(*ent) > 0 {
+			//				cid = append(cid, qutil.ObjToString((*ent)["company_id"]))
+			//			}
+			//		}
+			//	}
+			//	if len(cid) > 0 {
+			//		tmp["entidlist"] = cid
+			//		update["entidlist"] = cid
+			//		tmp_up := []map[string]interface{}{}
+			//		tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
+			//		tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
+			//		UpdataMgoCache <- tmp_up
+			//	}
+			//}
 			//对projectscope字段的索引处理
 			ps, _ := tmp["projectscope"].(string)
 			if len(ps) > ESLEN {
@@ -377,7 +377,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							if tmp[field] != nil && qutil.Int64All(tmp[field]) > 0 {
 								newTmp[field] = qutil.Int64All(tmp[field])
 							}
-						} else if field == "entidlist" {
+						} else if field == "s" {
 							newTmp[field] = tmp[field]
 						} else { //其它字段判断数据类型，不正确舍弃
 							if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {

udpcreateindex/src/biddingindex.go

@@ -54,7 +54,6 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 
 	//bidding库
 	session := mgo.GetMgoConn()
-	qutil.Debug(db, c)
 	count, _ := session.DB(db).C(c).Find(&q).Count()
 	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
 	n1, n2 := 0, 0

udpcreateindex/src/config.json

@@ -120,7 +120,7 @@
     }
   },
   "elastic": {
-    "addr": "http://192.168.3.128:9800",
+    "addr": "http://127.0.0.1:9800",
     "index": "bidding",
     "itype": "bidding",
     "pool": 12

udpcreateindex/src/main.go

@@ -9,8 +9,6 @@ import (
 	"qfw/util"
 	elastic "qfw/util/elastic"
 	"strings"
-
-	//"time"
 	u "util"
 )
 
@@ -49,7 +47,7 @@ var SP = make(chan bool, 5)
 func init() {
 	util.ReadConfig(&Sysconfig)
 	//inits()
-	go checkMapJob()
+	//go checkMapJob()
 	detailLength = util.IntAllDef(Sysconfig["detaillength"], 50000)
 	fileLength = util.IntAllDef(Sysconfig["filelength"], 50000)
 	updport, _ = Sysconfig["updport"].(string)
@@ -67,9 +65,9 @@ func init() {
 		MongodbAddr: mconf["addr"].(string),
 		Size:        util.IntAllDef(mconf["pool"], 5),
 		DbName:      mconf["db"].(string),
-		//UserName:	 Sysconfig["uname"].(string),
-		//Password:    Sysconfig["upwd"].(string),
-		//ReplSet: 	 "bidding",
+		UserName:	 Sysconfig["uname"].(string),
+		Password:    Sysconfig["upwd"].(string),
+		ReplSet: 	 "bidding",
 	}
 	mgo.InitPool()
 	project2db = &mongodb.MongodbSim{
@@ -108,9 +106,9 @@ func init() {
 		DbName:      standard["db"].(string),
 		UserName:    Sysconfig["uname"].(string),
 		Password:    Sysconfig["upwd"].(string),
-		ReplSet: 	 "bidding",
 	}
 	mgostandard.InitPool()
+
 	//初始化es
 	//bidding
 	econf := Sysconfig["elastic"].(map[string]interface{})
@@ -184,6 +182,7 @@ func init() {
 	log.Println(purchasinglistFields)
 	//初始化oss
 	u.InitOss()
+
 }
 
 func main() {

udpcreateindex/src/util/ossclient.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	ossEndpoint        = "oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
+	ossEndpoint        = "ss-cn-beijing.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
 	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
 	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
 	ossBucketName      = "topjy"