Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

fullproject/src_v1/project.go

@@ -515,7 +515,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	if thisinfo.TopType == "招标" {
 		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
 			set["zbtime"] = tmp["publishtime"]
-			p1.Zbtime = tmp["publishtime"].(int64)
+			p1.Zbtime = qu.Int64All(tmp["publishtime"])
 		}
 	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
 			set["jgtime"] = tmp["publishtime"]
@@ -542,6 +542,11 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 		p1.Bidamounttag = 1
 		set["bidamounttag"] = 1
 	}
+	if p1.Bidamount > 0 {
+		set["sortprice"] = p1.Bidamount
+	}else if p1.Budget > 0 {
+		set["sortprice"] = p1.Budget
+	}
 
 	if len(thisinfo.Winners) > 0 {
 		set["s_winner"] = strings.Join(thisinfo.Winners, ",")
@@ -698,18 +703,33 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		}
 	} else if thisinfo.TopType == "结果" {
 		if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
-			jg1 := int64(math.Abs(float64(pInfo.Jgtime - thisinfo.Publishtime)))
-			if pInfo.Jgtime <= 0 {
-				set["jgtime"] = tmp["publishtime"]
-				pInfo.Jgtime = thisinfo.Publishtime
-			}else if jg1 > p.jgTime {
+			if pInfo.Jgtime > 0 {
+				jg1 := int64(math.Abs(float64(pInfo.Jgtime - thisinfo.Publishtime)))
+				//公告状态和项目状态同样都是中标或者成交，
+				if (thisinfo.SubType == "中标" || thisinfo.SubType == "成交") && (pInfo.Bidstatus == "中标" || pInfo.Bidstatus == "成交") {
+					if jg1 > p.jgTime {
+						set["jgtime"] = tmp["publishtime"]
+						pInfo.Jgtime = thisinfo.Publishtime
+					}
+				//公告状态和项目状态同样是流标或者废标
+				}else if (thisinfo.SubType == "流标" || thisinfo.SubType == "废标") && (pInfo.Bidstatus == "流标" || pInfo.Bidstatus == "废标") {
+					if jg1 > p.jgTime {
+						set["jgtime"] = tmp["publishtime"]
+						pInfo.Jgtime = thisinfo.Publishtime
+					}
+				}
+			}else {
 				set["jgtime"] = tmp["publishtime"]
 				pInfo.Jgtime = thisinfo.Publishtime
 			}
-		}else if thisinfo.SubType == "合同" {
-			set["jgtime"] = tmp["publishtime"]
-			pInfo.Jgtime = thisinfo.Publishtime
 		}
+	} else if thisinfo.SubType == "合同" {
+		if pInfo.Bidstatus == "中标" || pInfo.Bidstatus == "成交" {
+			//中标、成交不更新jgtime
+			return
+		}
+		set["jgtime"] = tmp["publishtime"]
+		pInfo.Jgtime = thisinfo.Publishtime
 	}
 	if thisinfo.Bidopentime > pInfo.Bidopentime {
 		pInfo.Bidopentime = thisinfo.Bidopentime
@@ -842,9 +862,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		for _, k := range thisinfo.Winners {
 			if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
 				if BinarySearch(pInfo.Winners, k) != -1 {
-					arr := strings.Split(pInfo.Winners, ",")
-					deleteSlice(arr, k, "")
-					pInfo.Winners = strings.Join(pInfo.Winners, ",")
+					deleteSlice(pInfo.Winners, k, "")
 					sort.Strings(pInfo.Winners)
 				}
 			}else {
@@ -877,6 +895,11 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	} else {
 		set["bidamounttag"] = 1
 	}
+	if pInfo.Bidamount > 0 {
+		set["sortprice"] = pInfo.Bidamount
+	}else if pInfo.Budget > 0 {
+		set["sortprice"] = pInfo.Budget
+	}
 
 	infofiled := InfoField{
 		Budget:       thisinfo.Budget,

fullproject/src_v1/task.go

@@ -91,8 +91,8 @@ func NewPT() *ProjectTask {
 		//updateSign: make(chan bool, 1),
 		coll:       ProjectColl,
 		validTime:  int64(util.IntAllDef(Sysconfig["validdays"], 150) * 86400),
-		statusTime: int64(util.IntAllDef(Sysconfig["statusdays"], 7) * 86400),
-		jgTime:		int64(util.IntAllDef("", 2) * 86400),
+		statusTime: int64(util.IntAllDef(Sysconfig["statusdays"], 15) * 86400),
+		jgTime:		int64(util.IntAllDef(3, 3) * 86400),
 	}
 	return p
 }
@@ -398,6 +398,10 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 						<-pool
 					}()
 					if util.IntAll(tmp["repeat"]) == 0 {
+						if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 1 {
+							//增量	dataging为1不参与合并
+							return
+						}
 						p.fillInPlace(tmp)
 						info := ParseInfo(tmp)
 						p.currentTime = info.Publishtime

udpcreateindex/src/biddingall.go

@@ -230,25 +230,27 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 									}
 								}
 								newTmp[v] = newmap
-								attachments := mp["attachments"]
-								con := ""
-								if attachments != nil {
-									am, _ := attachments.(map[string]interface{})
-									if am != nil {
-										for _, v1 := range am {
-											vm, _ := v1.(map[string]interface{})
-											if vm != nil {
-												c, _ := vm["content"].(string)
-												con += c
-											}
+							}
+						} else if v == "purchasinglist" { //标的物处理
+							purchasinglist_new := []map[string]interface{}{}
+							if pcl, _ := tmp[v].([]interface{}); len(pcl) > 0 {
+								for _, ls := range pcl {
+									lsm_new := make(map[string]interface{})
+									lsm := ls.(map[string]interface{})
+									for _, pf := range purchasinglistFields {
+										if lsm[pf] != nil {
+											lsm_new[pf] = lsm[pf]
 										}
 									}
-								}
-								con = FilterDetailSpace(con)
-								if con != "" {
-									newTmp["attachments"] = con
+									if lsm_new != nil && len(lsm_new) > 0 {
+										purchasinglist_new = append(purchasinglist_new, lsm_new)
+									}
 								}
 							}
+							if len(purchasinglist_new) > 0 {
+								newTmp[v] = purchasinglist_new
+							}
+
 						} else {
 							if v == "detail" {
 								detail, _ := tmp[v].(string)

udpcreateindex/src/biddingdata.go

@@ -246,24 +246,24 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 									}
 								}
 								newTmp[v] = newmap
-								attachments := mp["attachments"]
-								con := ""
-								if attachments != nil {
-									am, _ := attachments.(map[string]interface{})
-									if am != nil {
-										for _, v1 := range am {
-											vm, _ := v1.(map[string]interface{})
-											if vm != nil {
-												c, _ := vm["content"].(string)
-												con += c
-											}
-										}
-									}
-								}
-								if con != "" {
-									con = FilterDetailSpace(con)
-									newTmp["attachments"] = con
-								}
+								// attachments := mp["attachments"]
+								// con := ""
+								// if attachments != nil {
+								// 	am, _ := attachments.(map[string]interface{})
+								// 	if am != nil {
+								// 		for _, v1 := range am {
+								// 			vm, _ := v1.(map[string]interface{})
+								// 			if vm != nil {
+								// 				c, _ := vm["content"].(string)
+								// 				con += c
+								// 			}
+								// 		}
+								// 	}
+								// }
+								// if con != "" {
+								// 	con = FilterDetailSpace(con)
+								// 	newTmp["attachments"] = con
+								// }
 							}
 						} else {
 							if v == "detail" {

udpcreateindex/src/biddingindex.go

@@ -216,9 +216,6 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		if qutil.IntAll(update["extracttype"]) != -1 {
 			newTmp := map[string]interface{}{}     //最终生索引的数据
 			for _, v := range biddingIndexFields { //索引字段
-				//			if tmp[v] != nil {
-				//				newTmp[v] = tmp[v]
-				//			}
 				if tmp[v] != nil {
 					if "projectinfo" == v {
 						mp, _ := tmp[v].(map[string]interface{})
@@ -230,24 +227,24 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 								}
 							}
 							newTmp[v] = newmap
-							attachments := mp["attachments"]
-							con := ""
-							if attachments != nil {
-								am, _ := attachments.(map[string]interface{})
-								if am != nil {
-									for _, v1 := range am {
-										vm, _ := v1.(map[string]interface{})
-										if vm != nil {
-											c, _ := vm["content"].(string)
-											con += c
-										}
-									}
-								}
-							}
-							con = FilterDetailSpace(con)
-							if con != "" {
-								newTmp["attachments"] = con
-							}
+							// attachments := mp["attachments"]
+							// con := ""
+							// if attachments != nil {
+							// 	am, _ := attachments.(map[string]interface{})
+							// 	if am != nil {
+							// 		for _, v1 := range am {
+							// 			vm, _ := v1.(map[string]interface{})
+							// 			if vm != nil {
+							// 				c, _ := vm["content"].(string)
+							// 				con += c
+							// 			}
+							// 		}
+							// 	}
+							// }
+							// con = FilterDetailSpace(con)
+							// if con != "" {
+							// 	newTmp["attachments"] = con
+							// }
 						}
 					} else {
 						if v == "detail" {
@@ -257,9 +254,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 							newTmp[v] = tmp[v]
 						}
 					}
-				} /*else if v == "budget" || v == "bidamount" {
-					newTmp[v] = nil
-				}*/
+				}
 			}
 			arrEs = append(arrEs, newTmp)
 		}

udpcreateindex/src/biddingindexback.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	qutil "qfw/util"
 	elastic "qfw/util/elastic"
+
 	//elastic "qfw/util/elastic_v5"
 	"regexp"
 	//	"strings"
@@ -115,24 +116,24 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 							}
 						}
 						newTmp[v] = newmap
-						attachments := mp["attachments"]
-						con := ""
-						if attachments != nil {
-							am, _ := attachments.(map[string]interface{})
-							if am != nil {
-								for _, v1 := range am {
-									vm, _ := v1.(map[string]interface{})
-									if vm != nil {
-										c, _ := vm["content"].(string)
-										con += c
-									}
-								}
-							}
-						}
-						con = FilterDetailSpace(con)
-						if con != "" {
-							newTmp["attachments"] = con
-						}
+						// attachments := mp["attachments"]
+						// con := ""
+						// if attachments != nil {
+						// 	am, _ := attachments.(map[string]interface{})
+						// 	if am != nil {
+						// 		for _, v1 := range am {
+						// 			vm, _ := v1.(map[string]interface{})
+						// 			if vm != nil {
+						// 				c, _ := vm["content"].(string)
+						// 				con += c
+						// 			}
+						// 		}
+						// 	}
+						// }
+						// con = FilterDetailSpace(con)
+						// if con != "" {
+						// 	newTmp["attachments"] = con
+						// }
 					}
 				} else {
 					if v == "detail" {

udpcreateindex/src/biddingindexback2.go

@@ -6,6 +6,7 @@ import (
 	qutil "qfw/util"
 	elastic "qfw/util/elastic"
 	"strings"
+
 	//elastic "qfw/util/elastic_v5"
 	//	"strings"
 	"sync"
@@ -108,24 +109,24 @@ func biddingBackTask2(data []byte, mapInfo map[string]interface{}) {
 							if len(newmap) > 0 {
 								newTmp[v] = newmap
 							}
-							attachments := mp["attachments"]
-							con := ""
-							if attachments != nil {
-								am, _ := attachments.(map[string]interface{})
-								if am != nil {
-									for _, v1 := range am {
-										vm, _ := v1.(map[string]interface{})
-										if vm != nil {
-											c, _ := vm["content"].(string)
-											con += c
-										}
-									}
-								}
-							}
-							if con != "" {
-								con = FilterDetailSpace(con)
-								newTmp["attachments"] = con
-							}
+							// attachments := mp["attachments"]
+							// con := ""
+							// if attachments != nil {
+							// 	am, _ := attachments.(map[string]interface{})
+							// 	if am != nil {
+							// 		for _, v1 := range am {
+							// 			vm, _ := v1.(map[string]interface{})
+							// 			if vm != nil {
+							// 				c, _ := vm["content"].(string)
+							// 				con += c
+							// 			}
+							// 		}
+							// 	}
+							// }
+							// if con != "" {
+							// 	con = FilterDetailSpace(con)
+							// 	newTmp["attachments"] = con
+							// }
 						}
 					} else {
 						if v == "detail" {

udpcreateindex/src/bidingpurchasing.go

@@ -43,12 +43,36 @@ func biddingPurchaingTask(q map[string]interface{}) {
 		}
 		newTmp := map[string]interface{}{} //最终生索引的数据
 		//oss拼装filetext
-		filetext := getFileText(tmp)
-		newTmp["filetext"] = filetext
+		if filetext := getFileText(tmp); len(filetext) > 0 {
+			newTmp["filetext"] = filetext
+		}
 		//purchasing
-		newTmp["purchasing"] = tmp["purchasing"]
+		if purchasing, ok := tmp["purchasing"].(string); ok {
+			if len(purchasing) > 0 {
+				newTmp["purchasing"] = tmp["purchasing"]
+			}
+		}
 		//purchasinglist
-		newTmp["purchasinglist"] = tmp["purchasinglist"]
+		if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok {
+			if len(purchasinglist) > 0 {
+				purchasinglist_new := []map[string]interface{}{}
+				for _, ls := range purchasinglist {
+					lsm_new := make(map[string]interface{})
+					lsm := ls.(map[string]interface{})
+					for _, pf := range purchasinglistFields {
+						if lsm[pf] != nil {
+							lsm_new[pf] = lsm[pf]
+						}
+					}
+					if lsm_new != nil && len(lsm_new) > 0 {
+						purchasinglist_new = append(purchasinglist_new, lsm_new)
+					}
+				}
+				if len(purchasinglist_new) > 0 {
+					newTmp["purchasinglist"] = purchasinglist_new
+				}
+			}
+		}
 
 		//处理数据
 		if tmp["supervisorrate"] != nil { //临时处理supervisorrate抽取类型为string不生索引
@@ -74,24 +98,24 @@ func biddingPurchaingTask(q map[string]interface{}) {
 							}
 						}
 						newTmp[v] = newmap
-						attachments := mp["attachments"]
-						con := ""
-						if attachments != nil {
-							am, _ := attachments.(map[string]interface{})
-							if am != nil {
-								for _, v1 := range am {
-									vm, _ := v1.(map[string]interface{})
-									if vm != nil {
-										c, _ := vm["content"].(string)
-										con += c
-									}
-								}
-							}
-						}
-						con = FilterDetailSpace(con)
-						if con != "" {
-							newTmp["attachments"] = con
-						}
+						// attachments := mp["attachments"]
+						// con := ""
+						// if attachments != nil {
+						// 	am, _ := attachments.(map[string]interface{})
+						// 	if am != nil {
+						// 		for _, v1 := range am {
+						// 			vm, _ := v1.(map[string]interface{})
+						// 			if vm != nil {
+						// 				c, _ := vm["content"].(string)
+						// 				con += c
+						// 			}
+						// 		}
+						// 	}
+						// }
+						// con = FilterDetailSpace(con)
+						// if con != "" {
+						// 	newTmp["attachments"] = con
+						// }
 					}
 				} else {
 					if v == "detail" {

udpcreateindex/src/config.json

@@ -40,6 +40,7 @@
         ],
         "fields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass",
         "projectinfo": "approvecode,approvecontent,approvestatus,approvetime,industry",
+        "purchasinglist":"itemname,model,unitname,number",
         "multiIndex": ""
     },
     "filelength": 100000,

udpcreateindex/src/main.go

@@ -15,19 +15,20 @@ import (
 )
 
 var (
-	Sysconfig          map[string]interface{} //配置文件
-	mgo                *mongodb.MongodbSim    //mongodb操作对象
-	extractmgo         *mongodb.MongodbSim    //mongodb操作对象
-	project2db         *mongodb.MongodbSim    //mongodb操作对象
-	mgostandard        *mongodb.MongodbSim    //mongodb操作对象
-	qyxydb             *mongodb.MongodbSim    //mongodb操作对象
-	udpclient          mu.UdpClient           //udp对象
-	updport            string
-	savesizei          = 500
-	biddingIndexFields = []string{"_id", "buyerclass", "s_winner", "title", "detail", "detail_bak", "area", "areaval", "site", "type", "amount", "bidopendate", "bidopentime", "buyer", "channel", "city", "comeintime", "contenthtml", "descript", "description", "extracttype", "href", "infoformat", "keywords", "projectcode", "projectname", "publishtime", "s_sha", "spidercode", "subtype", "summary", "toptype", "urltop", "winner", "agency", "budget", "bidamount", "s_subscopeclass", "projectscope", "bidstatus"}
-	projectinfoFields  []string
-	multiIndex         []string
-	BulkSize           = 400
+	Sysconfig            map[string]interface{} //配置文件
+	mgo                  *mongodb.MongodbSim    //mongodb操作对象
+	extractmgo           *mongodb.MongodbSim    //mongodb操作对象
+	project2db           *mongodb.MongodbSim    //mongodb操作对象
+	mgostandard          *mongodb.MongodbSim    //mongodb操作对象
+	qyxydb               *mongodb.MongodbSim    //mongodb操作对象
+	udpclient            mu.UdpClient           //udp对象
+	updport              string
+	savesizei            = 500
+	biddingIndexFields   = []string{"_id", "buyerclass", "s_winner", "title", "detail", "detail_bak", "area", "areaval", "site", "type", "amount", "bidopendate", "bidopentime", "buyer", "channel", "city", "comeintime", "contenthtml", "descript", "description", "extracttype", "href", "infoformat", "keywords", "projectcode", "projectname", "publishtime", "s_sha", "spidercode", "subtype", "summary", "toptype", "urltop", "winner", "agency", "budget", "bidamount", "s_subscopeclass", "projectscope", "bidstatus"}
+	projectinfoFields    []string
+	multiIndex           []string
+	purchasinglistFields []string
+	BulkSize             = 400
 
 	winner, bidding, biddingback, project, project2, buyer, standard, qyxy_ent map[string]interface{}
 )
@@ -102,6 +103,12 @@ func init() {
 			projectinfoFields = strings.Split(pf, ",")
 		}
 	}
+	if bidding["purchasinglist"] != nil {
+		pcl := util.ObjToString(bidding["purchasinglist"])
+		if pcl != "" {
+			purchasinglistFields = strings.Split(pcl, ",")
+		}
+	}
 	if bidding["multiIndex"] != nil {
 		mi := util.ObjToString(bidding["multiIndex"])
 		if mi != "" {
@@ -109,6 +116,7 @@ func init() {
 		}
 	}
 	log.Println(projectinfoFields)
+	log.Println(purchasinglistFields)
 	//初始化oss
 	u.InitOss()
 }

udpcreateindex/src/task.go

@@ -12,7 +12,7 @@ import (
 func task_index() {
 	c := cron.New()
 	c.AddFunc("20 30 5 * * *", func() { task_projects() })
-	c.AddFunc("0 15 * * * *", func() { task_biddingfile() }) //每两小时执行一次
+	c.AddFunc("0 30 * * * *", func() { task_biddingfile() }) //每两小时执行一次
 	//c.AddFunc("0 22 14 * * *", func() { task_qyxyindex() })
 	c.Start()
 }
@@ -21,8 +21,12 @@ func task_index() {
 func task_biddingfile() {
 	defer qutil.Catch()
 	q := map[string]interface{}{
-		"extract_state": map[string]interface{}{
-			"$eq": 3,
+		"comeintime": map[string]interface{}{
+			"$gte": time.Now().Unix() - 5400,
+			"$lte": time.Now().Unix() - 3600,
+		},
+		"attach_text": map[string]interface{}{
+			"$exists": true,
 		},
 	}
 	biddingPurchaingTask(q)