xg

udpcreateindex/src/biddingall.go

@@ -230,25 +230,27 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 									}
 								}
 								newTmp[v] = newmap
-								attachments := mp["attachments"]
-								con := ""
-								if attachments != nil {
-									am, _ := attachments.(map[string]interface{})
-									if am != nil {
-										for _, v1 := range am {
-											vm, _ := v1.(map[string]interface{})
-											if vm != nil {
-												c, _ := vm["content"].(string)
-												con += c
-											}
+							}
+						} else if v == "purchasinglist" { //标的物处理
+							purchasinglist_new := []map[string]interface{}{}
+							if pcl, _ := tmp[v].([]interface{}); len(pcl) > 0 {
+								for _, ls := range pcl {
+									lsm_new := make(map[string]interface{})
+									lsm := ls.(map[string]interface{})
+									for _, pf := range purchasinglistFields {
+										if lsm[pf] != nil {
+											lsm_new[pf] = lsm[pf]
 										}
 									}
-								}
-								con = FilterDetailSpace(con)
-								if con != "" {
-									newTmp["attachments"] = con
+									if lsm_new != nil && len(lsm_new) > 0 {
+										purchasinglist_new = append(purchasinglist_new, lsm_new)
+									}
 								}
 							}
+							if len(purchasinglist_new) > 0 {
+								newTmp[v] = purchasinglist_new
+							}
+
 						} else {
 							if v == "detail" {
 								detail, _ := tmp[v].(string)

udpcreateindex/src/biddingdata.go

@@ -246,24 +246,24 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 									}
 								}
 								newTmp[v] = newmap
-								attachments := mp["attachments"]
-								con := ""
-								if attachments != nil {
-									am, _ := attachments.(map[string]interface{})
-									if am != nil {
-										for _, v1 := range am {
-											vm, _ := v1.(map[string]interface{})
-											if vm != nil {
-												c, _ := vm["content"].(string)
-												con += c
-											}
-										}
-									}
-								}
-								if con != "" {
-									con = FilterDetailSpace(con)
-									newTmp["attachments"] = con
-								}
+								// attachments := mp["attachments"]
+								// con := ""
+								// if attachments != nil {
+								// 	am, _ := attachments.(map[string]interface{})
+								// 	if am != nil {
+								// 		for _, v1 := range am {
+								// 			vm, _ := v1.(map[string]interface{})
+								// 			if vm != nil {
+								// 				c, _ := vm["content"].(string)
+								// 				con += c
+								// 			}
+								// 		}
+								// 	}
+								// }
+								// if con != "" {
+								// 	con = FilterDetailSpace(con)
+								// 	newTmp["attachments"] = con
+								// }
 							}
 						} else {
 							if v == "detail" {

udpcreateindex/src/biddingindex.go

@@ -216,9 +216,6 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		if qutil.IntAll(update["extracttype"]) != -1 {
 			newTmp := map[string]interface{}{}     //最终生索引的数据
 			for _, v := range biddingIndexFields { //索引字段
-				//			if tmp[v] != nil {
-				//				newTmp[v] = tmp[v]
-				//			}
 				if tmp[v] != nil {
 					if "projectinfo" == v {
 						mp, _ := tmp[v].(map[string]interface{})
@@ -230,24 +227,24 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 								}
 							}
 							newTmp[v] = newmap
-							attachments := mp["attachments"]
-							con := ""
-							if attachments != nil {
-								am, _ := attachments.(map[string]interface{})
-								if am != nil {
-									for _, v1 := range am {
-										vm, _ := v1.(map[string]interface{})
-										if vm != nil {
-											c, _ := vm["content"].(string)
-											con += c
-										}
-									}
-								}
-							}
-							con = FilterDetailSpace(con)
-							if con != "" {
-								newTmp["attachments"] = con
-							}
+							// attachments := mp["attachments"]
+							// con := ""
+							// if attachments != nil {
+							// 	am, _ := attachments.(map[string]interface{})
+							// 	if am != nil {
+							// 		for _, v1 := range am {
+							// 			vm, _ := v1.(map[string]interface{})
+							// 			if vm != nil {
+							// 				c, _ := vm["content"].(string)
+							// 				con += c
+							// 			}
+							// 		}
+							// 	}
+							// }
+							// con = FilterDetailSpace(con)
+							// if con != "" {
+							// 	newTmp["attachments"] = con
+							// }
 						}
 					} else {
 						if v == "detail" {
@@ -257,9 +254,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 							newTmp[v] = tmp[v]
 						}
 					}
-				} /*else if v == "budget" || v == "bidamount" {
-					newTmp[v] = nil
-				}*/
+				}
 			}
 			arrEs = append(arrEs, newTmp)
 		}

udpcreateindex/src/biddingindexback.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	qutil "qfw/util"
 	elastic "qfw/util/elastic"
+
 	//elastic "qfw/util/elastic_v5"
 	"regexp"
 	//	"strings"
@@ -115,24 +116,24 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 							}
 						}
 						newTmp[v] = newmap
-						attachments := mp["attachments"]
-						con := ""
-						if attachments != nil {
-							am, _ := attachments.(map[string]interface{})
-							if am != nil {
-								for _, v1 := range am {
-									vm, _ := v1.(map[string]interface{})
-									if vm != nil {
-										c, _ := vm["content"].(string)
-										con += c
-									}
-								}
-							}
-						}
-						con = FilterDetailSpace(con)
-						if con != "" {
-							newTmp["attachments"] = con
-						}
+						// attachments := mp["attachments"]
+						// con := ""
+						// if attachments != nil {
+						// 	am, _ := attachments.(map[string]interface{})
+						// 	if am != nil {
+						// 		for _, v1 := range am {
+						// 			vm, _ := v1.(map[string]interface{})
+						// 			if vm != nil {
+						// 				c, _ := vm["content"].(string)
+						// 				con += c
+						// 			}
+						// 		}
+						// 	}
+						// }
+						// con = FilterDetailSpace(con)
+						// if con != "" {
+						// 	newTmp["attachments"] = con
+						// }
 					}
 				} else {
 					if v == "detail" {

udpcreateindex/src/biddingindexback2.go

@@ -6,6 +6,7 @@ import (
 	qutil "qfw/util"
 	elastic "qfw/util/elastic"
 	"strings"
+
 	//elastic "qfw/util/elastic_v5"
 	//	"strings"
 	"sync"
@@ -108,24 +109,24 @@ func biddingBackTask2(data []byte, mapInfo map[string]interface{}) {
 							if len(newmap) > 0 {
 								newTmp[v] = newmap
 							}
-							attachments := mp["attachments"]
-							con := ""
-							if attachments != nil {
-								am, _ := attachments.(map[string]interface{})
-								if am != nil {
-									for _, v1 := range am {
-										vm, _ := v1.(map[string]interface{})
-										if vm != nil {
-											c, _ := vm["content"].(string)
-											con += c
-										}
-									}
-								}
-							}
-							if con != "" {
-								con = FilterDetailSpace(con)
-								newTmp["attachments"] = con
-							}
+							// attachments := mp["attachments"]
+							// con := ""
+							// if attachments != nil {
+							// 	am, _ := attachments.(map[string]interface{})
+							// 	if am != nil {
+							// 		for _, v1 := range am {
+							// 			vm, _ := v1.(map[string]interface{})
+							// 			if vm != nil {
+							// 				c, _ := vm["content"].(string)
+							// 				con += c
+							// 			}
+							// 		}
+							// 	}
+							// }
+							// if con != "" {
+							// 	con = FilterDetailSpace(con)
+							// 	newTmp["attachments"] = con
+							// }
 						}
 					} else {
 						if v == "detail" {

udpcreateindex/src/bidingpurchasing.go

@@ -43,12 +43,36 @@ func biddingPurchaingTask(q map[string]interface{}) {
 		}
 		newTmp := map[string]interface{}{} //最终生索引的数据
 		//oss拼装filetext
-		filetext := getFileText(tmp)
-		newTmp["filetext"] = filetext
+		if filetext := getFileText(tmp); len(filetext) > 0 {
+			newTmp["filetext"] = filetext
+		}
 		//purchasing
-		newTmp["purchasing"] = tmp["purchasing"]
+		if purchasing, ok := tmp["purchasing"].(string); ok {
+			if len(purchasing) > 0 {
+				newTmp["purchasing"] = tmp["purchasing"]
+			}
+		}
 		//purchasinglist
-		newTmp["purchasinglist"] = tmp["purchasinglist"]
+		if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok {
+			if len(purchasinglist) > 0 {
+				purchasinglist_new := []map[string]interface{}{}
+				for _, ls := range purchasinglist {
+					lsm_new := make(map[string]interface{})
+					lsm := ls.(map[string]interface{})
+					for _, pf := range purchasinglistFields {
+						if lsm[pf] != nil {
+							lsm_new[pf] = lsm[pf]
+						}
+					}
+					if lsm_new != nil && len(lsm_new) > 0 {
+						purchasinglist_new = append(purchasinglist_new, lsm_new)
+					}
+				}
+				if len(purchasinglist_new) > 0 {
+					newTmp["purchasinglist"] = purchasinglist_new
+				}
+			}
+		}
 
 		//处理数据
 		if tmp["supervisorrate"] != nil { //临时处理supervisorrate抽取类型为string不生索引
@@ -74,24 +98,24 @@ func biddingPurchaingTask(q map[string]interface{}) {
 							}
 						}
 						newTmp[v] = newmap
-						attachments := mp["attachments"]
-						con := ""
-						if attachments != nil {
-							am, _ := attachments.(map[string]interface{})
-							if am != nil {
-								for _, v1 := range am {
-									vm, _ := v1.(map[string]interface{})
-									if vm != nil {
-										c, _ := vm["content"].(string)
-										con += c
-									}
-								}
-							}
-						}
-						con = FilterDetailSpace(con)
-						if con != "" {
-							newTmp["attachments"] = con
-						}
+						// attachments := mp["attachments"]
+						// con := ""
+						// if attachments != nil {
+						// 	am, _ := attachments.(map[string]interface{})
+						// 	if am != nil {
+						// 		for _, v1 := range am {
+						// 			vm, _ := v1.(map[string]interface{})
+						// 			if vm != nil {
+						// 				c, _ := vm["content"].(string)
+						// 				con += c
+						// 			}
+						// 		}
+						// 	}
+						// }
+						// con = FilterDetailSpace(con)
+						// if con != "" {
+						// 	newTmp["attachments"] = con
+						// }
 					}
 				} else {
 					if v == "detail" {

udpcreateindex/src/config.json

@@ -40,6 +40,7 @@
         ],
         "fields": "buyerzipcode,winnertel,winnerperson,contractcode,winneraddr,agencyaddr,buyeraddr,signaturedate,projectperiod,projectaddr,agencytel,agencyperson,buyerperson,agency,projectscope,projectcode,bidopentime,supervisorrate,buyertel,bidamount,winner,buyer,budget,projectname,buyerclass,topscopeclass,area,city,district,s_winner,toptype,subtype,subscopeclass,s_subscopeclass",
         "projectinfo": "approvecode,approvecontent,approvestatus,approvetime,industry",
+        "purchasinglist":"itemname,model,unitname,number",
         "multiIndex": ""
     },
     "filelength": 100000,

udpcreateindex/src/main.go

@@ -15,19 +15,20 @@ import (
 )
 
 var (
-	Sysconfig          map[string]interface{} //配置文件
-	mgo                *mongodb.MongodbSim    //mongodb操作对象
-	extractmgo         *mongodb.MongodbSim    //mongodb操作对象
-	project2db         *mongodb.MongodbSim    //mongodb操作对象
-	mgostandard        *mongodb.MongodbSim    //mongodb操作对象
-	qyxydb             *mongodb.MongodbSim    //mongodb操作对象
-	udpclient          mu.UdpClient           //udp对象
-	updport            string
-	savesizei          = 500
-	biddingIndexFields = []string{"_id", "buyerclass", "s_winner", "title", "detail", "detail_bak", "area", "areaval", "site", "type", "amount", "bidopendate", "bidopentime", "buyer", "channel", "city", "comeintime", "contenthtml", "descript", "description", "extracttype", "href", "infoformat", "keywords", "projectcode", "projectname", "publishtime", "s_sha", "spidercode", "subtype", "summary", "toptype", "urltop", "winner", "agency", "budget", "bidamount", "s_subscopeclass", "projectscope", "bidstatus"}
-	projectinfoFields  []string
-	multiIndex         []string
-	BulkSize           = 400
+	Sysconfig            map[string]interface{} //配置文件
+	mgo                  *mongodb.MongodbSim    //mongodb操作对象
+	extractmgo           *mongodb.MongodbSim    //mongodb操作对象
+	project2db           *mongodb.MongodbSim    //mongodb操作对象
+	mgostandard          *mongodb.MongodbSim    //mongodb操作对象
+	qyxydb               *mongodb.MongodbSim    //mongodb操作对象
+	udpclient            mu.UdpClient           //udp对象
+	updport              string
+	savesizei            = 500
+	biddingIndexFields   = []string{"_id", "buyerclass", "s_winner", "title", "detail", "detail_bak", "area", "areaval", "site", "type", "amount", "bidopendate", "bidopentime", "buyer", "channel", "city", "comeintime", "contenthtml", "descript", "description", "extracttype", "href", "infoformat", "keywords", "projectcode", "projectname", "publishtime", "s_sha", "spidercode", "subtype", "summary", "toptype", "urltop", "winner", "agency", "budget", "bidamount", "s_subscopeclass", "projectscope", "bidstatus"}
+	projectinfoFields    []string
+	multiIndex           []string
+	purchasinglistFields []string
+	BulkSize             = 400
 
 	winner, bidding, biddingback, project, project2, buyer, standard, qyxy_ent map[string]interface{}
 )
@@ -102,6 +103,12 @@ func init() {
 			projectinfoFields = strings.Split(pf, ",")
 		}
 	}
+	if bidding["purchasinglist"] != nil {
+		pcl := util.ObjToString(bidding["purchasinglist"])
+		if pcl != "" {
+			purchasinglistFields = strings.Split(pcl, ",")
+		}
+	}
 	if bidding["multiIndex"] != nil {
 		mi := util.ObjToString(bidding["multiIndex"])
 		if mi != "" {
@@ -109,6 +116,7 @@ func init() {
 		}
 	}
 	log.Println(projectinfoFields)
+	log.Println(purchasinglistFields)
 	//初始化oss
 	u.InitOss()
 }

udpcreateindex/src/task.go

@@ -12,7 +12,7 @@ import (
 func task_index() {
 	c := cron.New()
 	c.AddFunc("20 30 5 * * *", func() { task_projects() })
-	c.AddFunc("0 15 * * * *", func() { task_biddingfile() }) //每两小时执行一次
+	c.AddFunc("0 30 * * * *", func() { task_biddingfile() }) //每两小时执行一次
 	//c.AddFunc("0 22 14 * * *", func() { task_qyxyindex() })
 	c.Start()
 }
@@ -21,8 +21,12 @@ func task_index() {
 func task_biddingfile() {
 	defer qutil.Catch()
 	q := map[string]interface{}{
-		"extract_state": map[string]interface{}{
-			"$eq": 3,
+		"comeintime": map[string]interface{}{
+			"$gte": time.Now().Unix() - 5400,
+			"$lte": time.Now().Unix() - 3600,
+		},
+		"attach_text": map[string]interface{}{
+			"$exists": true,
 		},
 	}
 	biddingPurchaingTask(q)