Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

domainameclear/src/task.go

@@ -62,8 +62,8 @@ func StartTask() {
 	LatestId = endId //替换起始id
 	field := map[string]interface{}{"detail": 1}
 	logger.Debug("query：", q)
-	it := sess.DB("mxs").C("bidding_back").Find(q).Select(field).Sort("_id").Iter()
-	count := Mgo.Count("bidding_back", q)
+	it := sess.DB("qfw").C("bidding").Find(q).Select(field).Sort("_id").Iter()
+	count := Mgo.Count("bidding", q)
 	fmt.Println("共加载数据", count)
 	sum := 0
 	wg := &sync.WaitGroup{}
@@ -190,7 +190,7 @@ func InitDomainName() {
 func GetEndId(query map[string]interface{}) (string, bool) {
 	endId := ""
 	ok := false
-	list, _ := Mgo.Find("bidding_back", query, `{"_id":-1}`, `{_id:1}`, false, 0, 1)
+	list, _ := Mgo.Find("bidding", query, `{"_id":-1}`, `{_id:1}`, false, 0, 1)
 	if len(*list) == 1 {
 		endId = mgo.BsonIdToSId((*list)[0]["_id"])
 		if endId >= LatestId {

src/jy/extract/newextractcity.go

@@ -314,7 +314,7 @@ func GetByACDSimJb(pbrief, city, district, a_c_d string, e *ExtractTask, j *ju.J
 				PCDScore(j, "district", sim, 5, true)
 				for _, dfullAndCity := range dfullarr { //district简称对应的所有全称
 					for _, c := range dfullAndCity {
-						if c == nil{
+						if c == nil {
 							continue
 						}
 						tmpcity := c.Name      //城市全称

src/res/fieldscore.json

@@ -75,6 +75,11 @@
                 "describe": "以*结尾",
                 "regstr": ".{2,100}(项目|工程|施工|服务|设备|采购|设计|系统)$",
                 "score": 2
+            },
+            {
+                "describe": "+3",
+                "regstr": "(供货|采购|监测|招标|询价)",
+                "score": 3
             }
         ],
         "negativewords": [
@@ -90,12 +95,17 @@
             },
             {
                 "describe": "包含词-10",
-                "regstr": "(万元|本项目|详见公告|test|[0-9]{1}、)",
+                "regstr": "(万元|本项目|详见公告|test|^[0-9]{1}、\\W)",
                 "score": -10
             },
+            {
+                "describe": "包含词-3",
+                "regstr": "[,|，]",
+                "score": -5
+            },
             {
                 "describe": "包含词-5",
-                "regstr": "[.|,|，]",
+                "regstr": "[.|。]",
                 "score": -5
             },
             {
@@ -105,12 +115,7 @@
             },
             {
                 "describe": "乱码",
-                "regstr": "[±?？¨êí¤ìüàóμˉ÷°úéè]",
-                "score": -10
-            },
-            {
-                "describe": "符合",
-                "regstr": "[。!！]",
+                "regstr": "[±?？¨êí¤ìüàóμˉ÷°úéè!！]",
                 "score": -10
             }
         ],
@@ -197,7 +202,7 @@
             },
             {
                 "describe": "包含负分不再展示",
-                "regstr": "(详见|提出|面向|施工|获得|test|认定|一批|项目|系统)",
+                "regstr": "(详见|提出|面向|施工|获得|test|认定|一批|项目$|系统)",
                 "score": -50
             },
             {
@@ -794,22 +799,22 @@
         "type": "float",
         "describe": "min>val:1,min<=val<=max:3,max<val:1",
         "min": 100,
-        "max": 1000000000000,
+        "max": 10000000000,
         "score": [
             -3,
             2,
-            -50
+            -4
         ]
     },
     "budget": {
         "type": "float",
         "describe": "min>val:1,min<=val<=max:3,max<val:1",
         "min": 1000,
-        "max": 1000000000000,
+        "max": 10000000000,
         "score": [
             -3,
             2,
-            -50
+            -4
         ]
     },
     "supervisorrate": {

src/res/moneyclear.json

@@ -8,5 +8,37 @@
     "descript": "金额除以10000",
     "maxmoney": 10000000000,
     "divisor": 10000
+  },
+  "ha_hnszfcgw_htgg": {
+    "descript": "金额除以10000",
+    "maxmoney": 10000000000,
+    "divisor": 10000
+  },
+  "hn_hnszbtbjgw_zbhxrgs2": {
+    "descript": "金额除以10000",
+    "maxmoney": 10000000000,
+    "divisor": 10000
+  },
+  "a_zgzfcgw_bid_tender_new": {
+    "descript": "金额除以10000",
+    "maxmoney": 10000000000,
+    "divisor": 10000
+  },
+  "a_zgzfcgw_bid_tender_new_7_9": {
+    "descript": "金额除以10000",
+    "maxmoney": 10000000000,
+    "divisor": 10000
+  },
+  "a_zgzfcgw_bid_tender_new_4_6": {
+    "descript": "金额除以10000",
+    "maxmoney": 10000000000,
+    "divisor": 10000
+  },
+  "a_zgzfcgw_bid_tender_new_1_3": {
+    "descript": "金额除以10000",
+    "maxmoney": 10000000000,
+    "divisor": 10000
   }
+
+
 }

udpcreateindex/src/biddingall.go

@@ -84,6 +84,10 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 		// 	tmp = make(map[string]interface{})
 		// 	continue
 		// }
+		if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive != "" { //bidding中有敏感词，不生索引
+			tmp = make(map[string]interface{})
+			continue
+		}
 		update := map[string]interface{}{}
 		del := map[string]interface{}{} //记录extract没有值而bidding中有值的字段
 		//对比方法----------------

udpcreateindex/src/biddingindex.go

@@ -142,9 +142,14 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		// 	tmp = make(map[string]interface{})
 		// 	continue
 		// }
+		if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive != "" { //bidding中有敏感词，不生索引
+			tmp = make(map[string]interface{})
+			continue
+		}
 		update := map[string]interface{}{} //要更新的mongo数据
 		//对比方法----------------
 		tid := qutil.BsonIdToSId(tmp["_id"])
+
 		if eMap[tid] != nil {
 			compare = eMap[tid]
 			if qutil.IntAll(compare["dataging"]) == 1 { //extract中dataging=1不生索引

udpcreateindex/src/config.json

@@ -94,7 +94,7 @@
         "db": "mxs"
     },
     "elastic": {
-        "addr": "http://192.168.3.128:9800",
+        "addr": "http://192.168.3.11:9800",
         "pool": 12
     }
 }

udpcreateindex/src/task.go

@@ -11,7 +11,7 @@ import (
 
 func task_index() {
 	c := cron.New()
-	c.AddFunc("20 30 5 * * *", func() { task_projects() })
+	//c.AddFunc("20 30 5 * * *", func() { task_projects() })
 	//c.AddFunc("0 30 * * * *", func() { task_biddingfile() }) //每30分钟执行一次
 	//c.AddFunc("0 22 14 * * *", func() { task_qyxyindex() })