备份

udpfilterdup/src/README.md

@@ -52,7 +52,51 @@
 
 
 
-
+{
+    "udpport": ":1785",
+    "dupdays": 5,
+    "mongodb": {
+        "addr": "172.17.4.187:27083",
+        "pool": 5,
+        "db": "qfw",
+        "extract": "result_file_20200410",
+        "extract_back": "result_file_20200409",
+        "site": {
+            "dbname": "qfw",
+            "coll": "site"
+        }
+    },
+    "jkmail": {
+        "to": "zhengkun@topnet.net.cn,zhangjinkun@topnet.net.cn",
+        "api": "http://10.171.112.160:19281/_send/_mail"
+    },
+    "nextNode": [
+        {
+            "addr": "172.17.145.179",
+            "port": 1782,
+            "stype": "project",
+            "memo": "合并项目"
+        },
+        {
+            "addr": "127.0.0.1",
+            "port": 1783,
+            "stype": "bidding",
+            "memo": "创建招标数据索引new"
+        }
+    ],
+    "threads": 1,
+    "isMerger": false,
+    "isSort":true,
+    "lowHeavy":false,
+    "timingTask":false,
+    "timingSpanDay": 3,
+    "timingPubScope": 720,
+    "specialwords": "(重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研)",
+    "specialtitle_0": "(包|标段|标包)[(（]?[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９][)）]?",
+    "specialtitle_1": "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](次|包|标段|标包|批)",
+    "specialtitle_2": "项目[(（][0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９][)）]",
+    "beifen": "[(（]?[0-9一二三四五六七八九十零１２３４５６７８９再][)）]?[子分]?[次批标包]|重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研"
+}
 
 
 

udpfilterdup/src/config.json

@@ -1,9 +1,9 @@
 {
-    "udpport": ":11485",
-    "dupdays": 7,
+    "udpport": ":11995",
+    "dupdays": 5,
     "mongodb": {
         "addr": "172.17.4.187:27083",
-        "pool": 10,
+        "pool": 5,
         "db": "qfw",
         "extract": "result_file_20200410",
         "extract_back": "result_file_20200409",
@@ -13,7 +13,7 @@
         }
     },
     "jkmail": {
-        "to": "zhangjinkun@topnet.net.cn",
+        "to": "zhengkun@topnet.net.cn,zhangjinkun@topnet.net.cn",
         "api": "http://10.171.112.160:19281/_send/_mail"
     },
     "nextNode": [
@@ -30,14 +30,13 @@
             "memo": "创建招标数据索引new"
         }
     ],
-
     "threads": 1,
     "isMerger": false,
-    "isSort":false,
+    "isSort":true,
     "lowHeavy":false,
-    "timingTask":true,
+    "timingTask":false,
     "timingSpanDay": 3,
-    "timingPubScope": 1080,
+    "timingPubScope": 720,
     "specialwords": "(重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研)",
     "specialtitle_0": "(包|标段|标包)[(（]?[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９][)）]?",
     "specialtitle_1": "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](次|包|标段|标包|批)",

udpfilterdup/src/main.go

@@ -53,6 +53,7 @@ var (
 )
 
 func init() {
+	//5ea9a4800000000000000000
 	flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
 	flag.StringVar(&sid, "sid", "", "开始id")
 	flag.StringVar(&eid, "eid", "", "结束id")
@@ -107,6 +108,9 @@ func init() {
 
 func main() {
 
+	//TestXiuFu()
+	//return
+
 	go checkMapJob()
 	updport := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
@@ -133,8 +137,9 @@ func mainT() {
 		go timedTaskDay()
 		time.Sleep(99999 * time.Hour)
 	} else {
-		sid = "5c2c10fda5cb26b9b75e6f7f"
-		eid = "5e976e4a50b5ea296ef376b9"
+		//2020-4-24
+		sid = "5ea1bb800000000000000000"
+		eid = "5ea9a4800000000000000000"
 		log.Println("正常判重测试开始")
 		log.Println(sid, "---", eid)
 		mapinfo := map[string]interface{}{}
@@ -339,6 +344,10 @@ func task(data []byte, mapInfo map[string]interface{}) {
 					}
 				}
 
+
+				log.Println("最终结果","目标id:",info.id)
+
+
 				//重复数据打标签
 				updateExtract = append(updateExtract, []map[string]interface{}{
 					repeat_idMap,
@@ -812,7 +821,7 @@ func timedTaskOnce() {
 							"dataging":      0,
 						},
 					},
-				})
+			})
 			}else {
 				updateExtract = append(updateExtract, []map[string]interface{}{
 					map[string]interface{}{

udps/main.go

@@ -18,7 +18,7 @@ var startDate, endDate string
 
 func main() {
 	ip, p, tmptime, tmpkey, id1, id2, stype, q, bkey, param := "", 0, 0, "", "", "", "", "", "", ""
-	flag.StringVar(&startDate, "start", "", "开始日期2006-01-02")
+	flag.StringVar(&startDate, "start", "2020-04-30", "开始日期2006-01-02")
 	flag.StringVar(&endDate, "end", "", "结束日期2006-01-02")
 	flag.StringVar(&ip, "ip", "127.0.0.1", "ip")
 	flag.IntVar(&p, "p", 0, "端口")