抽取控制程序 - 抽取程序 - 修改完毕

data_monitoring/listen_data/src/main.go

@@ -105,7 +105,7 @@ func main()  {
 	}
 	save_mgo.InitPool()
 
-	exportSensitiveTestData()
+
 	return
 
 

src/config.json

@@ -33,14 +33,7 @@
     "brandgoods": false,
     "pricenumber":true,
     "udptaskid": "60b493c2e138234cb4adb640",
-    "udpport": "1484",
-    "nextNode": [
-        {
-            "addr": "127.0.0.1",
-            "port": 1485,
-            "memo": "抽取城市"
-        }
-    ],
+    "udpport": "6601",
     "esconfig": {
         "available": false,
         "AccessID": "",

src/jy/extract/extract.go

@@ -2425,8 +2425,8 @@ func checkFields(tmp map[string]interface{}) map[string]interface{} {
 	//	tmp["bidendtime"] = bidendtime_str
 	//}
 
-	jyhref:= fmt.Sprintf(JYUrl, qu.CommonEncodeArticle("content", BsonTOStringId(tmp["_id"])))
-	tmp["jyhref"] = jyhref
+	//jyhref:= fmt.Sprintf(JYUrl, qu.CommonEncodeArticle("content", BsonTOStringId(tmp["_id"])))
+	//tmp["jyhref"] = jyhref
 
 	return tmp
 }

src/jy/extract/extractudp.go

@@ -17,11 +17,9 @@ import (
 )
 
 var Udpclient mu.UdpClient //udp对象
-var nextNodes []map[string]interface{}
 
 //udp通知抽取
 func ExtractUdp() {
-	nextNodes = qu.ObjArrToMapArr(ju.Config["nextNode"].([]interface{}))
 	Udpclient = mu.UdpClient{Local: ":" + qu.ObjToString(ju.Config["udpport"]), BufSize: 1024}
 	Udpclient.Listen(processUdpMsg)
 }
@@ -58,28 +56,13 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 				if sid == "" || eid == "" {
 					log.Debug("err", "sid=", sid, ",eid=", eid)
 				} else {
-					udpinfo, _ := rep["key"].(string)
+					udpinfo, _ := rep["stype"].(string)
 					if udpinfo == "" {
 						udpinfo = "udpok"
 					}
-					go Udpclient.WriteUdp([]byte(udpinfo), mu.OP_NOOP, ra)
-					log.Debug("udp通知抽取id段", sid, " ", eid)
 					ExtractByUdp(sid, eid, ra)
-					for _, m := range nextNodes {
-						by, _ := json.Marshal(map[string]interface{}{
-							"gtid":  sid,
-							"lteid": eid,
-							"stype": qu.ObjToString(m["stype"]),
-						})
-						err := Udpclient.WriteUdp(by, mu.OP_TYPE_DATA, &net.UDPAddr{
-							IP:   net.ParseIP(m["addr"].(string)),
-							Port: qu.IntAll(m["port"]),
-						})
-						if err != nil {
-							log.Debug(err)
-						}
-					}
-					log.Debug("udp通知抽取完成,eid=", eid)
+					log.Debug("抽取完成udp通知抽取id段",udpinfo, sid, "~", eid)
+					Udpclient.WriteUdp([]byte(udpinfo), mu.OP_NOOP, ra)
 				}
 			}
 		}

udpextract/src/main.go

@@ -53,14 +53,14 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 			sid, _ := mapInfo["gtid"].(string)
 			eid, _ := mapInfo["lteid"].(string)
 			if sid == "" || eid == "" {
-				log.Println("接收id段-err ", "sid=", sid, ",eid=", eid)
+				log.Println("接收id段异常-err ", "sid=", sid, ",eid=", eid)
 			} else {
 				udpinfo, _ := mapInfo["key"].(string)
 				if udpinfo == "" {
 					udpinfo = "udpok"
 				}
 				go udpclient.WriteUdp([]byte(udpinfo), mu.OP_NOOP, ra)
-				log.Println("接收当前段落，udp通知抽取-需拆分",len(extractNode),"组", sid, "~", eid)
+				log.Println("接收当前段落，udp通知抽取-需拆分",len(extractNode),"组", sid, "~~", eid)
 				udplock.Lock()
 				resetExtractLevel() //重置状态
 				extractLevel["sid"]=sid
@@ -69,7 +69,6 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 
 				//拆分段落方法
 				splitArr:=splitIdMethod(sid,eid)
-				log.Println(splitArr)
 				if len(splitArr)!=len(extractNode){//直接发送整段
 					log.Println("段落划分异常...请检查程序...")
 				}
@@ -190,5 +189,5 @@ func sendNextNode(sid string,eid string)  {
 			log.Println(err)
 		}
 	}
-	log.Println("udp通知抽取完成...",sid,"~",eid)
-}
+	log.Println("udp通知抽取完成...通知下阶段udp-敏感词,补城市",sid,"~",eid)
+}

udps/main.go

@@ -22,8 +22,8 @@ func main() {
 	flag.IntVar(&p, "p", 1784, "端口")
 	flag.IntVar(&tmptime, "tmptime", 0, "时间查询")
 	flag.StringVar(&tmpkey, "tmpkey", "", "时间字段")
-	flag.StringVar(&id1, "gtid", "1fffffffffffffffffffffff", "gtid")
-	flag.StringVar(&id2, "lteid", "9fffffffffffffffffffffff", "lteid")
+	flag.StringVar(&id1, "gtid", "6152c2c71a75b8f446966fd5", "gtid")
+	flag.StringVar(&id2, "lteid", "6152c3f21a75b8f446967574", "lteid")
 	flag.StringVar(&ids, "ids", "", "id1,id2")
 	flag.StringVar(&stype, "stype", "biddingall", "stype,传递类型")
 	flag.StringVar(&bkey, "bkey", "", "bkey,加上此参数表示不生关键词和摘要")