判重 合并

udpfilterdup/src/config.json

@@ -5,8 +5,8 @@
         "addr": "192.168.3.207:27092",
         "pool": 5,
         "db": "extract_kf",
-        "extract": "zk_test",
-        "extract_back": "zk_test",
+        "extract": "zk_zk_test",
+        "extract_back": "zk_zk_test",
         "site": {
             "dbname": "zhaolongyue",
             "coll": "site"

udpfilterdup/src/dataMethod.go

@@ -161,10 +161,9 @@ func leadingElementSame(v *Info, info *Info) bool {
 	if info.title != "" && v.title == info.title {
 		isok++
 	}
-	if v.agency == info.agency {
+	if v.agency == info.agency &&info.agency != "" {
 		isok++
 	}
-
 	if v.winner == info.winner&&info.winner != "" {
 		isok++
 	}

udpfilterdup/src/dataMethodHeavy.go

@@ -240,18 +240,9 @@ func tenderRepeat_C(v *Info, info *Info) bool {
 	if v.budget != 0 && info.budget != 0 && v.budget != info.budget {
 		return true
 	}
-	//原始地址...
-	//if v.buyer != "" && info.buyer != "" && v.buyer != info.buyer {
-	//	return true
-	//}
-
 	if v.bidopentime != 0 && info.bidopentime != 0 && isBidopentimeInterval(info.bidopentime,v.bidopentime) {
 		return true
 	}
-	//if v.bidopenaddress != "" && info.bidopenaddress != "" && v.bidopenaddress != info.bidopenaddress {
-	//	return true
-	//}
-
 	return false
 }
 
@@ -344,7 +335,7 @@ func winningRepeat_B(v *Info, info *Info, reason string) (bool, string) {
 func winningRepeat_C(v *Info, info *Info) bool {
 
 	if v.bidamount != 0 && info.bidamount != 0 && isBidWinningAmount(v.bidamount,info.bidamount) {
-		//避免抽错金额- title+name+winner
+		//避免抽错金额-
 		if ((v.projectcode!=""&&info.projectcode!=""&&v.projectcode==info.projectcode)||
 			(v.contractnumber!=""&&info.contractnumber!=""&&v.contractnumber==info.contractnumber)) &&
 			(v.winner!=""&&info.winner!=""&&v.winner==info.winner) {

udpfilterdup/src/main.go

@@ -128,8 +128,8 @@ func mainT() {
 		time.Sleep(99999 * time.Hour)
 	} else {
 		//IdType = true  //打开id字符串模式
-		sid = "5da3f2c5a5cb26b9b79847f0"
-		eid = "5da40bdaa5cb26b9b7bea480"
+		sid = "5da3f3a2a5cb26b9b799aa65"
+		eid = "5da40b26a5cb26b9b7bde10b"
 		log.Println("正常判重测试开始")
 		log.Println(sid, "---", eid)
 		mapinfo := map[string]interface{}{}

udpfilterdup/src1/main.go

@@ -105,7 +105,7 @@ func init() {
 }
 
 
-func main() {
+func mainT() {
 
 	go checkMapJob()
 	updport := Sysconfig["udpport"].(string)
@@ -121,7 +121,8 @@ func main() {
 
 
 //测试组人员使用
-func mainT() {
+func main() {
+
 	if TimingTask {
 		log.Println("定时任务测试开始")
 		go timedTaskDay()
@@ -212,7 +213,9 @@ func task(data []byte, mapInfo map[string]interface{}) {
 			log.Println("current:", n, tmp["_id"], "repeateN:", repeateN)
 		}
 		source := util.ObjToMap(tmp["jsondata"]) //前置-jsondata判重
-		if util.IntAll((*source)["sourcewebsite"]) == 1 {
+
+
+		if util.IntAll((*source)["sourcewebsite"]) == 1 && tmp["attach_text"]!=nil{
 			repeateN++
 			updateExtract = append(updateExtract, []map[string]interface{}{
 				map[string]interface{}{