修改

udpfilterdup/src/datamap.go

@@ -28,7 +28,7 @@ type Info struct {
 	publishtime    int64   //发布时间
 	comeintime     int64   //入库时间
 	bidopentime    int64   //开标时间
-	agencyaddr     string  //开标地点
+	bidopenaddress string  //开标地点
 
 	site string //站点
 	href string //正文的url
@@ -218,7 +218,7 @@ func NewInfo(tmp map[string]interface{}) *Info {
 	info.publishtime = qutil.Int64All(tmp["publishtime"])
 	info.comeintime = qutil.Int64All(tmp["comeintime"])
 	info.bidopentime = qutil.Int64All(tmp["bidopentime"])
-	info.agencyaddr = qutil.ObjToString(tmp["agencyaddr"])
+	info.bidopenaddress = qutil.ObjToString(tmp["bidopenaddress"])
 	info.site = qutil.ObjToString(tmp["site"])
 	info.href = qutil.ObjToString(tmp["href"])
 	info.repeatid = qutil.ObjToString(tmp["repeatid"])
@@ -795,7 +795,7 @@ func tenderRepeat_A(v *Info, info *Info, reason string) (bool, string) {
 		ss = ss + "p9(开标时间)-"
 		p9 = true
 	}
-	if v.agencyaddr != "" && v.agencyaddr == info.agencyaddr {
+	if v.bidopenaddress != "" && v.bidopenaddress == info.bidopenaddress {
 		ss = ss + "p10(开标地点)-"
 		p10 = true
 	}
@@ -839,9 +839,9 @@ func tenderRepeat_B(v *Info, info *Info, reason string) (bool, string) {
 	if v.bidopentime != 0 && v.bidopentime == info.bidopentime {
 		m++
 	}
-	if v.agencyaddr != "" && v.agencyaddr == info.agencyaddr {
-		m++
-	}
+	// if v.bidopenaddress != "" && v.bidopenaddress == info.bidopenaddress {
+	// 	m++
+	// }
 	if len([]rune(v.title)) > 10 && len([]rune(info.title)) > 10 &&
 		(strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title)) {
 		m++
@@ -872,7 +872,7 @@ func tenderRepeat_C(v *Info, info *Info) bool {
 	if v.bidopentime != 0 && info.bidopentime != 0 && v.bidopentime != info.bidopentime {
 		return true
 	}
-	if v.agencyaddr != "" && info.agencyaddr != "" && v.agencyaddr != info.agencyaddr {
+	if v.bidopenaddress != "" && info.bidopenaddress != "" && v.bidopenaddress != info.bidopenaddress {
 		return true
 	}
 

udpfilterdup/src/main.go

@@ -107,8 +107,8 @@ func mainT() {
 		ObjectId("5e09c05f0cf41612e0626abc")
 	*/
 	log.Println("测试开始")
-	sid = "5da3f31aa5cb26b9b798d3aa"
-	eid = "5da418c4a5cb26b9b7e3e9a6"
+	sid = "596f127e5d11e1c7455cf2c0"
+	eid = "5970721a5d11e1c74565f930"
 	mapinfo := map[string]interface{}{}
 	if sid == "" || eid == "" {
 		log.Println("sid,eid参数不能为空")
@@ -199,6 +199,11 @@ func task(data []byte, mapInfo map[string]interface{}) {
 		if n%10000 == 0 {
 			log.Println("current:", n, tmp["_id"], "repeateN:", repeateN)
 		}
+		if util.IntAll(tmp["repeat"]) == 1 {
+			tmp = make(map[string]interface{})
+			repeateN++
+			continue
+		}
 		pool <- true
 		wg.Add(1)
 		go func(tmp map[string]interface{}) {
@@ -400,7 +405,7 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 	minTime, maxTime := int64(0), int64(0)
 	for tmp := make(map[string]interface{}); it.Next(&tmp); {
 		//取出最大最小时间
-		info_time:=tmp["comeintime"]
+		info_time := tmp["comeintime"]
 		if Is_Sort {
 			info_time = tmp["publishtime"]
 		}
@@ -885,7 +890,7 @@ func basicDataScore(v *Info, info *Info) bool {
 	if v.bidopentime != 0 {
 		m++
 	}
-	if v.agencyaddr != "" {
+	if v.bidopenaddress != "" {
 		m++
 	}
 	if v.agency != "" {
@@ -916,7 +921,7 @@ func basicDataScore(v *Info, info *Info) bool {
 	if info.bidopentime != 0 {
 		n++
 	}
-	if info.agencyaddr != "" {
+	if info.bidopenaddress != "" {
 		n++
 	}
 	if info.agency != "" {