去重逻辑

udpfilterdup/src/config.json

@@ -1,6 +1,6 @@
 {
     "udpport": ":1485",
-    "dupdays": 2,
+    "dupdays": 5,
     "mongodb": {
         "addr": "172.17.145.163:27080",
         "pool": 15,

udpfilterdup/src/datamap.go

@@ -54,7 +54,6 @@ func NewDatamap(days int, lastid string) *datamap {
 	if lastid == "" {
 		return dm
 	}
-
 	//初始化加载数据
 	sess := mgo.GetMgoConn()
 	defer mgo.DestoryMongoConn(sess)
@@ -65,7 +64,6 @@ func NewDatamap(days int, lastid string) *datamap {
 		//|| qutil.ObjToString(tmp["subtype"]) == "变更"
 		if qutil.IntAll(tmp["repeat"]) == 1  {
 			continuSum++
-
 		} else {
 			cm := tmp["comeintime"] //时间单位
 			//cm := tmp["publishtime"]
@@ -84,7 +82,6 @@ func NewDatamap(days int, lastid string) *datamap {
 				data := dm.data[k]
 				if data == nil {
 					data = []*Info{}
-					//log.Println(k)
 				}
 				data = append(data, info)
 				dm.data[k] = data
@@ -123,16 +120,11 @@ func NewInfo(tmp map[string]interface{}) *Info {
 	//info.winner = qutil.ObjToString(tmp["winner"])
 	info.publishtime = qutil.Int64All(tmp["publishtime"])
 
-
-
-
 	info.bidopentime  = qutil.Int64All(tmp["bidopentime"])
 	info.agencyaddr	 = qutil.ObjToString(tmp["agencyaddr"])
 	info.detail		= qutil.ObjToString(tmp["detail"])
 	info.site	 = qutil.ObjToString(tmp["site"])
 
-
-
 	return info
 }
 
@@ -213,7 +205,7 @@ L:
 								break L
 							}
 						}
-					}else {//非变更数据判重处理
+					}else {
 						n:=0 //三要素
 						m:=0 //二要素
 						x:=0 //四要素

udpfilterdup/src/main.go

@@ -31,7 +31,7 @@ var (
 	dupdays      = 5                      //初始化判重范围
 	DM           *datamap                 //判重数据
 	FilterRegexp = regexp.MustCompile("^_$")
-	lastid       = ""  //起始数据1条
+	lastid       = ""
 
 
 	FilterRegTitle = regexp.MustCompile("[0-9一二三四五六七八九十零１２３４５６７８９](次|包|标段)")
@@ -44,7 +44,7 @@ var (
 
 func init() {
 	flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
-	flag.Parse()// 类似于命令台 scanf
+	flag.Parse()
 	//172.17.145.163:27080
 	util.ReadConfig(&Sysconfig)
 	nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
@@ -77,31 +77,21 @@ func mainTest()  {
 
 	sess := mgo.GetMgoConn()
 	defer mgo.DestoryMongoConn(sess)
-
 	res, _ := mgo.Find(extract, nil, nil, nil, false, -1, -1)
 	res_copy, _ := mgo.Find(extract_copy, nil, nil, nil, false, -1, -1)
-
-	//fmt.Println(len(*res),len(*res_copy))
-
 	m1 :=map[string]int{} //新版本
 	m2 :=map[string]int{} //老版本
 	for _,v:=range *res{
 		m1[(v["_id"].(bson.ObjectId).Hex())]= util.IntAll(v["repeat"])
 	}
-
 	for _,v:=range *res_copy{
 		m2[(v["_id"].(bson.ObjectId).Hex())]= util.IntAll(v["repeat"])
 	}
 	fmt.Println(len(m1),len(m2))
-
-
 	n1:=0
 	n2:=0
-
-
 	var arr1 []string
 	var arr2 []string
-
 	for k,v:=range m2{
 		if m1[k]!=v{
 			if m1[k]==0{
@@ -114,7 +104,6 @@ func mainTest()  {
 			}
 		}
 	}
-
 	for _,v:=range arr1 {
 		log.Println(v)
 	}
@@ -129,8 +118,6 @@ func mainTest()  {
 
 	log.Println("0:1 对应数量",n1)
 	log.Println("1:0 对应数量",n2)
-
-
 }
 
 
@@ -198,9 +185,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
 
 	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
 
-		//fmt.Println(util.ObjToString(tmp["subtype"]))
 		if util.ObjToString(tmp["subtype"]) == "变更" {
-			//fmt.Println("变更信息")
 			//continue
 		}
 
@@ -218,7 +203,6 @@ func task(data []byte, mapInfo map[string]interface{}) {
 
 			//是否为无效数据
 			if invalidData(info.buyer,info.projectname,info.projectcode) {
-				//
 				inV_n++
 				mapLock.Lock()
 				updateExtract = append(updateExtract, []map[string]interface{}{
@@ -234,12 +218,9 @@ func task(data []byte, mapInfo map[string]interface{}) {
 
 				if len(updateExtract) > 500 {
 					mgo.UpdateBulk(extract, updateExtract...)
-					//mgo.UpdateBulk(bidding, updateBidding...)
-					//updateExtract, updateBidding = [][]map[string]interface{}{}, [][]map[string]interface{}{}
 					updateExtract = [][]map[string]interface{}{}
 				}
 				mapLock.Unlock()
-
 			}else  {
 				b, id := DM.check(info)
 				if b { //有重复,生成更新语句，更新抽取和更新招标
@@ -260,11 +241,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
 
 					if len(updateExtract) > 500 {
 						mgo.UpdateBulk(extract, updateExtract...)
-						//mgo.UpdateBulk(bidding, updateBidding...)
-						//updateExtract, updateBidding = [][]map[string]interface{}{}, [][]map[string]interface{}{}
 						updateExtract = [][]map[string]interface{}{}
-
-
 					}
 					mapLock.Unlock()
 				} else {