Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

udpfilterdup/src/datamap.go

@@ -14,29 +14,30 @@ import (
 
 type Info struct {
 	id                 string
-	title              string
-	area               string
-	city               string
-	subtype            string
-	buyer              string
-	agency             string //代理机构
-	winner             string //中标单位
-	budget             float64 //预算金额
-	bidamount		   float64//中标金额
-	projectname        string
-	projectcode        string
-	publishtime        int64
-	comeintime         int64
-	bidopentime        int64 //开标时间
-	agencyaddr		   string//开标地点
-	detail		   	   string//招标内容
-	site			   string//站点
-	href			   string//正文的url
-	titleSpecialWord bool //标题特殊次
-	specialWord bool	  //再次判断的特殊次
+	title              string		//标题
+	area               string		//省份
+	city               string		//城市
+	subtype            string		//信息类型
+	buyer              string		//采购单位
+	agency             string 		//代理机构
+	winner             string 		//中标单位
+	budget             float64 		//预算金额
+	bidamount		   float64		//中标金额
+	projectname        string		//项目名称
+	projectcode        string		//项目编号
+	publishtime        int64		//发布时间
+	comeintime         int64		//采集时间
+	bidopentime        int64 		//开标时间
+	agencyaddr		   string		//开标地点
+	detail		   	   string		//招标内容
+	site			   string		//站点
+	href			   string		//正文的url
+	titleSpecialWord bool 			//标题特殊词
+	specialWord bool	 			//再次判断的特殊词
 }
 
 var datelimit = float64(432000)
+var reason string //判重记录
 
 type datamap struct {
 	lock   sync.Mutex //锁
@@ -132,9 +133,10 @@ func NewInfo(tmp map[string]interface{}) *Info {
 	info.href	 = qutil.ObjToString(tmp["href"])
 	return info
 }
+// 486 396 315
 
-
-func (d *datamap) check(info *Info) (b bool, id string) {
+func (d *datamap) check(info *Info) (b bool, id string,reasons string) {
+	reason = ""
 	d.lock.Lock()
 	defer d.lock.Unlock()
 	keys := []string{}
@@ -149,219 +151,84 @@ L:
 		data := d.data[k]
 		if len(data) > 0 { //对比
 			for _, v := range data {
-				//正常重复
-				if v.id == info.id {
-
-					return false, v.id
+				if v.id == info.id {//正常重复
+					return false, v.id,""
 				}
 				if math.Abs(qutil.Float64All(v.publishtime-info.publishtime)) > datelimit {
-					continue
-				}
-				if v.agency != "" && info.agency != "" && v.agency != info.agency {
-					continue
+					continue   //是否为5天内数据
 				}
 
-				if info.subtype==v.subtype {
-					if info.subtype == "变更" {
-						//以下为新增方法 ， 变更数据判重处理   v为原数据  info为目标数据
-						if info.publishtime<v.publishtime{
-							continue
-						}
-						if info.titleSpecialWord&&info.title!=v.title&&v.title!="" {
-							continue
-						}
-						if v.projectcode != info.projectcode&&len([]rune(info.projectcode)) >=10&&v.projectcode!=""{
-							continue
-						}
+				//if v.agency != "" && info.agency != "" && v.agency != info.agency {
+				//	continue
+				//}
 
-						//同城判定有效
-						first_judge:= false
-						if (v.projectcode != ""&&v.projectcode==info.projectcode&&v.projectname != ""&&v.projectname==info.projectname)||
-							(v.projectcode != ""&&v.projectcode==info.projectcode&&v.bidopentime != 0&&v.bidopentime==info.bidopentime)||
-							(v.title != ""&&v.title==info.title&&v.agencyaddr != ""&&v.agencyaddr==info.agencyaddr&&v.bidopentime != 0&&v.bidopentime==info.bidopentime) {
-							first_judge = true
+				//是否走站内判重
+				if info.site != "" && v.site == info.site {
+					//独有判重... 待定  jsondata配置
 
-						}
-						//3/6等判断
-						n := 0
-						if v.title != "" && v.title == info.title {
-							n++
-						}
-						if v.projectname != "" && v.projectname == info.projectname {
-							n++
-						}
-						if v.projectcode != "" && v.projectcode == info.projectcode {
-							n++
-						}
-						if v.bidopentime != 0 && v.bidopentime == info.bidopentime {
-							n++
-						}
-						if v.agencyaddr != "" && v.agencyaddr == info.agencyaddr {
-							n++
-						}
-						if v.detail != "" && v.detail == info.detail {
-							n++
-						}
-						t:= judgeCityType(v.area,info.area,v.city,info.city)
-
-						//同站点url判重
-						if info.site != "" && v.site == info.site {
-							if v.href==info.href&&info.href!="" {
-								b = true
-								id = v.id
-								log.Println("href满足过滤:",info.id)
-								break L
+					//站内url判重
+					//if v.href==info.href&&info.href!="" {
+					//	b = true
+					//	id = v.id
+					//	break L
+					//}
+				}
+				//类型分组-相同类型继续
+				if info.subtype==v.subtype {
+					//代理机构相同-非空相等
+					if v.agency != "" && info.agency != "" && v.agency == info.agency {
+						reason = fmt.Sprintf(reason,"同机构,")
+						//省份相同
+						if v.area!=""&&info.area!=""&&v.area==info.area {
+							reason = fmt.Sprintf(reason,"同省,")
+							if v.city!=""&&info.city!=""&&v.city==info.city {
+								reason = fmt.Sprintf(reason,"同市,")
+								if quickHeavyMethodTwo(v,info) {
+									b = true
+									id = v.id
+									reasons = reason
+									break L
+								}
 							}else {
-								continue
-							}
-						}
-
-
-						if n>=3||first_judge==true {
-							if t==2 {//同城
-								if conditionAgainRepeat(v,info) {
-									continue
+								reason = fmt.Sprintf(reason,"非同市,")
+								if quickHeavyMethodOne(v,info) {
+									b = true
+									id = v.id
+									reasons = reason
+									break L
 								}
-								b = true
-								id = v.id
-								break L
 							}
-						}
-					}else {
-						n:=0 //三要素
-						m:=0 //二要素
-						x:=0 //四要素
-						if info.buyer != "" &&v.buyer == info.buyer {
-							n++
-							x++
-						}
-						if info.projectname != ""&&v.projectname == info.projectname {
-							n++
-							m++
-							x++
-						}
-						if info.projectcode != ""&&v.projectcode == info.projectcode {
-							n++
-							m++
-							x++
-						}
-						if info.title != ""&&v.title == info.title {
-							x++
-						}
-
-						t:= judgeCityType(v.area,info.area,v.city,info.city)
-						c_1 :=conditionTitle(v.title,info.title) //标题满足
-						c_2 :=conditionNum(v.projectcode,info.projectcode) //编号满足
-						c_3 :=conditionTitleBuyer(v.title,info.title,v.buyer,info.buyer) //标题+采购单位
-						c_4 :=conditionCodeTime(v.bidopentime,info.bidopentime,v.projectcode,info.projectcode) //编号+开标时间
-
-
-						//同站点url判重
-						if info.site != "" && v.site == info.site {
-							if v.href==info.href&&info.href!="" {
+						}else {
+							reason = fmt.Sprintf(reason,"非同省,")
+							if quickHeavyMethodOne(v,info) {
 								b = true
 								id = v.id
-								log.Println("href满足过滤:",info.id)
+								reasons = reason
 								break L
-							}else {
-								continue
 							}
 						}
-
-						
-
-						site_b := false
-						if info.site != "" {
-							for _,v := range siteArr {
-								if info.site==v["addr"] {
-									site_b=true
-									break
-								}
-							}
-						}
-						if info.site != "" && v.site == info.site&&site_b {
-							if n>1||c_1||c_2 {
-								if conditionAgainRepeat(v,info) {
-									continue
-								}
+					}else {
+						reason = fmt.Sprintf(reason,"非同机构,")
+						//含有城市
+						if v.city != "" && info.city != ""&&v.city==info.city {
+							reason = fmt.Sprintf(reason,"同市,")
+							if quickHeavyMethodTwo(v,info) {
 								b = true
 								id = v.id
-								log.Println("站点满足过滤")
+								reasons = reason
 								break L
 							}
 						}else {
-							if info.titleSpecialWord&&info.title!=v.title&&v.title!="" {
-								continue
-							}
-							if v.projectcode != info.projectcode&&len([]rune(info.projectcode)) >=10&&v.projectcode!=""{
-								continue
-							}
-							//先决条件满足三要素,条件4
-							if n==3||c_4{
-								if conditionAgainRepeat(v,info) {
-									continue
-								}
+							reason = fmt.Sprintf(reason,"非同市,")
+							if quickHeavyMethodOne(v,info) {
 								b = true
 								id = v.id
+								reasons = reason
 								break L
 							}
-
-							//城市判断
-							if t==0||t==1 { //最少一个全国
-								if c_1 && (c_2||n>1) {
-
-									if conditionAgainRepeat(v,info) {
-										continue
-									}
-									b = true
-									id = v.id
-									break L
-								}
-								if c_2&&x>2{
-
-									if conditionAgainRepeat(v,info) {
-										continue
-									}
-									b = true
-									id = v.id
-									break L
-								}
-							}else if t==2 { //	省-市
-								if c_1||c_2||n>1 {
-									//新增二次判断逻辑
-									if conditionAgainRepeat(v,info) {
-										continue
-									}
-									b = true
-									id = v.id
-									break L
-								}
-							}else if t==3 {//	!省 !市
-								if (c_1&&n>1)||(c_2&&x>2){
-
-									if conditionAgainRepeat(v,info) {
-										continue
-									}
-									b = true
-									id = v.id
-									break L
-								}
-
-							}else if t==4 {//	省 !市
-								if m>1||(c_1&&m>0)||(c_2&&x>1)||(c_3&&n>1){
-
-									if conditionAgainRepeat(v,info) {
-										continue
-									}
-									b = true
-									id = v.id
-									break L
-								}
-							}else  {
-
-							}
 						}
 					}
+
 				}
 			}
 		}
@@ -389,102 +256,363 @@ L:
 }
 
 
-//判断是否同城等情况
-func judgeCityType(v string, info string,v_c string,info_c string) (t int) {
-
-	t=0
-	if  (v=="全国"||v=="")&&(info=="全国"||info=="") {//均为全国
-		t=0
-	}else if  v!="全国"&&info!="全国"&&v!=""&&info!=""&&
-		v_c!="全国"&&info_c!="全国"&&v_c!=""&&info_c!=""{//均非全国
-		if v==info &&v_c==info_c { //同省同城
-			t=2
-		}else if v!=info&&v_c!=info_c{//非同省非同城
-			t=3
-		}else {//同省非同城
-			t=4
+//判重方法1
+func quickHeavyMethodOne(v *Info ,info *Info) bool {
+
+	if info.subtype=="招标"||info.subtype=="邀标"||info.subtype=="询价"||
+		info.subtype=="竞谈"||info.subtype=="单一"||info.subtype=="竞价"||
+		info.subtype=="变更"||info.subtype=="其他" {
+		//招标结果
+		if tenderRepeat_A(v,info) {
+			if tenderRepeat_C(v,info) {
+				return false
+			}else {
+				reason = fmt.Sprintf(reason,"---招标类")
+				return true
+			}
+		}else {
+			return false
+		}
+
+	}else if info.subtype=="中标"||info.subtype=="成交"||info.subtype=="废标"||info.subtype=="流标"{
+		//中标结果
+		if winningRepeat_A(v,info) {
+			if winningRepeat_C(v,info) {
+				return false
+			}else {
+				reason = fmt.Sprintf(reason,"---中标类")
+				return true
+			}
+		}else {
+			return false
+		}
+
+	}else if info.subtype=="合同"||info.subtype=="验收"||info.subtype=="违规"{
+		//合同
+		if contractRepeat_A(v,info) {
+			if contractRepeat_C(v,info) {
+				return false
+			}else {
+				reason = fmt.Sprintf(reason,"---合同类")
+				return true
+			}
+		}else {
+			return false
 		}
-	}else {//有且一个全国 ，包含多种情况，
-		t=1
+	}else {
+
 	}
-	return t
+
+	return false
 }
+//判重方法2
+func quickHeavyMethodTwo(v *Info ,info *Info) bool {
 
-//条件一 标题
-func conditionTitle(t1 string, t2 string) bool {
+	//相同
+	if v.agency==info.agency &&v.agency!=""&&info.agency!="" {
 
-	if len([]rune(t1))>10 && len([]rune(t2))>10&&
-		(strings.Contains(t1, t2)||strings.Contains(t2, t1)) {
-		return true
+		if info.subtype=="招标"||info.subtype=="邀标"||info.subtype=="询价"||
+			info.subtype=="竞谈"||info.subtype=="单一"||info.subtype=="竞价"||
+			info.subtype=="变更"||info.subtype=="其他" {
+			//招标结果
+			if tenderRepeat_B(v,info) {
+				if tenderRepeat_C(v,info) { //有不同
+					return false
+				}else {
+					reason = fmt.Sprintf(reason,"---招标类")
+					return true
+				}
+			}else{
+				return false
+			}
+
+		}else if info.subtype=="中标"||info.subtype=="成交"||info.subtype=="废标"||info.subtype=="流标"{
+			//中标结果
+			if winningRepeat_B(v,info) {
+				if winningRepeat_C(v,info) { //有不同
+					return false
+				}else {
+					reason = fmt.Sprintf(reason,"---中标类")
+					return true
+				}
+			}else{
+				return false
+			}
+
+		}else if info.subtype=="合同"||info.subtype=="验收"||info.subtype=="违规"{
+			//合同
+			if contractRepeat_B(v,info) {
+				if contractRepeat_C(v,info) { //有不同
+					return false
+				}else {
+					reason = fmt.Sprintf(reason,"---合同类")
+					return true
+				}
+			}else {
+				return false
+			}
+		}else {
+
+		}
+	}
+
+	//不同
+	if v.agency!=info.agency &&v.agency!=""&&info.agency!="" {
+		return false
+	}
+	//机构最少一个为空
+	if v.agency==""||info.agency=="" {
+		if quickHeavyMethodOne(v,info) {
+			reason = fmt.Sprintf(reason,"---机构最少一个空")
+			return true
+		}else {
+			return false
+		}
 	}
 	return false
 }
-//条件二 项目编号
-func conditionNum(c1 string ,c2 string) bool {
 
-	if c1 == c2&&len([]rune(c1)) >=10 {
+
+
+//招标_A
+func tenderRepeat_A(v *Info ,info *Info) bool {
+
+	var ss string
+	p1,p2,p3,p4,p9,p10,p11 := false,false,false,false,false,false,false
+	if v.projectname!=""&&v.projectname==info.projectname {
+		ss = fmt.Sprintf(ss,"p1(名称)-")
+		p1 = true
+	}
+	if v.buyer!=""&&v.buyer==info.buyer {
+		ss = fmt.Sprintf(ss,"p2(单位)-")
+		p2 = true
+	}
+	if v.projectcode!=""&&v.projectcode==info.projectcode {
+		ss = fmt.Sprintf(ss,"p3(编号)-")
+		p3 = true
+	}
+	if v.budget!=0&&v.budget==info.budget {
+		ss = fmt.Sprintf(ss,"p4(预算)-")
+		p4 = true
+	}
+	if v.bidopentime!=0&&v.bidopentime==info.bidopentime {
+		ss = fmt.Sprintf(ss,"p9(开标时间)-")
+		p9 = true
+	}
+	if v.agencyaddr!=""&&v.agencyaddr==info.agencyaddr {
+		ss = fmt.Sprintf(ss,"p10(开标地点)-")
+		p10 = true
+	}
+	if len([]rune(v.title))>10 && len([]rune(info.title))>10&&
+		(strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title)) {
+		ss = fmt.Sprintf(ss,"p11(标题)-")
+		p11 = true
+	}
+
+	if (p1&&p2&&p3)||(p1&&p2&&p4)||(p1&&p2&&p9)||
+		(p1&&p2&&p10)||(p1&&p3&&p9)||(p1&&p3&&p10)||
+		(p1&&p4&&p9)||(p1&&p4&&p10)||(p2&&p3&&p4)||
+		(p2&&p3&&p9)||(p2&&p3&&p10)||(p2&&p3&&p11)||
+		(p2&&p4&&p9)||(p2&&p4&&p10)||(p2&&p4&&p11)||
+		(p3&&p4&&p9)||(p3&&p4&&p10)||(p3&&p4&&p11)||
+		(p4&&p9&&p10)||(p4&&p9&&p11)||(p9&&p10&&p11) {
+		reason = fmt.Sprintf(reason,"满足招标A，3要素组合-",ss,",")
 		return true
 	}
 	return false
 }
-//条件三 采购单位+标题
-func conditionTitleBuyer(t1 string ,t2 string,b1 string,b2 string) bool {
 
-	if t1==t2&&b1==b2 {
-		return true
+//招标_B
+func tenderRepeat_B(v *Info ,info *Info) bool {
+
+	m,n :=0,0
+	if v.projectname!=""&&v.projectname==info.projectname {
+		m++
+		n++
+	}
+	if v.buyer!=""&&v.buyer==info.buyer {
+		m++
+	}
+	if v.projectcode!=""&&v.projectcode==info.projectcode {
+		m++
+	}
+	if v.budget!=0&&v.budget==info.budget {
+		m++
+	}
+	if v.bidopentime!=0&&v.bidopentime==info.bidopentime {
+		m++
+	}
+	if v.agencyaddr!=""&&v.agencyaddr==info.agencyaddr {
+		m++
+	}
+	if len([]rune(v.title))>10 && len([]rune(info.title))>10&&
+		(strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title)) {
+		m++
+		n++
+	}
+	if m>=2 {
+		if n==2 &&m==2 {
+			return false
+		}else {
+			reason = fmt.Sprintf(reason,"满足招标B，七选二,")
+			return true
+		}
 	}
 	return false
 }
-//条件四 编号+开标时间
-func conditionCodeTime(t1 int64 ,t2 int64,c1 string,c2 string) bool {
+//招标_C
+func tenderRepeat_C(v *Info ,info *Info) bool {
+
+	if v.budget!=0&&info.budget!=0&&v.budget!=info.budget {
+
+		return true
+	}
+	//原始地址...
 
-	if c1 != ""&&c1==c2&&t1 != 0&&t1==t2&&len([]rune(c1)) >=10 {
+	if v.bidopentime!=0&&info.bidopentime!=0&&v.bidopentime!=info.bidopentime {
+		return true
+	}
+	if v.agencyaddr!=""&&info.agencyaddr!=""&&v.agencyaddr!=info.agencyaddr {
 		return true
 	}
 	return false
 }
 
-func conditionAgainRepeat(v *Info ,info *Info) bool {
-	//同省情况下
-	//if v.area==info.area&&v.area!="全国"&&v.area!=""&&v.city==info.city {
-	//
-	//}
-
-	// 编号过短+金额  不等时，暂存
+//中标_A
+func winningRepeat_A(v *Info ,info *Info) bool {
 
+	var ss string
+	p1,p2,p3,p5,p6,p11 := false,false,false,false,false,false
+	if v.projectname!=""&&v.projectname==info.projectname {
+		ss = fmt.Sprintf(ss,"p1(标题)-")
+		p1 = true
+	}
+	if v.buyer!=""&&v.buyer==info.buyer {
+		ss = fmt.Sprintf(ss,"p2(单位)-")
+		p2 = true
+	}
+	if v.projectcode!=""&&v.projectcode==info.projectcode {
+		ss = fmt.Sprintf(ss,"p3(编号)-")
+		p3 = true
+	}
+	if v.bidamount!=0&&v.bidamount==info.bidamount {
+		ss = fmt.Sprintf(ss,"p5(中标金)-")
+		p5 = true
+	}
+	if v.winner!=""&&v.winner==info.winner {
+		ss = fmt.Sprintf(ss,"p6(中标人)-")
+		p6 = true
+	}
+	if len([]rune(v.title))>10 && len([]rune(info.title))>10&&
+		(strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title)) {
+		ss = fmt.Sprintf(ss,"p11(标题)-")
+		p11 = true
+	}
 
+	if (p1&&p2&&p3)||(p1&&p2&&p5)||(p1&&p2&&p6)||
+		(p1&&p3&&p5)||(p1&&p3&&p6)||(p1&&p5&&p6)||
+		(p2&&p3&&p5)||(p2&&p3&&p6)||(p2&&p3&&p11)||
+		(p2&&p5&&p6)||(p2&&p5&&p11)||(p2&&p6&&p11)||
+		(p3&&p5&&p6)||(p3&&p5&&p11)||(p3&&p6&&p11)||
+		(p5&&p6&&p11){
+		reason = fmt.Sprintf(reason,"满足中标A，3要素组合-",ss,",")
+		return true
+	}
 
+	return false
+}
+//中标_B
+func winningRepeat_B(v *Info ,info *Info) bool {
 
-	//相同采购单位下
-	if info.buyer != "" &&v.buyer == info.buyer {
-		//满足标题
-		if  len([]rune(v.title)) >=10&&len([]rune(info.title)) >=10&&v.title!=info.title&&(info.specialWord||v.specialWord){
+	m,n :=0,0
+	if v.projectname!=""&&v.projectname==info.projectname {
+		m++
+		n++
+	}
+	if v.buyer!=""&&v.buyer==info.buyer {
+		m++
+	}
+	if v.projectcode!=""&&v.projectcode==info.projectcode {
+		m++
+	}
+	if v.bidamount!=0&&v.bidamount==info.bidamount {
+		m++
+	}
+	if v.winner!=""&&v.winner==info.winner {
+		m++
+	}
+	if len([]rune(v.title))>10 && len([]rune(info.title))>10&&
+		(strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title)) {
+		m++
+		n++
+	}
+	if m>=2 {
+		if n==2 &&m==2 {
+			return false
+		}else {
+			reason = fmt.Sprintf(reason,"满足中标B，六选二,")
 			return true
 		}
+	}
+	return false
+}
 
-		if info.subtype=="招标"||info.subtype=="邀标"||info.subtype=="询价"||
-			info.subtype=="竞谈"||info.subtype=="单一"||info.subtype=="竞价"||
-			info.subtype=="其他"||info.subtype=="变更" {
-			//预算金额满足条件
-			if v.budget!=info.budget&&v.budget!=0&&info.budget!=0 {
-				return true
-			}
-		}else if info.subtype=="中标"||info.subtype=="成交"||info.subtype=="废标"||
-			info.subtype=="流标"||info.subtype=="合同"||info.subtype=="验收"||
-			info.subtype=="违规"{
-			//中标金额单位满足条件
-			if (v.bidamount!=info.bidamount&&v.bidamount!=0&&info.bidamount!=0)||
-				(v.winner!=info.winner&&v.winner!=""&&info.winner!=""){
-				return true
-			}
-		}else {
+//中标_C
+func winningRepeat_C(v *Info ,info *Info) bool {
 
-		}
+	if v.bidamount!=0&&info.bidamount!=0&&v.bidamount!=info.bidamount {
+		return true
 	}
+	if v.winner!=""&&info.winner!=""&&v.winner!=info.winner {
+		return true
+	}
+	//原始地址...
+
 
 	return false
 }
 
+//合同_A
+func contractRepeat_A(v *Info ,info *Info) bool {
+
+	if tenderRepeat_A(v,info) {
+		return true
+	}
+	if winningRepeat_A(v,info) {
+		return true
+	}
+	return false
+}
+
+//合同_B
+func contractRepeat_B(v *Info ,info *Info) bool {
+
+	if tenderRepeat_B(v,info) {
+		return true
+	}
+	if winningRepeat_B(v,info) {
+		return true
+	}
+	return false
+}
+
+//合同_C
+func contractRepeat_C(v *Info ,info *Info) bool {
+
+	if tenderRepeat_C(v,info) {
+		return true
+	}
+	if winningRepeat_C(v,info) {
+		return true
+	}
+	return false
+}
+
+
+
+
+
+
 func (d *datamap) update(t int64) {
 	//每天0点清除历史数据
 	d.keymap = d.GetLatelyFiveDay(t)

udpfilterdup/src/main.go

@@ -66,12 +66,12 @@ func init() {
 
 
 	//测试临时注释
-	dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
-	//加载数据
-	DM = NewDatamap(dupdays, lastid)
-	FilterRegTitle = regexp.MustCompile(util.ObjToString(Sysconfig["specialwords"]))
-	FilterRegTitle_1 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_1"]))
-	FilterRegTitle_2 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_2"]))
+	//dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
+	////加载数据
+	//DM = NewDatamap(dupdays, lastid)
+	//FilterRegTitle = regexp.MustCompile(util.ObjToString(Sysconfig["specialwords"]))
+	//FilterRegTitle_1 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_1"]))
+	//FilterRegTitle_2 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_2"]))
 
 
 
@@ -90,7 +90,8 @@ func init() {
 }
 
 //新增一个方法 判断
-func mainTest()  {
+func main()  {
+
 	//log.Println("1")
 	//代码copy数据
 	//sessTest :=mgoTest.GetMgoConn()
@@ -103,7 +104,7 @@ func mainTest()  {
 	//
 	//res_test := sessTest.DB("qfw").C("bidding").Find(mongodb.ObjToMQ(`{"comeintime":{"$gte": 1571025600, "$lte": 1571976000}}`, true)).Iter()
 	//res :=sess.DB("extract_kf").C("a_testbidding")
-	//
+	//5
 	//
 	//
 	//
@@ -158,10 +159,13 @@ func mainTest()  {
 	n2:=0
 	n3:=0
 	n4:=0
+	n5:=0
+	n6:=0
 
 	var arr1 []string
 	var arr2 []string
 	for k,v:=range m1{
+
 		if m2[k]==1&&v==0{//0:1
 			n1++
 			arr2 = append(arr2,fmt.Sprintf("目标_id:%s",k))
@@ -176,24 +180,28 @@ func mainTest()  {
 		if m2[k]==1&&v==1{//1:1
 			n4++
 		}
+		if m2[k]==-1&&v==0{ //0:-1
+			n5++
+		}
+		if m2[k]==-1&&v==1{//1:-1
+			n6++
+		}
+
 	}
-	//打印 1：0情况 少打印 300条    38841
+	//打印 1：0情况    66989
 	mm:=0
 	for _,v:=range arr1 {
 		mm++
-		if mm%130==0 {
+		if mm%222==0 {
 			log.Println(v)
 		}
 	}
 
 	log.Println("分割线---------------")
 	log.Println("分割线---------------")
-	log.Println("分割线---------------")
-	log.Println("分割线---------------")
-	log.Println("分割线---------------")
 
 
-	//打印 1：0情况 少打印  1000条   15518
+	//打印 0：1情况  8729
 	nn:=0
 	for _,v:=range arr2 {
 		nn++
@@ -206,14 +214,14 @@ func mainTest()  {
 	log.Println("V1 1:0---",n2)
 	log.Println("V1 0:0---",n3)
 	log.Println("V1 1:1---",n4)
-
-
+	log.Println("V1 0:-1---",n5)
+	log.Println("V1 1:-1---",n6)
 
 }
 
 
 
-func main() {
+func mainTest() {
 	go checkMapJob()
 
 	updport := Sysconfig["udpport"].(string)
@@ -276,6 +284,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
 
 	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
 
+
 		if util.ObjToString(tmp["subtype"]) == "变更" {
 			//continue
 		}
@@ -313,9 +322,9 @@ func task(data []byte, mapInfo map[string]interface{}) {
 				}
 				mapLock.Unlock()
 			}else  {
-				b, id := DM.check(info)
+				b, id,reason := DM.check(info)
 				if b { //有重复,生成更新语句，更新抽取和更新招标
-
+					log.Println("判重具体原因：",reason)
 					repeateN++
 					mapLock.Lock()
 					updateExtract = append(updateExtract, []map[string]interface{}{