前置条件 完善

udpfilterdup/src/datamap.go

@@ -311,17 +311,21 @@ L:
 					//前置条件3 - 标题相关,均含有关键词
 					if ((info.titleSpecialWord && v.titleSpecialWord) || (info.specialWord && v.specialWord)) &&
 						len([]rune(v.title)) > 10 && len([]rune(info.title)) > 10 && v.title != "" && info.title != "" {
-						//判断是否为 1、 重招组
 
-
-
-						if !(strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title)) {
-							continue //无包含关系
+						letter1,letter2:=v.title,info.title
+						res, _ := regexp.Compile("[0-9a-zA-Z]+");
+						if res.MatchString(letter1)||res.MatchString(letter2) {
+							letter1=convertArabicNumeralsAndLetters(letter1)
+							letter2=convertArabicNumeralsAndLetters(letter2)
 						}
-						if strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title) {
+						if strings.Contains(letter1,"重新招标")|| strings.Contains(letter2,"重新招标"){
+							letter1,letter2=dealWithSpecialPhrases(letter1,letter2)
+						}
+						if !(strings.Contains(letter1, letter2) || strings.Contains(letter2, letter1)) {
+							continue
+						}else {
 							reason = reason + "标题关键词且包含关系"
-							//继续二级金额判断
-							if !againRepeat(v, info) {
+							if !againRepeat(v, info) {//继续二级金额判断
 								b = true
 								source = v
 								reasons = reason
@@ -330,7 +334,6 @@ L:
 						}
 					}
 
-
 					//新增快速数据过少判重
 					if LowHeavy {
 						repeat := false

udpfilterdup/src/main.go

@@ -51,6 +51,7 @@ var (
 )
 
 func init() {
+	
 	flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
 	flag.StringVar(&sid, "sid", "", "开始id")
 	flag.StringVar(&eid, "eid", "", "结束id")