判重~备份更新

src/dataMethod.go

@@ -73,7 +73,7 @@ func againRepeat(v *Info, info *Info, site bool) bool {
 	if isBidWinningAmount(v.bidamount, info.bidamount) && v.bidamount != 0 && info.bidamount != 0 {
 		return true
 	}
-	if deleteExtraSpace(v.winner) != deleteExtraSpace(info.winner) && v.winner != "" && info.winner != "" {
+	if v.winner != info.winner && v.winner != "" && info.winner != "" {
 		return true
 	}
 	if v.contractnumber != "" && info.contractnumber != "" && v.contractnumber != info.contractnumber {
@@ -104,7 +104,7 @@ func againContainSpecialWord(v *Info, info *Info) bool {
 	if isBidWinningAmount(v.bidamount, info.bidamount) && v.bidamount != 0 && info.bidamount != 0 {
 		return true
 	}
-	if deleteExtraSpace(v.winner) != deleteExtraSpace(info.winner) && v.winner != "" && info.winner != "" {
+	if v.winner != info.winner && v.winner != "" && info.winner != "" {
 		return true
 	}
 	if v.contractnumber != "" && info.contractnumber != "" && v.contractnumber != info.contractnumber {
@@ -139,12 +139,12 @@ func dealTitleSpecial(title1 string, title2 string) bool {
 	}
 	//根据提取的结果，在进行清洗
 	if str1 != "" {
-		str1 = deleteExtraSpace(str1)
+		str1 = deleteExtraSpaceName(str1)
 		str1 = cleanNameReg_0.ReplaceAllString(str1, "")
 		str1 = convertArabicNumeralsAndLetters(str1)
 	}
 	if str2 != "" {
-		str2 = deleteExtraSpace(str2)
+		str2 = deleteExtraSpaceName(str2)
 		str2 = cleanNameReg_0.ReplaceAllString(str2, "")
 		str2 = convertArabicNumeralsAndLetters(str2)
 	}
@@ -156,7 +156,7 @@ func dealTitleSpecial(title1 string, title2 string) bool {
 }
 
 //删除中标单位字符串中多余的空格(含tab)
-func deleteExtraSpace(s string) string {
+func deleteExtraSpaceName(s string) string {
 	//删除字符串中的多余空格，有多个空格时，仅保留一个空格
 	s1 := strings.Replace(s, "  ", " ", -1)      //替换tab为空格
 	regstr := "\\s{2,}"                          //两个及两个以上空格的正则表达式
@@ -279,6 +279,23 @@ func leadingElementSame(v *Info, info *Info) bool {
 	return false
 }
 
+//前置0 竞品要素简易计算
+func jingPinElementSame(v *Info, info *Info) bool {
+	if info.projectname != "" && v.projectname != info.projectname {
+		return false
+	}
+	if info.buyer != "" && v.buyer != info.buyer {
+		return false
+	}
+	if info.projectcode != "" && v.projectcode != info.projectcode {
+		return false
+	}
+	if v.agency != info.agency {
+		return false
+	}
+	return true
+}
+
 //buyer的优先级
 func buyerIsContinue(v *Info, info *Info) bool {
 	if !isTheSameDay(info.publishtime, v.publishtime) {
@@ -377,10 +394,13 @@ func IsJpHref(href string) bool {
 
 //验证竞品是否重复
 func confirmJingPinIsRepeatData(v *Info, info *Info) bool {
+
 	//标题验证~是否有关联~是否需要清洗数据
 	if v.c_title != "" && info.c_title != "" { //标题相似判断
 		if !(strings.Contains(v.c_title, info.c_title) || strings.Contains(info.c_title, v.c_title)) {
-			return false
+			if !jingPinElementSame(v, info) {
+				return false
+			}
 		}
 		if !isTheSameDay(v.publishtime, info.publishtime) {
 			return false
@@ -391,7 +411,7 @@ func confirmJingPinIsRepeatData(v *Info, info *Info) bool {
 		if isBidWinningAmount(v.bidamount, info.bidamount) && v.bidamount != 0.0 && info.bidamount != 0.0 {
 			return false
 		}
-		if v.winner != "" && info.winner != "" && deleteExtraSpace(v.winner) != deleteExtraSpace(info.winner) {
+		if v.winner != "" && info.winner != "" && v.winner != info.winner {
 			return false
 		}
 		if v.contractnumber != "" && info.contractnumber != "" && v.contractnumber != info.contractnumber {

src/dataMethodHeavy.go

@@ -283,7 +283,7 @@ func winningRepeat_A(v *Info, info *Info, reason string) (bool, string) {
 		ss = ss + "p5-中标金-"
 		p5 = true
 	}
-	if v.winner != "" && deleteExtraSpace(v.winner) == deleteExtraSpace(info.winner) {
+	if v.winner != "" && v.winner == info.winner {
 		ss = ss + "p6-中标人-"
 		p6 = true
 	}
@@ -326,7 +326,7 @@ func winningRepeat_B(v *Info, info *Info, reason string) (bool, string) {
 	if v.bidamount != 0 && !isBidWinningAmount(v.bidamount, info.bidamount) {
 		m++
 	}
-	if v.winner != "" && deleteExtraSpace(v.winner) == deleteExtraSpace(info.winner) {
+	if v.winner != "" && v.winner == info.winner {
 		m++
 	}
 	if len([]rune(v.title)) > 10 && len([]rune(info.title)) > 10 &&
@@ -357,7 +357,7 @@ func winningRepeat_C(v *Info, info *Info) bool {
 		}
 		return true
 	}
-	if v.winner != "" && info.winner != "" && deleteExtraSpace(v.winner) != deleteExtraSpace(info.winner) {
+	if v.winner != "" && info.winner != "" && v.winner != info.winner {
 		return true
 	}
 	return false

src/datamap.go

@@ -215,7 +215,7 @@ func NewInfo(tmp map[string]interface{}) *Info {
 	info.projectcode = qutil.ObjToString(tmp["projectcode"])
 	info.city = qutil.ObjToString(tmp["city"])
 	info.agency = qutil.ObjToString(tmp["agency"])
-	info.winner = qutil.ObjToString(tmp["winner"])
+	info.winner = deleteExtraSpaceName(qutil.ObjToString(tmp["winner"]))
 	info.budget = qutil.Float64All(tmp["budget"])
 	info.bidamount = qutil.Float64All(tmp["bidamount"])
 	info.publishtime = qutil.Int64All(tmp["publishtime"])

src/historyRepeat.go

@@ -37,7 +37,7 @@ func historyRepeat() {
 		it_last := task_sess.DB(task_mgo.DbName).C(task_collName).Find(&q).Sort("-_id").Iter()
 		isRepeatStatus := false
 		for tmp := make(map[string]interface{}); it_last.Next(&tmp); {
-			is_repeat_status := util.IntAll(tmp["is_repeat_status"])
+			is_repeat_status := util.IntAll(tmp["repeat_status"])
 			if is_repeat_status == 1 {
 				lteid = util.ObjToString(tmp["lteid"])
 				log.Println("查询的最后一个已标记的任务lteid:", lteid)
@@ -55,6 +55,7 @@ func historyRepeat() {
 			continue
 		}
 		log.Println("查询完毕-找到有标记的lteid-先睡眠5分钟", gtid, lteid)
+
 		if isUpdateSite {
 			initSite()
 		}

src/increaseRepeat.go

@@ -160,8 +160,10 @@ func increaseRepeat(mapInfo map[string]interface{}) {
 	fmt.Println("")
 	log.Println("当前~判重~结束~", total, "重复~", repeatN)
 	//更新Ocr的标记
-	updateOcrFileData(mapInfo["lteid"].(string))
-	time.Sleep(10 * time.Second)
+	//updateOcrFileData(mapInfo["lteid"].(string))
+	updateProcessUdpIdsInfo(qu.ObjToString(mapInfo["gtid"]), qu.ObjToString(mapInfo["lteid"]))
+
+	time.Sleep(15 * time.Second)
 	//任务完成，开始发送广播通知下面节点
 	log.Println("判重任务完成发送udp")
 	for _, to := range nextNode {
@@ -184,7 +186,40 @@ func increaseRepeat(mapInfo map[string]interface{}) {
 	}
 }
 
-//更新ocr表
+//更新流程记录id段落
+func updateProcessUdpIdsInfo(sid string, eid string) {
+	//判重有合并操作~所以要联合查询
+	query := map[string]interface{}{
+		"gtid": map[string]interface{}{
+			"$gte": sid,
+		},
+		"lteid": map[string]interface{}{
+			"$lte": eid,
+		},
+	}
+	datas, _ := task_mgo.Find(task_collName, query, nil, nil)
+	if len(datas) > 0 {
+		log.Println("开始更新流程段落记录~~", len(datas), "段")
+		for _, v := range datas {
+			up_id := BsonTOStringId(v["_id"])
+			if up_id != "" {
+				update := map[string]interface{}{
+					"$set": map[string]interface{}{
+						"dataprocess":   6,
+						"repeat_status": 1,
+						"updatetime":    time.Now().Unix(),
+					},
+				}
+				task_mgo.UpdateById(task_collName, up_id, update)
+				log.Println("流程段落记录~~更新完毕~", update)
+			}
+		}
+	} else {
+		log.Println("未查询到记录id段落~", query)
+	}
+}
+
+//更新ocr表~弃用
 func updateOcrFileData(cur_lteid string) {
 	//更新ocr 分类表-判重的状态
 	log.Println("开始更新Ocr表-标记", cur_lteid)

src/main.go

@@ -156,12 +156,10 @@ func init() {
 }
 
 func mainT() {
-
 	IsFull = true
 	//AddGroupPool = newAddGroupPool()
 	//go AddGroupPool.addGroupData()
 	//fullDataRepeat() //全量判重
-
 	increaseRepeat(map[string]interface{}{
 		"gtid":  "12ec61170ae152a3c2310f02",
 		"lteid": "92ec61170ae152a3c2310f02",
@@ -174,11 +172,13 @@ func mainT() {
 
 //主函数
 func main() {
+
 	go checkMapJob()
 	updport := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
 	udpclient.Listen(processUdpMsg)
 	log.Println("Udp服务监听", updport)
+
 	if TimingTask {
 		log.Println("正常历史部署")
 		go historyRepeat()
@@ -231,12 +231,35 @@ func getRepeatTask() {
 	for {
 		if len(taskList) > 0 {
 			updatelock.Lock()
-			mapInfo := taskList[0]
-			if mapInfo != nil {
-				increaseRepeat(mapInfo) //判重方法
+			len_list := len(taskList)
+			if len_list > 1 {
+				first_id := taskList[0]["gtid"]
+				end_id := taskList[len_list-1]["lteid"]
+				if first_id != "" && end_id != "" {
+					log.Println("合并段落~正常~", first_id, "~", end_id)
+					increaseRepeat(map[string]interface{}{
+						"gtid":  first_id,
+						"lteid": end_id,
+					})
+					taskList = taskList[len_list:]
+					log.Println("此段落结束当前任务池...", len(taskList), taskList)
+				} else {
+					log.Println("合并段落~错误~正常取段落~~~")
+					mapInfo := taskList[0]
+					if mapInfo != nil {
+						increaseRepeat(mapInfo) //判重方法
+					}
+					taskList = taskList[1:]
+					log.Println("此段落结束当前任务池...", len(taskList), taskList)
+				}
+			} else {
+				mapInfo := taskList[0]
+				if mapInfo != nil {
+					increaseRepeat(mapInfo) //判重方法
+				}
+				taskList = taskList[1:]
+				log.Println("此段落结束当前任务池...", len(taskList), taskList)
 			}
-			taskList = taskList[1:]
-			log.Println("此段落结束当前任务池...", len(taskList), taskList)
 			updatelock.Unlock()
 		} else {
 			time.Sleep(15 * time.Second)

src/updateMethod.go

@@ -6,22 +6,25 @@ import (
 )
 
 var sp = make(chan bool, 5)
-type updateInfo struct {//更新或新增通道
+
+type updateInfo struct { //更新或新增通道
 	updatePool chan []map[string]interface{}
-	saveSize   	int
+	saveSize   int
 }
+
 func newUpdatePool() *updateInfo {
-	update:=&updateInfo{make(chan []map[string]interface{}, 50000),200}
+	update := &updateInfo{make(chan []map[string]interface{}, 50000), 200}
 	return update
 }
 
 //临时~新增组
 type addGroupInfo struct {
-	pool chan map[string]interface{}
-	saveSize   	int
+	pool     chan map[string]interface{}
+	saveSize int
 }
+
 func newAddGroupPool() *addGroupInfo {
-	info:=&addGroupInfo{make(chan map[string]interface{}, 50000),200}
+	info := &addGroupInfo{make(chan map[string]interface{}, 50000), 200}
 	return info
 }
 
@@ -46,7 +49,7 @@ func (update *updateInfo) updateData() {
 				tmpArr = make([][]map[string]interface{}, update.saveSize)
 				tmpIndex = 0
 			}
-		case <-time.After(5 * time.Second)://无反应时每x秒检测一次
+		case <-time.After(5 * time.Second): //无反应时每x秒检测一次
 			if tmpIndex > 0 {
 				sp <- true
 				go func(dataArr [][]map[string]interface{}) {
@@ -82,7 +85,7 @@ func (info *addGroupInfo) addGroupData() {
 				tmpArr = make([]map[string]interface{}, info.saveSize)
 				tmpIndex = 0
 			}
-		case <-time.After(10 * time.Second)://无反应时每x秒检测一次
+		case <-time.After(7 * time.Second): //无反应时每x秒检测一次
 			if tmpIndex > 0 {
 				sp <- true
 				go func(dataArr []map[string]interface{}) {
@@ -96,4 +99,4 @@ func (info *addGroupInfo) addGroupData() {
 			}
 		}
 	}
-}
+}

work_repeat/src/main.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 )
+
 var zhb_key_list = []string{"budget", "buyer", "agency", "s_winner", "bidamount", "projectcode", "contractcode"}
 var packreg *regexp.Regexp
 var Mgo *MongodbSim
@@ -21,30 +22,31 @@ type dataSource struct {
 	projectname, projectcode, contractcode string
 	buyer, agency, s_winner                string
 	budget, bidamount                      float64
-	budget_isnull,bidamount_isnull 		   bool
+	budget_isnull, bidamount_isnull        bool
 	isrepeat                               bool
 	repeat_id_source                       string
 	repeat_id                              map[string]string
 	repeatText                             string
-	publishtime							   int64
+	publishtime                            int64
 }
 
 //var addr, dbname, table, startTime, endTime, sortType *string
-var addr, dbname, table,  sortType string
+var addr, dbname, table, sortType string
 var cycle int64
-var sysconfig    map[string]interface{} 	//配置文件
+var sysconfig map[string]interface{} //配置文件
 
-func initConfig()  {
+func initConfig() {
 	qu.ReadConfig(&sysconfig)
-	addr =  sysconfig["mgo_addr"].(string)
-	dbname =  sysconfig["mgo_db"].(string)
-	table =  sysconfig["mgo_table"].(string)
-	sortType =  sysconfig["mgo_sort"].(string)
-	month := qu.IntAllDef(sysconfig["cycle_month"],6)
+	addr = sysconfig["mgo_addr"].(string)
+	dbname = sysconfig["mgo_db"].(string)
+	table = sysconfig["mgo_table"].(string)
+	sortType = sysconfig["mgo_sort"].(string)
+	month := qu.IntAllDef(sysconfig["cycle_month"], 6)
 	cycle = qu.Int64All(month)
 }
+
 //创建mgo索引
-func createMgoIndex(){
+func createMgoIndex() {
 	mongoDBDialInfo := &mgo.DialInfo{
 		Addrs:    []string{addr},
 		Timeout:  60 * time.Second,
@@ -56,12 +58,9 @@ func createMgoIndex(){
 	}
 	coll := session.DB(dbname).C(table)
 	err = coll.EnsureIndexKey("publishtime")
-	fmt.Println("创建索引~publishtime",err)
-
-	//查询所有的已存在索引
-	//indexs, err := coll.Indexes()
-	//fmt.Println("indexs--------------:", indexs)
+	fmt.Println("创建索引~publishtime", err)
 }
+
 //初始化
 func init() {
 	initConfig()
@@ -77,9 +76,10 @@ func init() {
 	//packreg, _ = regexp.Compile(`([包标段][:：]?[a-zA-Z0-9①②ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ一二三四五六七八九十]|[a-zA-Z0-9①②ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ一二三四五六七八九十][包标段]){1,}`)
 	//packreg, _ = regexp.MustCompile("([a-zA-Z0-9①②ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ一二三四五六七八九十](包|标|段)[：:]?)")
 }
+
 //主函数
 func main() {
-	if addr == "" || dbname == "" || table == "" ||  sortType == ""  {
+	if addr == "" || dbname == "" || table == "" || sortType == "" {
 		log.Println("参数配置有误")
 		fmt.Printf("数据库地址：%s\n数据库名称：%s\n表名：%s\n排序方式：%s\n", addr, dbname, table, sortType)
 		os.Exit(0)
@@ -106,7 +106,7 @@ func main() {
 		sort = "-publishtime"
 	}
 	//log.Println(sort)
-	q:=map[string]interface{}{}
+	q := map[string]interface{}{}
 	sess := Mgo.GetMgoConn()
 	defer Mgo.DestoryMongoConn(sess)
 
@@ -117,7 +117,7 @@ func main() {
 	//var filterReg = regexp.MustCompile("[`~!@#$^&*()=|{}':;,\\[\\].<>/?！￥…（）—【】‘；：”“。，、？%+_－-]")
 	index := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
-		info :=tmp["v_baseinfo"].(map[string]interface{})
+		info := tmp["v_baseinfo"].(map[string]interface{})
 		d := &dataSource{
 			_id:          BsonTOStringId(tmp["_id"]),
 			id:           qu.ObjToString(tmp["id"]),
@@ -133,11 +133,11 @@ func main() {
 			publishtime:  qu.Int64All(info["publishtime"]),
 			repeat_id:    map[string]string{},
 		}
-		if info["budget"]==nil{
-			d.budget_isnull=true
+		if info["budget"] == nil {
+			d.budget_isnull = true
 		}
-		if info["bidamount"]==nil{
-			d.bidamount_isnull=true
+		if info["bidamount"] == nil {
+			d.bidamount_isnull = true
 		}
 		//log.Println(tmp["_id"],tmp["title"],tmp["projectname"])
 		if index%10000 == 0 {
@@ -146,9 +146,9 @@ func main() {
 		listSource = append(listSource, d)
 		tmp = map[string]interface{}{}
 	}
-	log.Println("数据加载完成",len(listSource))
-	for i:=0;i<len(listSource);i++ {
-		for j:=0;j<len(listSource)-i-1;j++ {
+	log.Println("数据加载完成", len(listSource))
+	for i := 0; i < len(listSource); i++ {
+		for j := 0; j < len(listSource)-i-1; j++ {
 			if listSource[j].publishtime > listSource[j+1].publishtime {
 				listSource[j], listSource[j+1] = listSource[j+1], listSource[j]
 			}
@@ -162,35 +162,36 @@ func main() {
 			dd++
 		}
 		//更新数据
-		if len(a.repeat_id) ==0{
+		if len(a.repeat_id) == 0 {
 			Mgo.UpdateById(table, a._id,
 				map[string]interface{}{"$set": map[string]interface{}{
 					//重复数据看repeatid
-					"repeatid":     a.repeat_id_source, //和哪条数据重复id
-					"repeat":       a.isrepeat,         //本条数据是否重复数据
-					"repeattext":   a.repeatText,       //本数据被判重的原因
+					"repeatid":   a.repeat_id_source, //和哪条数据重复id
+					"repeat":     a.isrepeat,         //本条数据是否重复数据
+					"repeattext": a.repeatText,       //本数据被判重的原因
 				}})
-		}else {
-			if len(a.repeat_id) > 0{
-				arr:=[]string{}
-				for k,_:=range a.repeat_id{
-					arr = append(arr,k)
+		} else {
+			if len(a.repeat_id) > 0 {
+				arr := []string{}
+				for k, _ := range a.repeat_id {
+					arr = append(arr, k)
 				}
 				Mgo.UpdateById(table, a._id,
 					map[string]interface{}{"$set": map[string]interface{}{
 						//原始数据看repeatid_ids_str
-						"repeatid":     a.repeat_id_source, //和哪条数据重复id
-						"repeat":       a.isrepeat,         //本条数据是否重复数据
+						"repeatid": a.repeat_id_source, //和哪条数据重复id
+						"repeat":   a.isrepeat,         //本条数据是否重复数据
 						//"repeatid_ids": a.repeat_id,        //和我重复的数据都有哪些
-						"repeatid_ids_str": strings.Join(arr,","),
-						"repeattext":   a.repeatText,       //本数据被判重的原因
-					}})}
+						"repeatid_ids_str": strings.Join(arr, ","),
+						"repeattext":       a.repeatText, //本数据被判重的原因
+					}})
+			}
 		}
 		if i%1000 == 0 {
 			log.Println("已更新：", i)
 		}
 	}
-	log.Println("重复数据量：",dd)
+	log.Println("重复数据量：", dd)
 }
 
 var listSize = 20000
@@ -205,7 +206,7 @@ func dataItem() {
 		for j := i + 1; j < len(listSource); j++ {
 			b = listSource[j]
 			if sortType == "1" {
-				if publishtime_b_a(*a,*b){
+				if publishtime_b_a(*a, *b) {
 					// if b.isrepeat {
 					// 	continue
 					// }
@@ -216,8 +217,8 @@ func dataItem() {
 					// 	log.Println("sss", a.id, b.isrepeat, b.repeat_id)
 					// }
 				}
-			}else{
-				if publishtime_a_b(*a,*b){
+			} else {
+				if publishtime_a_b(*a, *b) {
 					// if b.isrepeat {
 					// 	continue
 					// }
@@ -241,14 +242,14 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 	case a.title == b.title: //标题相等
 		if pankong(a.contractcode) && pankong(b.contractcode) && a.contractcode != b.contractcode {
 
-		} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount  {
-			if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer)  && pankong(a.buyer) && pankong(b.buyer) {
+		} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount {
+			if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer) && pankong(a.buyer) && pankong(b.buyer) {
 				if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner {
 					b.repeat_id_source = a.id
 					a.repeat_id[b.id] = ""
 					b.isrepeat = true
 					b.repeatText = "标题相等 && bidamount && buyer && s_winner"
-				}else{
+				} else {
 					r := key_list(a, b)
 					if r {
 						b.repeat_id_source = a.id
@@ -257,12 +258,12 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 						b.repeatText = "标题相等 && bidamount && buyer && key_list"
 					}
 				}
-			} else if  pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner {
+			} else if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner {
 				b.repeat_id_source = a.id
 				a.repeat_id[b.id] = ""
 				b.isrepeat = true
 				b.repeatText = "标题相等 && bidamount && s_winner"
-			}else {
+			} else {
 				r := key_list(a, b)
 				if r {
 					b.repeat_id_source = a.id
@@ -271,7 +272,7 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 					b.repeatText = "标题相等 && bidamount && key_list"
 				}
 			}
-		}else if pankong(a.projectcode) && pankong(b.projectcode) && a.projectcode == b.projectcode {
+		} else if pankong(a.projectcode) && pankong(b.projectcode) && a.projectcode == b.projectcode {
 			r := key_list(a, b)
 			if r {
 				b.repeat_id_source = a.id
@@ -279,10 +280,10 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 				b.isrepeat = true
 				b.repeatText = "标题相等 && projectcode && key_list"
 			}
-		}else if !a.budget_isnull && !b.budget_isnull && a.budget == b.budget  {
-			if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer)  && pankong(a.buyer) && pankong(b.buyer) {
+		} else if !a.budget_isnull && !b.budget_isnull && a.budget == b.budget {
+			if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer) && pankong(a.buyer) && pankong(b.buyer) {
 				if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner {
-					if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount  {
+					if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount {
 						b.repeat_id_source = a.id
 						a.repeat_id[b.id] = ""
 						b.isrepeat = true
@@ -299,7 +300,7 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 					b.repeatText = "标题相等 && budget && key_list"
 				}
 			}
-		}   else {
+		} else {
 			//
 		}
 	case a.title != b.title: //标题不相等
@@ -312,10 +313,10 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 				if a.projectname == b.projectname {
 					if pankong(a.contractcode) && pankong(b.contractcode) && a.contractcode != b.contractcode {
 						//
-					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount  {
-						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner != b.s_winner{
+					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount {
+						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner != b.s_winner {
 
-						}else{
+						} else {
 							b.repeat_id_source = a.id
 							a.repeat_id[b.id] = ""
 							b.isrepeat = true
@@ -325,10 +326,10 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 						//a.repeat_id[b.id] = ""
 						//b.isrepeat = true
 						//b.repeatText = "标题不相等-->有分包 && projectname && bidamount"
-					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount != b.bidamount  {
+					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount != b.bidamount {
 						//
 					} else {
-						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner && !a.budget_isnull && !b.budget_isnull && a.budget == b.budget && (a.budget >=0 || b.budget >= 0) {
+						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner && !a.budget_isnull && !b.budget_isnull && a.budget == b.budget && (a.budget >= 0 || b.budget >= 0) {
 							b.repeat_id_source = a.id
 							a.repeat_id[b.id] = ""
 							b.isrepeat = true
@@ -338,7 +339,7 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 				} else { //项目名称包含
 					if pankong(a.contractcode) && pankong(b.contractcode) && a.contractcode != b.contractcode {
 						//
-					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount  {
+					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount {
 						if pankong(a.projectcode) && pankong(b.projectcode) && a.projectcode == b.projectcode {
 							b.repeat_id_source = a.id
 							a.repeat_id[b.id] = ""
@@ -349,13 +350,13 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 							a.repeat_id[b.id] = ""
 							b.isrepeat = true
 							b.repeatText = "标题不相等-->有分包 && projectname包含 && bidamount && s_winner"
-						} else if !a.budget_isnull && !b.budget_isnull && a.budget == b.budget  {
-							if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer)  && pankong(a.buyer) && pankong(b.buyer) {
+						} else if !a.budget_isnull && !b.budget_isnull && a.budget == b.budget {
+							if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer) && pankong(a.buyer) && pankong(b.buyer) {
 								b.repeat_id_source = a.id
 								a.repeat_id[b.id] = ""
 								b.isrepeat = true
 								b.repeatText = "标题不相等-->有分包 && projectname包含 && bidamount && budget && buyer"
-							} else if strings.Contains(a.agency, b.agency) || strings.Contains(b.agency, a.agency)  && pankong(a.agency) && pankong(b.agency) {
+							} else if strings.Contains(a.agency, b.agency) || strings.Contains(b.agency, a.agency) && pankong(a.agency) && pankong(b.agency) {
 								b.repeat_id_source = a.id
 								a.repeat_id[b.id] = ""
 								b.isrepeat = true
@@ -367,7 +368,7 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount != b.bidamount {
 						//
 					} else {
-						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner && !a.budget_isnull && !b.budget_isnull && a.budget == b.budget  {
+						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner && !a.budget_isnull && !b.budget_isnull && a.budget == b.budget {
 							b.repeat_id_source = a.id
 							a.repeat_id[b.id] = ""
 							b.isrepeat = true
@@ -382,20 +383,20 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 				if a.projectname == b.projectname {
 					if pankong(a.contractcode) && pankong(b.contractcode) && a.contractcode != b.contractcode {
 						//
-					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount  {
+					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount {
 						b.repeat_id_source = a.id
 						a.repeat_id[b.id] = ""
 						b.isrepeat = true
 						b.repeatText = "标题不相等-->无分包 && projectname && bidamount"
-					} else if !a.bidamount_isnull && !b.bidamount_isnull &&  a.bidamount != b.bidamount  {
+					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount != b.bidamount {
 						//
 					} else {
 						if pankong(a.projectcode) && pankong(b.projectcode) && a.projectcode == b.projectcode {
 							if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner != b.s_winner {
 
-							}else if !a.budget_isnull && !b.budget_isnull && a.budget != b.budget{
+							} else if !a.budget_isnull && !b.budget_isnull && a.budget != b.budget {
 
-							}else{
+							} else {
 								b.repeat_id_source = a.id
 								a.repeat_id[b.id] = ""
 								b.isrepeat = true
@@ -424,13 +425,13 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 							//	b.isrepeat = true
 							//	b.repeatText = "标题不相等-->无分包 && projectname && s_winner && key_list"
 							//}
-						} else if !a.budget_isnull && !b.budget_isnull && a.budget == b.budget  {
-							if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer)  && pankong(a.buyer) && pankong(b.buyer) {
+						} else if !a.budget_isnull && !b.budget_isnull && a.budget == b.budget {
+							if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer) && pankong(a.buyer) && pankong(b.buyer) {
 								b.repeat_id_source = a.id
 								a.repeat_id[b.id] = ""
 								b.isrepeat = true
 								b.repeatText = "标题不相等-->无分包 && projectname && budget && buyer"
-							} else if strings.Contains(a.agency, b.agency) || strings.Contains(b.agency, a.agency)  && pankong(a.agency) && pankong(b.agency) {
+							} else if strings.Contains(a.agency, b.agency) || strings.Contains(b.agency, a.agency) && pankong(a.agency) && pankong(b.agency) {
 								b.repeat_id_source = a.id
 								a.repeat_id[b.id] = ""
 								b.isrepeat = true
@@ -443,7 +444,7 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 				} else { //项目名称包含
 					if pankong(a.contractcode) && pankong(b.contractcode) && a.contractcode != b.contractcode {
 						//
-					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount  {
+					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount == b.bidamount {
 						if pankong(a.projectcode) && pankong(b.projectcode) && a.projectcode == b.projectcode {
 							b.repeat_id_source = a.id
 							a.repeat_id[b.id] = ""
@@ -454,13 +455,13 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 							a.repeat_id[b.id] = ""
 							b.isrepeat = true
 							b.repeatText = "标题不相等-->无分包 && projectname包含 && bidamount && s_winner"
-						} else if !a.budget_isnull && !b.budget_isnull && a.budget == b.budget  {
-							if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer)  && pankong(a.buyer) && pankong(b.buyer) {
+						} else if !a.budget_isnull && !b.budget_isnull && a.budget == b.budget {
+							if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer) && pankong(a.buyer) && pankong(b.buyer) {
 								b.repeat_id_source = a.id
 								a.repeat_id[b.id] = ""
 								b.isrepeat = true
 								b.repeatText = "标题不相等-->无分包 && projectname包含 && budget && buyer"
-							} else if strings.Contains(a.agency, b.agency) || strings.Contains(b.agency, a.agency)  && pankong(a.agency) && pankong(b.agency) {
+							} else if strings.Contains(a.agency, b.agency) || strings.Contains(b.agency, a.agency) && pankong(a.agency) && pankong(b.agency) {
 								b.repeat_id_source = a.id
 								a.repeat_id[b.id] = ""
 								b.isrepeat = true
@@ -471,10 +472,10 @@ func panchong(a, b dataSource) (c, d *dataSource) {
 						} else {
 							//
 						}
-					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount != b.bidamount  {
+					} else if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount != b.bidamount {
 						//
 					} else {
-						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner && !a.budget_isnull && !b.budget_isnull && a.budget == b.budget  {
+						if pankong(a.s_winner) && pankong(b.s_winner) && a.s_winner == b.s_winner && !a.budget_isnull && !b.budget_isnull && a.budget == b.budget {
 							b.repeat_id_source = a.id
 							a.repeat_id[b.id] = ""
 							b.isrepeat = true
@@ -496,19 +497,19 @@ func key_list(a, b dataSource) bool {
 		key := zhb_key_list[i]
 		switch key {
 		case "budget":
-			if !a.budget_isnull && !b.budget_isnull && a.budget != b.budget  {
+			if !a.budget_isnull && !b.budget_isnull && a.budget != b.budget {
 				return false
 			} else {
 				continue
 			}
 		case "buyer":
-			if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer)  && pankong(a.buyer) && pankong(b.buyer) {
+			if strings.Contains(a.buyer, b.buyer) || strings.Contains(b.buyer, a.buyer) && pankong(a.buyer) && pankong(b.buyer) {
 				continue
 			} else {
 				return false
 			}
 		case "agency":
-			if strings.Contains(a.agency, b.agency) || strings.Contains(b.agency, a.agency)  && pankong(a.agency) && pankong(b.agency) {
+			if strings.Contains(a.agency, b.agency) || strings.Contains(b.agency, a.agency) && pankong(a.agency) && pankong(b.agency) {
 				continue
 			} else {
 				return false
@@ -520,7 +521,7 @@ func key_list(a, b dataSource) bool {
 				continue
 			}
 		case "bidamount":
-			if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount != b.bidamount   {
+			if !a.bidamount_isnull && !b.bidamount_isnull && a.bidamount != b.bidamount {
 				return false
 			} else {
 				continue
@@ -541,17 +542,18 @@ func key_list(a, b dataSource) bool {
 	}
 	return true
 }
+
 //发布时间判断
 //正序
-func publishtime_b_a(a,b dataSource) bool{
-	return b.publishtime-a.publishtime < 86400 * 31 * cycle
+func publishtime_b_a(a, b dataSource) bool {
+	return b.publishtime-a.publishtime < 86400*31*cycle
 }
+
 //倒序
-func publishtime_a_b(a,b dataSource) bool {
-	return a.publishtime-b.publishtime < 86400 * 31 * cycle
+func publishtime_a_b(a, b dataSource) bool {
+	return a.publishtime-b.publishtime < 86400*31*cycle
 }
 
-
 //
 func pankong(a string) bool {
 	if a != "" {