加入详情请访问原网页！类似情况redis判重

src/saveServer/bidcheckrepeat/redischeck.go

@@ -2,6 +2,8 @@ package bidcheckrepeat
 
 import (
 	"log"
+	"regexp"
+
 	//"qfw/util/redis"
 	"saveServer/tools"
 )
@@ -13,9 +15,11 @@ const (
 	//sha              = "e6d1f45ff6176304f9d2394ed57e518d28f6a8a4b92982e2a2af168496e4b43b"
 )
 
+var SpecialTextReg = regexp.MustCompile("(无|原网页)")
+
 //var KS = []string{} //redis [redisdb,old]
 
-func BidCheck(href, shaid string) (res bool) {
+func BidCheck(href, shaid, detail string) (res bool) {
 	//分别检查href和shaid是否存
 	var e error
 	var v string
@@ -27,27 +31,21 @@ func BidCheck(href, shaid string) (res bool) {
 	} else { //存在
 		res = true
 	}
-	if e == nil && !res && shaid != "-1" && shaid != ANNEX_DETAIL_SHA {
+	if e == nil && !res && shaid != "-1" && shaid != ANNEX_DETAIL_SHA && !DetailExists(detail) {
 		res, e = tools.ExistRedis("shaid", 0, shaid)
 	}
 	if e != nil {
 		log.Println("redis-check-error", e)
 	}
 	return
-	//	var e error
-	//	for _, k := range KS {
-	//		res, e = redis.Exists(k, href)
-	//		if e == nil && !res && shaid != "-1" {
-	//			res, e = redis.Exists(k, shaid)
-	//		}
-	//		if res {
-	//			break
-	//		}
-	//	}
-	//	if e != nil {
-	//		log.Println("redis-check-error", e)
-	//	}
-	//	return
+}
+
+//同“详情请访问原网页！”一样，加入了“访问原网页”、“见原网页”等类似正文内容不做判重的处理
+func DetailExists(detail string) bool {
+	if len([]rune(detail)) < 20 && SpecialTextReg.MatchString(detail) {
+		return true
+	}
+	return false
 }
 
 //存redis

src/saveServer/savedata.go

@@ -177,11 +177,11 @@ func NewSaveBidding(tmp map[string]interface{}) (b bool, res int) {
 	}
 	infoformat := qutil.IntAll(tmp["infoformat"])
 	uuid := "-1"
+	detail := qutil.ObjToString(tmp["detail"])
 	if infoformat == 1 {
-		detail := qutil.ObjToString(tmp["detail"])
 		uuid = tools.Sha(detail)
 	}
-	isExist := bidcheckrepeat.BidCheck(href, uuid) //查redis
+	isExist := bidcheckrepeat.BidCheck(href, uuid, detail) //查redis
 	//qutil.Debug("href:", href, "redis isexist:", isExist, "dataging:", tmp["dataging"])
 	dataging := qutil.IntAll(tmp["dataging"])
 	if isExist && dataging == 0 { //重复数据
@@ -291,6 +291,7 @@ func NewSaveBidding(tmp map[string]interface{}) (b bool, res int) {
 	return
 }
 
+/*
 func SaveBidding(tmp map[string]interface{}) (b bool, res int) {
 	T := qutil.ObjToString(tmp["T"])
 	if T != SaveColl {
@@ -382,7 +383,7 @@ func SaveBidding(tmp map[string]interface{}) (b bool, res int) {
 	b = true
 	return
 }
-
+*/
 //通用保存服务
 var msave = map[string][]map[string]interface{}{}
 var mlock = sync.Mutex{}