redis全量库改为href的hash值

src/spider/msgservice.go

@@ -278,6 +278,7 @@ func SaveObj(event int, checkAtrr string, data map[string]interface{}, saveredis
 		href := fmt.Sprint(data["href"])
 		if len(href) > 5 && saveredis { //有效数据
 			db := HexToBigIntMod(href) //根据href的哈希值选择Redis的db
+			hashHref := HexText(href)
 			//增量
 			isExist, _ := util.ExistRedis("title_repeat_judgement", 0, "url_repeat_"+href)
 			id := ""
@@ -292,9 +293,9 @@ func SaveObj(event int, checkAtrr string, data map[string]interface{}, saveredis
 				util.PutRedis("title_repeat_judgement", 0, "url_repeat_"+href, href, 3600*24*30)
 				if !flag { //保存服务发送成功
 					//全量(判断是否已存在防止覆盖id)
-					isExist, _ := util.ExistRedis("title_repeat_fulljudgement", db, "url_repeat_"+href)
+					isExist, _ := util.ExistRedis("title_repeat_fulljudgement", db, hashHref)
 					if !isExist {
-						util.PutRedis("title_repeat_fulljudgement", db, "url_repeat_"+href, "lua_"+id, -1)
+						util.PutRedis("title_repeat_fulljudgement", db, hashHref, "lua_"+id, -1)
 					}
 				}
 			}

src/spider/spider.go

@@ -179,9 +179,10 @@ func DownloadHighDetail(code string) {
 						util.PutRedis("title_repeat_judgement", 0, "url_repeat_"+href, href, 3600*24*30)
 						//全量
 						db := HexToBigIntMod(tmphref)
-						isExist, _ := util.ExistRedis("title_repeat_fulljudgement", db, "url_repeat_"+href)
+						hashHref := HexText(href)
+						isExist, _ := util.ExistRedis("title_repeat_fulljudgement", db, hashHref)
 						if !isExist {
-							util.PutRedis("title_repeat_fulljudgement", db, "url_repeat_"+href, "", -1)
+							util.PutRedis("title_repeat_fulljudgement", db, hashHref, "", -1)
 						}
 					}
 					if !success { //下载失败更新次数和状态
@@ -349,3 +350,10 @@ func HexToBigIntMod(href string) int {
 	n, _ = n.SetString(hex[2:], 16)
 	return int(n.Mod(n, big.NewInt(16)).Int64())
 }
+
+//求hash
+func HexText(href string) string {
+	h := sha256.New()
+	h.Write([]byte(href))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}

src/spider/store.go

@@ -79,12 +79,13 @@ func Store(mode, event int, c, coverAttr string, data map[string]interface{}, fl
 		href := fmt.Sprint(data["href"])
 		if len(href) > 5 && flag { //有效数据
 			db := HexToBigIntMod(href) //根据href的哈希值选择Redis的db
+			hashHref := HexText(href)
 			//增量
 			lu.PutRedis("title_repeat_judgement", 0, "url_repeat_"+href, href, 3600*24*30)
 			//全量
-			isExist, _ := lu.ExistRedis("title_repeat_fulljudgement", db, "url_repeat_"+href)
+			isExist, _ := lu.ExistRedis("title_repeat_fulljudgement", db, hashHref)
 			if !isExist {
-				lu.PutRedis("title_repeat_fulljudgement", db, "url_repeat_"+href, "", -1)
+				lu.PutRedis("title_repeat_fulljudgement", db, hashHref, "", -1)
 			}
 		}
 	} else if mode == 2 {