超大过滤数据存储

src/spider/msgservice.go

@@ -250,7 +250,11 @@ func SaveObj(event int, checkAtrr string, data map[string]interface{}, saveredis
 	bs, _ := json.Marshal(data)
 	size := len(bs) / (1024 * 1024)
 	if size > 10 {
-		log.Println(event, checkAtrr, data["href"], data["title"], len(bs))
+		href := fmt.Sprint(data["href"])
+		hashHref := util.HexText(href)
+		util.RedisClusterSet(hashHref, "", -1)
+		MgoS.Save("spider_filterdata", data)
+		//log.Println(event, checkAtrr, data["href"], data["title"], len(bs))
 		return
 	}
 	defer mu.Catch()