no message

src/config.json

@@ -41,6 +41,7 @@
     "172.17.4.240:2579",
     "172.17.4.84:2379"
   ],
+  "redis_jy_add": "href=172.17.4.84:1679,detail=172.17.4.239:2679",
   "jy_pro": {
     "mgodb": "192.168.3.207:29099",
     "dbsize": 5,

src/service/repair_service.go

@@ -18,9 +18,9 @@ import (
 	"net/http"
 	"net/url"
 	qu "qfw/util"
-	"qfw/util/elastic"
 	"qfw/util/redis"
 	"regexp"
+	"spiderutil"
 	"strconv"
 	"strings"
 	"time"
@@ -154,14 +154,16 @@ func (jy *RepairRule) RepairNewSave() {
 	// 保存服务
 	href := qu.ObjToString((*updata)["href"])
 	//db := HexToBigIntMod(href)
-	hashHref := HexText(href)
-	shaid := Sha(detail)
+	//hashHref := HexText(href)
+	//shaid := Sha(detail)
 	if !strings.Contains(href, "https://www.jianyu360.cn/") {
 		// 剑鱼链接 不需要存redis
 		//PutRedis("title_repeat_fulljudgement", db, hashHref, mongodb.BsonIdToSId((*updata)["_id"]), -1) // 全量redis
-		RedisClusterPut(RedisCluster, context.TODO(), hashHref, "", -1) // 全量redis
+		//RedisClusterPut(RedisCluster, context.TODO(), hashHref, "", -1) // 全量redis
+		_, _ = spiderutil.AddBloomRedis("href", href)
 	}
-	RedisClusterPut(RedisCluster, context.TODO(), shaid, "", 365*86400) //	增量reids
+	//RedisClusterPut(RedisCluster, context.TODO(), shaid, "", 365*86400) //	增量reids
+	_, _ = spiderutil.AddBloomRedis("detail", spiderutil.FilterDetail(detail))
 	//PutRedis("title_repeat_listpagehref", 0, href, "", 3600*24*30*24)                               // 列表页增量redis
 	if b {
 		log.Println("当前新增id：", mongodb.BsonIdToSId((*updata)["_id"]))
@@ -272,17 +274,17 @@ func (jy *RepairRule) RepairPub() {
 		// 保存服务
 		href := qu.ObjToString(save["href"])
 		//db := HexToBigIntMod(href)
-		hashHref := HexText(href)
-		shaid := Sha(qu.ObjToString(save["detail"]))
+		//hashHref := HexText(href)
+		//shaid := Sha(qu.ObjToString(save["detail"]))
 		if !strings.Contains(href, "https://www.jianyu360.cn/") {
 			// 剑鱼链接 不需要存redis
 			//PutRedis("title_repeat_fulljudgement", db, hashHref, mongodb.BsonIdToSId(save["_id"]), -1)
-			r := RedisClusterPut(RedisCluster, context.TODO(), hashHref, "", -1)
-			qu.Debug(r.String())
+			//r := RedisClusterPut(RedisCluster, context.TODO(), hashHref, "", -1)
+			_, _ = spiderutil.AddBloomRedis("href", href)
 
 		}
-		r := RedisClusterPut(RedisCluster, context.TODO(), shaid, "", 365*86400)
-		qu.Debug(r.String())
+		//r := RedisClusterPut(RedisCluster, context.TODO(), shaid, "", 365*86400)
+		_, _ = spiderutil.AddBloomRedis("detail", spiderutil.FilterDetail(qu.ObjToString(save["detail"])))
 		log.Println("当前新增id：", mongodb.BsonIdToSId(save["_id"]))
 		//日志记录
 		user := jy.GetSession("user").(map[string]interface{})
@@ -1177,7 +1179,7 @@ func (jy *RepairRule) EsCount() {
 	value := jy.GetString("data")
 	field := jy.GetString("field")
 	esquery := `{"query":{"bool":{"must":[{"term":{"` + field + `":"` + value + `"}}]}}}`
-	count := elastic.Count(EsIndex, EsType, esquery)
+	count := JyEs.Count(EsIndex, EsType, esquery)
 	jy.ServeJson(map[string]interface{}{
 		"rep":   true,
 		"count": count,
@@ -1190,7 +1192,7 @@ func (jy *RepairRule) EsDelBy() {
 	field := jy.GetString("field")
 	count, _ := jy.GetInt("count")
 	esquery := `{"query":{"bool":{"must":[{"term":{"` + field + `":"` + value + `"}}]}}}`
-	bol := elastic.Del(EsIndex, EsType, esquery)
+	bol := JyEs.Del(EsIndex, EsType, esquery)
 	Mgo.Save("jy_es_del_log", bson.M{"type": field, "value": value, "count": count, "createtime": time.Now().Unix()})
 	jy.ServeJson(map[string]interface{}{
 		"rep": bol,

src/util/config.go

@@ -5,6 +5,7 @@ import (
 	"mongodb"
 	qu "qfw/util"
 	"qfw/util/redis"
+	"spiderutil"
 )
 
 var (
@@ -127,8 +128,10 @@ func initJYRedis() {
 	RedisDelKey5 = qu.ObjToString(jyredis["redisdelkey5"])
 	InitRedis(qu.ObjToString(jyredis["redis_jy_save"]))
 
-	adds := qu.ObjArrToStringArr(Sysconfig["redis_cluster"].([]interface{}))
-	RedisCluster = NewClusterClient(adds)
+	//adds := qu.ObjArrToStringArr(Sysconfig["redis_cluster"].([]interface{}))
+	//RedisCluster = NewClusterClient(adds)
+	adds := qu.ObjToString(Sysconfig["redis_jy_add"])
+	spiderutil.InitBloomRedisClient(adds)
 }
 
 func InitOther() {
@@ -201,8 +204,11 @@ func initEs() {
 	EsIndex, EsType = qu.ObjToString(Sysconfig["elasticsearch_index"]), qu.ObjToString(Sysconfig["elasticsearch_type"])
 
 	JyEs = &Elastic{
-		S_esurl: qu.ObjToString(Sysconfig["elasticsearch"]),
-		I_size:  qu.IntAllDef(qu.Int64All(Sysconfig["elasticPoolSize"]), 30),
+		S_esurl:  qu.ObjToString(Sysconfig["elasticsearch"]),
+		I_size:   qu.IntAllDef(qu.Int64All(Sysconfig["elasticPoolSize"]), 30),
+		Username: "es_all",
+		Password: "TopJkO2E_d1x",
 	}
+
 	JyEs.InitElasticSize()
 }

src/util/elasticSim.go

@@ -304,6 +304,40 @@ func (e *Elastic) Count(index, itype string, query interface{}) int64 {
 	return 0
 }
 
+//删除某个索引，根据查询
+func (e *Elastic) Del(index, itype string, query interface{}) bool {
+	client := e.GetEsConn()
+	defer e.DestoryEsConn(client)
+	b := false
+	if client != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Println("[E]", r)
+				for skip := 1; ; skip++ {
+					_, file, line, ok := runtime.Caller(skip)
+					if !ok {
+						break
+					}
+					go log.Printf("%v,%v\n", file, line)
+				}
+			}
+		}()
+		var err error
+		if qs, ok := query.(string); ok {
+			temp := es.NewQueryStringQuery(qs)
+			_, err = client.DeleteByQuery().Index(index).Query(temp).Do(context.Background())
+		} else if qi, ok2 := query.(es.Query); ok2 {
+			_, err = client.DeleteByQuery().Index(index).Query(qi).Do(context.Background())
+		}
+		if err != nil {
+			log.Println("删除索引出错:", err.Error())
+		} else {
+			b = true
+		}
+	}
+	return b
+}
+
 //更新一个字段
 //func (e *Elastic) BulkUpdateArr(index, itype string, update []map[string]string) {
 //	client := e.GetEsConn()