redis工具类修改

src/spiderutil/redis.go

@@ -0,0 +1,93 @@
+package spiderutil
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/go-redis/redis"
+	"qfw/util"
+	"strings"
+	"time"
+)
+
+var redisClient *redis.Client
+var redisClientMap map[string]*redis.Client
+
+func InitRedisClient(addrs string) {
+	//list=192.168.3.207:1679@jytopnet123,href=192.168.3.207:1679
+	redisClientMap = map[string]*redis.Client{}
+	addrArr := strings.Split(addrs, ",")
+	for _, addr := range addrArr {
+		saddr := strings.Split(addr, "=")
+		if len(saddr) == 2 {
+			code := saddr[0]
+			url_pass := strings.Split(saddr[1], "@")
+			password := ""
+			url := url_pass[0]
+			if len(url_pass) == 2 {
+				password = url_pass[1]
+			}
+			client := redis.NewClient(&redis.Options{
+				Addr:     url,
+				Password: password,
+				DB:       0,
+			})
+			_, err := client.Ping(context.Background()).Result()
+			if err != nil {
+				fmt.Println("Redis Init Err：", addr, err)
+				continue
+			}
+			redisClientMap[code] = client
+		} else {
+			fmt.Println("Redis Init Err,Url Is Wrong")
+		}
+	}
+}
+
+func RedisSet(code, key string, val interface{}, timeout int) bool {
+	defer util.Catch()
+	client := redisClientMap[code]
+	if client == nil {
+		fmt.Println("Redis Not Init")
+		return false
+	}
+	//err := client.Set(context.Background(), key, val, time.Duration(timeout)*time.Second).Err()
+	result, err := client.Set(context.Background(), key, val, time.Duration(timeout)*time.Second).Result()
+	if result == "OK" {
+		return true
+	}
+	if err != nil {
+		fmt.Println("Set Redis Error：", err)
+	}
+	return false
+}
+
+func RedisExist(code, key string) bool {
+	defer util.Catch()
+	client := redisClientMap[code]
+	if client == nil {
+		fmt.Println("Redis Not Init")
+		return false
+	}
+	result, err := client.Exists(context.Background(), key).Result()
+	if err != nil {
+		fmt.Println("Exists Redis Error：", err, result)
+		return false
+	}
+	return result == 1
+}
+
+func RedisGet(code, key string) (string, error) {
+	defer util.Catch()
+	client := redisClientMap[code]
+	if client == nil {
+		fmt.Println("Redis Not Init")
+		return "", errors.New("Redis Not Init")
+	}
+	result, err := client.Get(context.Background(), key).Result()
+	if err != nil {
+		fmt.Println("Get Redis Error：", err, result)
+		return "", err
+	}
+	return result, err
+}

src/spiderutil/redisbloom.go

@@ -1,52 +1,53 @@
 package spiderutil
 
-import (
-	"fmt"
-	redisbloom "github.com/RedisBloom/redisbloom-go"
-	qu "qfw/util"
-	"strings"
-)
-
-var bloomRedisClient map[string]*redisbloom.Client
-
-func InitBloomRedisClient(addr string) {
-	bloomRedisClient = map[string]*redisbloom.Client{}
-	addrs := strings.Split(addr, ",") //暂时不支持同两个相同名称的bloom过滤器在不同服务器
-	for _, v := range addrs {
-		name_addr := strings.Split(v, "=")
-		pool := redisbloom.NewSingleHostPool(name_addr[1], nil)
-		bloomRedisClient[name_addr[0]] = &redisbloom.Client{
-			Pool: pool,
-		}
-	}
-}
-
-//Add
-func AddBloomRedis(name, key string) bool {
-	defer qu.Catch()
-	if bloomclient := bloomRedisClient[name]; bloomclient != nil {
-		exists, err := bloomclient.Add(name, key)
-		if err != nil {
-			fmt.Println("Error:", err)
-		}
-		return exists
-	} else {
-		fmt.Println("Redis Client is Null")
-		return false
-	}
-}
-
-//Exists
-func ExistsBloomRedis(name, key string) bool {
-	defer qu.Catch()
-	if bloomclient := bloomRedisClient[name]; bloomclient != nil {
-		exists, err := bloomclient.Exists(name, key)
-		if err != nil {
-			fmt.Println("Error:", err)
-		}
-		return exists
-	} else {
-		fmt.Println("Redis Client is Null")
-		return false
-	}
-}
+//
+//import (
+//	"fmt"
+//	redisbloom "github.com/RedisBloom/redisbloom-go"
+//	qu "qfw/util"
+//	"strings"
+//)
+//
+//var bloomRedisClient map[string]*redisbloom.Client
+//
+//func InitBloomRedisClient(addr string) {
+//	bloomRedisClient = map[string]*redisbloom.Client{}
+//	addrs := strings.Split(addr, ",") //暂时不支持同两个相同名称的bloom过滤器在不同服务器
+//	for _, v := range addrs {
+//		name_addr := strings.Split(v, "=")
+//		pool := redisbloom.NewSingleHostPool(name_addr[1], nil)
+//		bloomRedisClient[name_addr[0]] = &redisbloom.Client{
+//			Pool: pool,
+//		}
+//	}
+//}
+//
+////Add
+//func AddBloomRedis(name, key string) bool {
+//	defer qu.Catch()
+//	if bloomclient := bloomRedisClient[name]; bloomclient != nil {
+//		exists, err := bloomclient.Add(name, key)
+//		if err != nil {
+//			fmt.Println("Error:", err)
+//		}
+//		return exists
+//	} else {
+//		fmt.Println("Redis Client is Null")
+//		return false
+//	}
+//}
+//
+////Exists
+//func ExistsBloomRedis(name, key string) bool {
+//	defer qu.Catch()
+//	if bloomclient := bloomRedisClient[name]; bloomclient != nil {
+//		exists, err := bloomclient.Exists(name, key)
+//		if err != nil {
+//			fmt.Println("Error:", err)
+//		}
+//		return exists
+//	} else {
+//		fmt.Println("Redis Client is Null")
+//		return false
+//	}
+//}

src/spiderutil/redisbloomutil.go

@@ -0,0 +1,47 @@
+package spiderutil
+
+import (
+	"context"
+	"errors"
+	"github.com/go-redis/redis"
+	qu "qfw/util"
+	"strings"
+)
+
+var bloomRedisClientNew map[string]*redis.Client
+
+func InitBloomRedisClient(addr string) {
+	bloomRedisClientNew = map[string]*redis.Client{}
+	addrs := strings.Split(addr, ",") //暂时不支持同两个相同名称的bloom过滤器在不同服务器
+	for _, v := range addrs {
+		name_addr := strings.Split(v, "=")
+		client := redis.NewClient(&redis.Options{
+			Addr:     name_addr[1],
+			Password: "", // no password set
+			DB:       0,  // use default DB
+		})
+		bloomRedisClientNew[name_addr[0]] = client
+	}
+}
+
+//Set
+func AddBloomRedis(name, key string) (ok bool, err error) {
+	defer qu.Catch()
+	if bloomclient := bloomRedisClientNew[name]; bloomclient != nil {
+		ok, err = bloomclient.Do(context.Background(), "BF.ADD", name, key).Bool()
+	} else {
+		return false, errors.New("Redis Client is Null")
+	}
+	return
+}
+
+//Exists
+func ExistsBloomRedis(name, key string) (ok bool, err error) {
+	defer qu.Catch()
+	if bloomclient := bloomRedisClientNew[name]; bloomclient != nil {
+		ok, err = bloomclient.Do(context.Background(), "BF.EXISTS", name, key).Bool()
+	} else {
+		return false, errors.New("Redis Client is Null")
+	}
+	return
+}

src/spiderutil/redisutil.go

@@ -1,123 +1,166 @@
 package spiderutil
 
-import (
-	"errors"
-	"log"
-	"strings"
-	"time"
-
-	red "github.com/gomodule/redigo/redis"
-)
-
-var redisPool map[string]*red.Pool
-
-func InitRedis(addrs string) {
-	redisPool = map[string]*red.Pool{}
-	addr := strings.Split(addrs, ",")
-	for _, v := range addr {
-		saddr := strings.Split(v, "=")
-		redisPool[saddr[0]] = &red.Pool{
-			MaxIdle:     256,
-			MaxActive:   0,
-			IdleTimeout: time.Duration(120),
-			Dial: func() (red.Conn, error) {
-				return red.Dial(
-					"tcp",
-					saddr[1],
-					red.DialReadTimeout(time.Duration(2000)*time.Millisecond),
-					red.DialWriteTimeout(time.Duration(2000)*time.Millisecond),
-					red.DialConnectTimeout(time.Duration(2000)*time.Millisecond),
-					red.DialDatabase(0),
-				)
-			},
-		}
-	}
-}
-
-func PutRedis(code string, db int, key, val interface{}, timeout int) bool {
-	b := false
-	var err error
-	con := redisPool[code].Get()
-	if err = con.Err(); err != nil {
-		log.Println("redisutil-conn Error", err)
-		return b
-	}
-	defer con.Close()
-	_, err = con.Do("select", db)
-	if err != nil {
-		log.Println("redisutil-select Error", err)
-		return b
-	}
-	if timeout <= 0 {
-		_, err = con.Do("SET", key, val)
-	} else {
-		_, err = con.Do("SET", key, val, "EX", timeout)
-	}
-	if err != nil {
-		log.Println("redisutil-set Error", err)
-		return b
-	} else {
-		b = true
-	}
-	return b
-}
-
-func GetRedisStr(code string, db int, key interface{}) (string, error) {
-	res, err := GetNewInterface(code, db, key)
-	if err != nil {
-		return "", err
-	} else {
-		if bt, ok := res.([]byte); ok {
-			return string(bt), nil
-		} else {
-			return "", errors.New("Val does not exist.")
-		}
-	}
-}
-
-func GetNewInterface(code string, db int, key interface{}) (res interface{}, err error) {
-	con := redisPool[code].Get()
-	if err = con.Err(); err != nil {
-		return nil, err
-	}
-	defer con.Close()
-	_, err = con.Do("select", db)
-	if err != nil {
-		log.Println(err)
-		return nil, err
-	}
-	return con.Do("GET", key)
-}
-
-func ExistRedis(code string, db int, key interface{}, args ...interface{}) (bool, error) {
-	con := redisPool[code].Get()
-	if err := con.Err(); err != nil {
-		log.Println("redis exist error:", err)
-		return false, err
-	}
-	defer con.Close()
-	_, err := con.Do("select", db)
-	if err != nil {
-		log.Println("redis select error:", err)
-		return false, err
-	}
-	parmas := make([]interface{}, 0)
-	parmas = append(parmas, key)
-	if len(args) > 0 {
-		for _, v := range args {
-			parmas = append(parmas, v)
-		}
-	}
-	repl, err := con.Do("exists", parmas...)
-	ret, _ := red.Int(repl, err)
-	return ret == 1, err
-}
-
-//func clearredis() {
-//	for i := 0; i < 16; i++ {
-//		con := redisPool.Get()
-//		defer con.Close()
-//		con.Do("select", i)
-//		con.Do("flushdb")
+//
+//import (
+//	"errors"
+//	"log"
+//	"strings"
+//	"time"
+//
+//	red "github.com/gomodule/redigo/redis"
+//)
+//
+//var redisPool map[string]*red.Pool
+//
+//func InitRedis(addrs string) {
+//	redisPool = map[string]*red.Pool{}
+//	addr := strings.Split(addrs, ",")
+//	for _, v := range addr {
+//		saddr := strings.Split(v, "=")
+//		redisPool[saddr[0]] = &red.Pool{
+//			MaxIdle:     256,
+//			MaxActive:   0,
+//			IdleTimeout: time.Duration(120),
+//			Dial: func() (red.Conn, error) {
+//				return red.Dial(
+//					"tcp",
+//					saddr[1],
+//					red.DialReadTimeout(time.Duration(2000)*time.Millisecond),
+//					red.DialWriteTimeout(time.Duration(2000)*time.Millisecond),
+//					red.DialConnectTimeout(time.Duration(2000)*time.Millisecond),
+//					red.DialDatabase(0),
+//				)
+//			},
+//		}
+//	}
+//}
+//
+//func PutRedisByDb(code string, db int, key, val interface{}, timeout int) bool {
+//	b := false
+//	var err error
+//	con := redisPool[code].Get()
+//	if err = con.Err(); err != nil {
+//		log.Println("redisutil-conn Error", err)
+//		return b
+//	}
+//	defer con.Close()
+//	_, err = con.Do("select", db)
+//	if err != nil {
+//		log.Println("redisutil-select Error", err)
+//		return b
+//	}
+//	if timeout <= 0 {
+//		_, err = con.Do("SET", key, val)
+//	} else {
+//		_, err = con.Do("SET", key, val, "EX", timeout)
+//	}
+//	if err != nil {
+//		log.Println("redisutil-set Error", err)
+//		return b
+//	} else {
+//		b = true
+//	}
+//	return b
+//}
+//
+//func GetRedisStrByDb(code string, db int, key interface{}) (string, error) {
+//	res, err := GetNewInterfaceByDb(code, db, key)
+//	if err != nil {
+//		return "", err
+//	} else {
+//		if bt, ok := res.([]byte); ok {
+//			return string(bt), nil
+//		} else {
+//			return "", errors.New("Val does not exist.")
+//		}
+//	}
+//}
+//
+//func GetNewInterfaceByDb(code string, db int, key interface{}) (res interface{}, err error) {
+//	con := redisPool[code].Get()
+//	if err = con.Err(); err != nil {
+//		return nil, err
+//	}
+//	defer con.Close()
+//	_, err = con.Do("select", db)
+//	if err != nil {
+//		log.Println(err)
+//		return nil, err
+//	}
+//	return con.Do("GET", key)
+//}
+//
+//func ExistRedisByDb(code string, db int, key interface{}, args ...interface{}) (bool, error) {
+//	con := redisPool[code].Get()
+//	if err := con.Err(); err != nil {
+//		log.Println("redis exist error:", err)
+//		return false, err
+//	}
+//	defer con.Close()
+//	_, err := con.Do("select", db)
+//	if err != nil {
+//		log.Println("redis select error:", err)
+//		return false, err
+//	}
+//	parmas := make([]interface{}, 0)
+//	parmas = append(parmas, key)
+//	if len(args) > 0 {
+//		for _, v := range args {
+//			parmas = append(parmas, v)
+//		}
+//	}
+//	repl, err := con.Do("exists", parmas...)
+//	ret, _ := red.Int(repl, err)
+//	return ret == 1, err
+//}
+//
+//func PutRedis(code string, key, val interface{}, timeout int) bool {
+//	b := false
+//	var err error
+//	con := redisPool[code].Get()
+//	if err = con.Err(); err != nil {
+//		log.Println("redisutil-conn Error", err)
+//		return b
+//	}
+//	defer con.Close()
+//	if timeout <= 0 {
+//		_, err = con.Do("SET", key, val)
+//	} else {
+//		_, err = con.Do("SET", key, val, "EX", timeout)
+//	}
+//	if err != nil {
+//		log.Println("redisutil-set Error", err)
+//		return b
+//	} else {
+//		b = true
+//	}
+//	return b
+//}
+//
+//func ExistRedis(code string, key interface{}, args ...interface{}) (bool, error) {
+//	con := redisPool[code].Get()
+//	if err := con.Err(); err != nil {
+//		log.Println("redis exist error:", err)
+//		return false, err
+//	}
+//	defer con.Close()
+//	parmas := make([]interface{}, 0)
+//	parmas = append(parmas, key)
+//	if len(args) > 0 {
+//		for _, v := range args {
+//			parmas = append(parmas, v)
+//		}
 //	}
+//	repl, err := con.Do("exists", parmas...)
+//	ret, _ := red.Int(repl, err)
+//	return ret == 1, err
 //}
+//
+////func clearredis() {
+////	for i := 0; i < 16; i++ {
+////		con := redisPool.Get()
+////		defer con.Close()
+////		con.Do("select", i)
+////		con.Do("flushdb")
+////	}
+////}

src/spiderutil/sysconfig.go

@@ -26,6 +26,7 @@ type config struct {
 	Uploadevent       int                        `json:"uploadevent"`
 	Redistype         string                     `json:"redistype"`
 	Redisservers      string                     `json:"redisservers"`
+	BloomRedisservers string                     `json:"bloomredisservers"`
 	Redishosts        []string                   `json:"redishosts"`
 	FileServer        string                     `json:"fileServer"`
 	Luadisablelib     map[string]map[string]bool `json:"luadisablelib"`

src/spiderutil/util.go

@@ -210,13 +210,17 @@ func HexTextByte(text []byte) string {
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
-func ShaDetail(con string) string {
+func Sha(con string) string {
 	h := sha256.New()
 	con = Reg.ReplaceAllString(Filter.ReplaceAllString(con, ""), "")
 	h.Write([]byte(con))
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
+func FilterDetail(con string) string {
+	return Reg.ReplaceAllString(Filter.ReplaceAllString(con, ""), "")
+}
+
 //判断当前时间是否是工作时间，工作时间周一至周五早7点至晚7点
 func IsWorkTime() bool {
 	tt := time.Now()