update:更新redis服务配置

FworkSpider/feapder/dedup/README.md

@@ -95,7 +95,7 @@ def test_filter():
 from feapder.dedup import Dedup
 
 def test_filter():
-    dedup = Dedup(Dedup.RedisFilter, redis_url=["192.168.3.207:2179", "192.168.3.166:2379"], expire_time=60)
+    dedup = Dedup(Dedup.RedisFilter, ip_ports=["192.168.3.207:2179", "192.168.3.166:2379"], expire_time=60)
 
     # 制造已存在数据
     datas = ["xxx", "bbb"]

FworkSpider/feapder/dedup/__init__.py

@@ -45,6 +45,7 @@ class Dedup:
             )
         elif filter_type == Dedup.RedisFilter:
             self.dedup = RedisFilter(
+                ip_ports=kwargs.get("ip_ports"),
                 redis_url=kwargs.get("redis_url"),
                 expire_time=kwargs.get("expire_time")
             )

FworkSpider/feapder/dedup/redisfilter.py

@@ -16,16 +16,18 @@ import feapder.utils.tools as tools
 class RedisFilter(BaseFilter):
     redis_db = None
 
-    def __init__(self, redis_url=None, expire_time=None):
+    def __init__(self, ip_ports=None, redis_url=None, expire_time=None):
         if not self.__class__.redis_db:
-            if isinstance(redis_url, list) and len(redis_url) > 1:
+            if isinstance(ip_ports, list) and len(ip_ports) > 1:
                 self.__class__.redis_db = RedisDB(
-                    ip_ports=redis_url,
+                    ip_ports=ip_ports,
                     decode_responses=True,
                     user_pass='',
                 )  # 集群
+            elif redis_url:
+                self.__class__.redis_db = RedisDB(redis_url=redis_url)  # 单机
             else:
-                self.__class__.redis_db = RedisDB(url=redis_url)  # 单机
+                self.__class__.redis_db = RedisDB(ip_ports=ip_ports)  # 单机
 
         self._ex = expire_time or 86400 * 365 * 2  # 2年 = 86400 * 365 * 2
         self._prefix1 = 'list_'

FworkSpider/setting.py

@@ -69,8 +69,8 @@ PROXY_ENABLE = True
 ITEM_FILTER_ENABLE = True
 ITEM_FILTER_SETTING = dict(
     filter_type=5,  # 永久去重（BloomFilter）= 1; 内存去重（MemoryFilter）= 2; 临时去重（ExpireFilter）= 3; 轻量去重（LiteFilter）= 4; redis去重 = 5；
+    ip_ports=["172.17.4.239:2479", "172.17.4.240:2579", "172.17.4.84:2379"],  # 集群节点
     expire_time=63072000,  # 过期时间2年
-    redis_url=["172.17.4.239:2479", "172.17.4.240:2579", "172.17.4.84:2379"],  # 集群节点
 )
 
 # 企业微信报警