fixed:优化redis模糊查询keys阻塞查询

services/proxy.py

@@ -84,8 +84,22 @@ class BaseProxyPool(BaseServer):
     def get_redis_name(self, proxy):
         return f"{self.proxy_queue}_{proxy['fingerprint']}"
 
+    def str_scan(self, pattern, count=1000):
+        cursor = '0'
+        while True:
+            cursor, keys = self.redis_db.scan(cursor, pattern, count)
+            if len(keys) > 0:
+                yield from keys
+
+            if cursor == 0:
+                break
+
     def get_redis_name_lst(self, pattern='*'):
-        return self.redis_db.keys(self.proxy_queue + pattern)
+        results = []
+        pattern = self.proxy_queue + pattern
+        for key in self.str_scan(pattern, count=5000):
+            results.append(key)
+        return results
 
     def get_proxy(self, name):
         items = self.redis_db.hgetall(name)
@@ -124,6 +138,7 @@ class BaseProxyPool(BaseServer):
             is_ok = True
         except requests.RequestException:
             pass
+
         msg = "正常" if is_ok else "失效"
         logger.debug(f"[{self.proxy_name}]检查代理Ip - {netloc} --通信{msg}")
         return proxy, is_ok
@@ -132,8 +147,7 @@ class BaseProxyPool(BaseServer):
         netloc = get_netloc(proxy)
         logger.debug(f"[{self.proxy_name}]代理Ip - {netloc} --删除")
         if self.exists(proxy):
-            redis_name = self.get_redis_name(proxy)
-            self.redis_db.delete(redis_name)
+            self.redis_db.delete(self.get_redis_name(proxy))
 
     def add_proxy(self, proxy):
         netloc = get_netloc(proxy)
@@ -241,7 +255,6 @@ class ProxyPoolClient(BaseProxyPool):
         sub_name = f'{tools.get_localhost_ip()}:{current_process.pid}'
         self.lock_label = f'{redis_label}:{sub_name}'
 
-    @property
     def proxy_total(self):
         return len(self.get_redis_name_lst())
 
@@ -290,11 +303,14 @@ class ProxyPoolClient(BaseProxyPool):
     def proxies(self, **kwargs):
         with OptimisticLock(self.redis_db, self.lock_label):
             proxy = {}
-            if self.proxy_total > 0:
+            if self.proxy_total() > 0:
                 proxy_lst = self.get_all_proxy(**kwargs)
                 proxy = proxy_lst.popleft()
                 name = self.get_redis_name(proxy)
-                self.redis_db.hset(name, 'usage', proxy['usage'] + 1)
-                self.redis_db.hset(name, 'last_time', tools.now_ts())
+                mapping = {
+                    'usage': proxy['usage'] + 1,
+                    'last_time': tools.now_ts()
+                }
+                self.redis_db.hset(name, None, None, mapping)
 
             return proxy.get('proxies')