实例添加代理属性、代理池功能

zgztb_cookie/FworkSpider/feapder/network/request.py

@@ -18,6 +18,7 @@ import feapder.setting as setting
 import feapder.utils.tools as tools
 from feapder.db.redisdb import RedisDB
 from feapder.network import user_agent
+from feapder.network.proxy_pool import ProxyPool
 from feapder.network.response import Response
 from feapder.utils.log import log
 from feapder.utils.webdriver import WebDriverPool
@@ -30,6 +31,7 @@ class Request(object):
     session = None
     webdriver_pool: WebDriverPool = None
     user_agent_pool = user_agent
+    proxies_pool: ProxyPool = None
 
     cache_db = None  # redis / pika
     cached_redis_key = None  # 缓存response的文件文件夹 response_cached:cached_redis_key:md5
@@ -199,6 +201,13 @@ class Request(object):
 
         return self.__class__.webdriver_pool
 
+    @property
+    def _proxies_pool(self):
+        if not self.__class__.proxies_pool:
+            self.__class__.proxies_pool = ProxyPool(check_valid=False)
+
+        return self.__class__.proxies_pool
+
     @property
     def to_dict(self):
         request_dict = {}
@@ -297,7 +306,7 @@ class Request(object):
         proxies = self.requests_kwargs.get("proxies", -1)
         if proxies == -1 and setting.PROXY_ENABLE and setting.PROXY_EXTRACT_API:
             while True:
-                proxies = self.get_proxy()
+                proxies = self._proxies_pool.get()
                 if proxies:
                     self.requests_kwargs.update(proxies=proxies)
                     break
@@ -434,14 +443,6 @@ class Request(object):
                 "https", ""
             ).strip("https://")
 
-    def get_proxy(self):
-        headers = {
-            "Authorization": setting.JIANYU_PROXY_AUTHOR
-        }
-        proxy = requests.get(setting.JIANYU_PROXY_URL, headers=headers).json()
-        print(f"切换代理：{proxy.get('data')}")
-        return proxy.get("data")
-
     def user_agent(self):
         headers = self.requests_kwargs.get("headers")
         if headers: