Merge branch 'master' of http://192.168.3.207:8080/data_processing/crawlab_feader

A数据处理/site_monitor/monitor.py

@@ -6,6 +6,7 @@ Created on 2023-05-10
 ---------
 @author: Dzr
 """
+import copy
 import threading
 
 import bson
@@ -68,18 +69,24 @@ class MonitorParser(threading.Thread):
             except requests_exceptions.SSLError:
                 url = url.replace('https', 'http')
                 request = Request(url=url, render=True, **kwargs)
+            except requests_exceptions.ConnectionError:
+                kw = copy.deepcopy(kwargs)
+                kw.pop('proxies', '')
+                request = Request(url=url, render=True, **kw)
             except requests_exceptions as e:
                 logger.exception(e)
                 break
 
+        # 释放浏览器
+        response.close_browser(request)
+
         logger.debug(
             """
-                -------------- response for ----------------
-                thread = %s
+                -------------- %s response for ----------------
                 url = %s
                 title = %s
                 response = %s
-                """
+            """
             % (
                 self.getName(),
                 url,

A数据处理/site_monitor/network/response.py

@@ -350,7 +350,7 @@ class Response(res):
 
     def close_browser(self, request):
         if hasattr(self, "browser"):
-            request._webdriver_pool.remove(self.browser)
+            request._render_downloader.webdriver_pool.remove(self.browser)
             del self.browser
 
     def __del__(self):