update - 自定义最大重试次数与超时时间

find_source/crawler/download.py

@@ -45,13 +45,14 @@ class Downloader:
         return request_params
 
     def _requests_by_get(self, url, **kw):
+        max_retries = (kw.pop('max_retries', None) or self._max_retries)
         disable_debug_log = kw.pop('disable_debug_log', True)
         request_params = self.prepare_params(**kw)
         response = Response()
         response.encoding = 'utf-8'
         response.status_code = 10001
         retries = 0
-        while retries < self._max_retries:
+        while retries < max_retries:
             try:
                 response = self._session.get(url, **request_params)
                 # 解决重定向的网站