import threading import requests import urllib3 from loguru import logger from requests.adapters import HTTPAdapter from requests.models import Response, REDIRECT_STATI from urllib3.util.retry import Retry from config.load import headers urllib3.disable_warnings() '''特殊编码需要解码''' SPECIAL_ENCODINGS = [ 'Windows-1254' ] class Downloader: def __init__(self, connect=5, backoff_factor=0.1): self._connect = connect self._backoff_factor = backoff_factor self._max_retries = 3 @staticmethod def prepare_params(**kw): request_params = {} request_params.setdefault('allow_redirects', False) request_params.setdefault('timeout', (kw.pop('timeout', None) or 10)) for key, val in kw.items(): if key != 'headers' and key in request_params: request_params.update({key: val}) else: request_params.setdefault(key, val) request_headers = (kw.pop('headers', None) or headers) for key, val in request_headers.items(): if key in request_headers: request_headers.update({key: val}) else: request_headers.setdefault(key, val) request_params.setdefault('headers', request_headers) return request_params def _requests_by_get(self, url, **kw): disable_debug_log = kw.pop('disable_debug_log', True) request_params = self.prepare_params(**kw) response = Response() response.encoding = 'utf-8' response.status_code = 10001 retries = 0 while retries < self._max_retries: try: response = self._session.get(url, **request_params) # 解决重定向的网站 if response.status_code in REDIRECT_STATI: request_params.update({'allow_redirects': True}) continue response.encoding = response.apparent_encoding if response.encoding in SPECIAL_ENCODINGS: response.encoding = 'utf-8' break except requests.exceptions.SSLError as e: response.reason = e.__class__.__name__ if 'verify' not in request_params: request_params.setdefault('verify', False) else: if 'verify' in request_params: del request_params['verify'] url = url.replace('https', 'http') retries += 1 except requests.RequestException as e: response.reason = e.__class__.__name__ retries += 1 if not disable_debug_log: t_name = threading.currentThread().getName() logger.debug(f'<{t_name}> - {url} - 响应 - {response}') return response def get(self, url, **kw): """ 网络请求 :param url: 访问地址 :param kw: requests.GET请求参数 :return: 响应对象 """ return self._requests_by_get(url, **kw) @property def _session(self): retry = Retry( connect=self._connect, backoff_factor=0.1 ) adapter = HTTPAdapter(max_retries=retry) session = requests.Session() session.mount('http://', adapter) session.mount('https://', adapter) return session