import requests import urllib3 from requests.models import Response, REDIRECT_STATI from config.load import headers urllib3.disable_warnings() '''特殊编码需要解码''' SPECIAL_ENCODINGS = [ 'Windows-1254' ] class Downloader: def __init__(self): self.timeout = 15 self.max_retries = 3 def prepare_params(self, **kw): request_params = {} request_params.setdefault('allow_redirects', False) request_params.setdefault('timeout', self.timeout) for key, val in kw.items(): if key != 'headers' and key in request_params: request_params.update({key: val}) else: request_params.setdefault(key, val) request_headers = (kw.get('headers') or headers) for key, val in request_headers.items(): if key in request_headers: request_headers.update({key: val}) else: request_headers.setdefault(key, val) request_params.setdefault('headers', request_headers) return request_params def _requests_by_get(self, url, **kw): request_params = self.prepare_params(**kw) response = Response() response.encoding = 'utf-8' response.status_code = 10400 retries = 0 while retries < self.max_retries: try: response = requests.get(url, **request_params) # 解决重定向的网站 if response.status_code in REDIRECT_STATI: request_params.pop('allow_redirects') continue response.encoding = response.apparent_encoding if response.encoding in SPECIAL_ENCODINGS: response.encoding = 'utf-8' break except requests.exceptions.SSLError as e: response.reason = e.__class__.__name__ if 'verify' not in request_params: request_params.setdefault('verify', True) else: url = url.replace('https', 'http') retries += 1 except requests.RequestException as e: response.reason = e.__class__.__name__ retries += 1 return response def get(self, url, **kw): """ 网络请求 :param url: 访问地址 :param kw: requests.GET请求参数 :return: 响应对象 """ return self._requests_by_get(url, **kw)