import requests import urllib3 from requests.models import Response from config.load import headers urllib3.disable_warnings() class Downloader: @staticmethod def _requests_by_get(url, max_retries=3, **kw): request_params = {} request_params.setdefault('allow_redirects', False) request_params.setdefault('timeout', 15) request_params.setdefault('headers', headers) for key, val in kw.items(): if key in request_params: request_params.update({key: val}) else: request_params.setdefault(key, val) retries = 0 response = Response() response.encoding = 'utf-8' response.status_code = 10400 setattr(response, 'loop_times', 1) # 异常通信频次默认值1 while retries < max_retries: try: response = requests.get(url, **request_params) response.encoding = response.apparent_encoding setattr(response, 'loop_times', 0) # 正常通信频次默认值0 break except requests.exceptions.SSLError as e: response.reason = e.__class__.__name__ if 'verify' not in request_params: request_params.setdefault('verify', True) else: url = url.replace('https', 'http') retries += 1 except requests.RequestException as e: response.reason = e.__class__.__name__ retries += 1 return response def get(self, url, **kw): """ 网络请求 :param url: 访问地址 :param kw: 额外请求参数 :return: 响应对象 """ return self._requests_by_get(url, **kw) if __name__ == '__main__': # r = Downloader().get('https://news.lzu.edu.cn/c/202204/88967.html') r = Downloader().get('http://www.xxls.gov.cn/') print(r, r.loop_times)