| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061 |
- import requests
- import urllib3
- from requests.models import Response
- from config.load import headers
- urllib3.disable_warnings()
- class Downloader:
- @staticmethod
- def _requests_by_get(url, max_retries=3, **kw):
- request_params = {}
- request_params.setdefault('allow_redirects', False)
- request_params.setdefault('timeout', 15)
- request_params.setdefault('headers', headers)
- for key, val in kw.items():
- if key in request_params:
- request_params.update({key: val})
- else:
- request_params.setdefault(key, val)
- retries = 0
- response = Response()
- response.encoding = 'utf-8'
- response.status_code = 10400
- setattr(response, 'loop_times', 1) # 异常通信频次默认值1
- while retries < max_retries:
- try:
- response = requests.get(url, **request_params)
- response.encoding = response.apparent_encoding
- setattr(response, 'loop_times', 0) # 正常通信频次默认值0
- break
- except requests.exceptions.SSLError as e:
- response.reason = e.__class__.__name__
- if 'verify' not in request_params:
- request_params.setdefault('verify', True)
- else:
- url = url.replace('https', 'http')
- retries += 1
- except requests.RequestException as e:
- response.reason = e.__class__.__name__
- retries += 1
- return response
- def get(self, url, **kw):
- """
- 网络请求
- :param url: 访问地址
- :param kw: 额外请求参数
- :return: 响应对象
- """
- return self._requests_by_get(url, **kw)
- if __name__ == '__main__':
- # r = Downloader().get('https://news.lzu.edu.cn/c/202204/88967.html')
- r = Downloader().get('http://www.xxls.gov.cn/')
- print(r, r.loop_times)
|
