| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465 |
- import requests
- import urllib3
- from requests.models import Response
- from config.load import headers
- urllib3.disable_warnings()
- class Downloader:
- @staticmethod
- def _requests_by_get(url, max_retries=3, **kw):
- request_params = {}
- request_params.setdefault('allow_redirects', False)
- request_params.setdefault('timeout', 15)
- request_params.setdefault('headers', headers)
- for key, val in kw.items():
- if key == 'headers':
- for k, v in val.items():
- if str(k).lower() == 'user-agent':
- del headers['User-Agent']
- headers.update(val)
- request_params.update({'headers': headers})
- elif key in request_params:
- request_params.update({key: val})
- else:
- request_params.setdefault(key, val)
- retries = 0
- response = Response()
- response.encoding = 'utf-8'
- response.status_code = 10400
- setattr(response, 'loop_times', 1) # 异常通信频次默认值1
- while retries < max_retries:
- try:
- response = requests.get(url, **request_params)
- setattr(response, 'loop_times', 0) # 正常通信频次默认值0
- break
- except requests.exceptions.SSLError as e:
- response.reason = e.__class__.__name__
- if 'verify' not in request_params:
- request_params.setdefault('verify', True)
- else:
- url = url.replace('https', 'http')
- retries += 1
- except requests.RequestException as e:
- response.reason = e.__class__.__name__
- retries += 1
- return response
- def get(self, url, **kw):
- """
- 网络请求
- :param url: 访问地址
- :param kw: 额外请求参数
- :return: 响应对象
- """
- return self._requests_by_get(url, **kw)
- # if __name__ == '__main__':
- # r = Downloader().get('https://news.lzu.edu.cn/c/202204/88967.html')
- # print(r, r.loop_times)
|
