topic_spider/find_source/crawler/download.py

import requests
import urllib3
from requests.models import Response, REDIRECT_STATI

from config.load import headers

urllib3.disable_warnings()

'''特殊编码需要解码'''
SPECIAL_ENCODINGS = [
    'Windows-1254'
]


class Downloader:

    def __init__(self):
        self.timeout = 15
        self.max_retries = 3

    def prepare_params(self, **kw):
        request_params = {}
        request_params.setdefault('allow_redirects', False)
        request_params.setdefault('timeout', self.timeout)
        for key, val in kw.items():
            if key != 'headers' and key in request_params:
                request_params.update({key: val})
            else:
                request_params.setdefault(key, val)
        request_headers = (kw.get('headers') or headers)
        for key, val in request_headers.items():
            if key in request_headers:
                request_headers.update({key: val})
            else:
                request_headers.setdefault(key, val)
        request_params.setdefault('headers', request_headers)
        return request_params

    def _requests_by_get(self, url, **kw):
        request_params = self.prepare_params(**kw)
        response = Response()
        response.encoding = 'utf-8'
        response.status_code = 10400
        retries = 0
        while retries < self.max_retries:
            try:
                response = requests.get(url, **request_params)
                # 解决重定向的网站
                if response.status_code in REDIRECT_STATI:
                    request_params.pop('allow_redirects')
                    continue
                response.encoding = response.apparent_encoding
                if response.encoding in SPECIAL_ENCODINGS:
                    response.encoding = 'utf-8'
                break
            except requests.exceptions.SSLError as e:
                response.reason = e.__class__.__name__
                if 'verify' not in request_params:
                    request_params.setdefault('verify', True)
                else:
                    url = url.replace('https', 'http')
                retries += 1
            except requests.RequestException as e:
                response.reason = e.__class__.__name__
                retries += 1
        return response

    def get(self, url, **kw):
        """
        网络请求

        :param url: 访问地址
        :param kw: requests.GET请求参数
        :return: 响应对象
        """
        return self._requests_by_get(url, **kw)