topic_spider/find_source/crawler/download.py

import threading

import chardet
import requests
import urllib3
from loguru import logger
from requests.adapters import HTTPAdapter
from requests.models import Response, REDIRECT_STATI
from urllib3.util.retry import Retry

from config.load import headers

urllib3.disable_warnings()

'''特殊编码需要解码'''
SPECIAL_ENCODINGS = [
    'Windows-1254'
]


class Downloader:

    def __init__(self, max_retries=3, retry_interval=0.1):
        self._max_retries = max_retries
        self._backoff_factor = retry_interval

    @staticmethod
    def prepare_params(**kw):
        request_params = {}
        request_params.setdefault('allow_redirects', False)
        request_params.setdefault('timeout', (kw.pop('timeout', None) or 10))
        for key, val in kw.items():
            if key != 'headers' and key in request_params:
                request_params.update({key: val})
            else:
                request_params.setdefault(key, val)

        request_headers = (kw.pop('headers', None) or headers)
        for key, val in request_headers.items():
            if key in request_headers:
                request_headers.update({key: val})
            else:
                request_headers.setdefault(key, val)
        request_params.setdefault('headers', request_headers)
        return request_params

    def _requests_by_get(self, url, **kw):
        max_retries = (kw.pop('max_retries', 3))
        disable_debug_log = kw.pop('disable_debug_log', True)
        request_params = self.prepare_params(**kw)
        response = Response()
        response.encoding = 'utf-8'
        response.status_code = 10001
        retries = 0
        while retries < max_retries:
            try:
                response = self._session.get(url, **request_params)
                # 解决重定向的网站
                if response.status_code in REDIRECT_STATI:
                    request_params.update({'allow_redirects': True})
                    continue
                response.encoding = chardet.detect(response.content)['encoding']
                # response.encoding = response.apparent_encoding
                # if response.encoding in SPECIAL_ENCODINGS:
                #     response.encoding = 'utf-8'
                break
            except requests.exceptions.SSLError as e:
                response.reason = e.__class__.__name__
                if 'verify' not in request_params:
                    request_params.setdefault('verify', False)
                else:
                    if 'verify' in request_params:
                        del request_params['verify']
                    url = url.replace('https', 'http')
                retries += 1
            except requests.RequestException as e:
                response.reason = e.__class__.__name__
                retries += 1
        if not disable_debug_log:
            t_name = threading.currentThread().getName()
            logger.debug(f'<{t_name}> - {url} - 响应 - {response}')
        return response

    def get(self, url, **kw):
        """
        网络请求

        :param url: 访问地址
        :param kw: requests.GET请求参数
        :return: 响应对象
        """
        return self._requests_by_get(url, **kw)

    @property
    def _session(self):
        retry = Retry(
            total=self._max_retries,
            backoff_factor=self._backoff_factor
        )
        adapter = HTTPAdapter(max_retries=retry)
        session = requests.Session()
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        return session