topic_spider/find_source/crawler/download.py

import threading

import chardet
import requests
import urllib3
from loguru import logger
from requests.adapters import HTTPAdapter
from requests.models import Response, REDIRECT_STATI
from requests.utils import get_encodings_from_content
from urllib3.util.retry import Retry

from config.load import headers

urllib3.disable_warnings()

'''特殊编码需要解码'''
SPECIAL_ENCODINGS = [
    'Windows-1254',
    'ISO-8859-1'
]


class Downloader:

    def __init__(self, max_retries=3, retry_interval=0.1):
        self._max_retries = max_retries
        self._backoff_factor = retry_interval

    @staticmethod
    def prepare_params(**kw):
        request_params = {}
        request_params.setdefault('allow_redirects', False)
        request_params.setdefault('timeout', (kw.pop('timeout', None) or 10))
        for key, val in kw.items():
            if key != 'headers' and key in request_params:
                request_params.update({key: val})
            else:
                request_params.setdefault(key, val)

        request_headers = (kw.pop('headers', None) or headers)
        for key, val in request_headers.items():
            if key in request_headers:
                request_headers.update({key: val})
            else:
                request_headers.setdefault(key, val)
        request_params.setdefault('headers', request_headers)
        return request_params

    @staticmethod
    def apparent_encoding(response):
        encoding = response.encoding
        if encoding in SPECIAL_ENCODINGS:
            # 根据真正的编码格式对内容进行解码
            true_encoding = get_encodings_from_content(response.text)
            if true_encoding:
                encoding = true_encoding[0]
            else:
                encoding = chardet.detect(response.content)['encoding']
        return encoding

    def _requests_by_get(self, url, **kw):
        max_retries = (kw.pop('max_retries', 3))
        disable_debug_log = kw.pop('disable_debug_log', True)
        request_params = self.prepare_params(**kw)
        response = Response()
        response.encoding = 'utf-8'
        response.status_code = 10001
        retries = 0
        while retries < max_retries:
            try:
                response = self._session.get(url, **request_params)
                # 解决重定向的网站
                if response.status_code in REDIRECT_STATI:
                    request_params.update({'allow_redirects': True})
                    continue
                response.encoding = self.apparent_encoding(response)
                break
            except requests.exceptions.SSLError as e:
                response.reason = e.__class__.__name__
                if 'verify' not in request_params:
                    request_params.setdefault('verify', False)
                else:
                    if 'verify' in request_params:
                        del request_params['verify']
                    url = url.replace('https', 'http')
            except requests.RequestException as e:
                response.reason = e.__class__.__name__
            finally:
                retries += 1
        if not disable_debug_log:
            t_name = threading.currentThread().getName()
            logger.debug(f'<{t_name}> - {url} - 响应 - {response}')
        return response

    def get(self, url, **kw):
        """
        网络请求

        :param url: 访问地址
        :param kw: requests.GET请求参数
        :return: 响应对象
        """
        return self._requests_by_get(url, **kw)

    @property
    def _session(self):
        retry = Retry(
            total=self._max_retries,
            backoff_factor=self._backoff_factor
        )
        adapter = HTTPAdapter(max_retries=retry)
        session = requests.Session()
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        return session


class RenderDownloader(Downloader):

    def get(self, url, **kw):
        splash_url = 'http://8.131.72.226:8998/render.json'
        args = {
            'url': url,
            'html': 1,
            'iframes': 1,
            'headers': headers,
            'timeout': kw.pop('timeout', 2),
            'wait': kw.pop('wait', 0.5),
            'viewport': kw.pop('viewport', 'full'),
        }
        resp = requests.post(splash_url,
                             json=args,
                             headers={'content-type': 'application/json'})
        return resp