crawlab_feader/FworkSpider/feapder/network/downloader/_requests.py

# -*- coding: utf-8 -*-
"""
Created on 2022/4/10 5:57 下午
---------
@summary:
---------
@author: Boris
@email: boris_liu@foxmail.com
"""

import random

import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.ssl_ import create_urllib3_context

import feapder.setting as setting
from feapder.network.downloader.base import Downloader
from feapder.network.response import Response


class RequestsDownloader(Downloader):
    def download(self, request) -> Response:
        response = requests.request(
            request.method, request.url, **request.requests_kwargs
        )
        response = Response(response)
        return response


class RequestsSessionDownloader(Downloader):
    session = None

    @property
    def _session(self):
        if not self.__class__.session:
            self.__class__.session = requests.Session()
            # pool_connections – 缓存的 urllib3 连接池个数  pool_maxsize – 连接池中保存的最大连接数
            http_adapter = HTTPAdapter(pool_connections=1000, pool_maxsize=1000)
            # 任何使用该session会话的 HTTP 请求，只要其 URL 是以给定的前缀开头，该传输适配器就会被使用到。
            self.__class__.session.mount("http", http_adapter)

        return self.__class__.session

    def download(self, request) -> Response:
        response = self._session.request(
            request.method, request.url, **request.requests_kwargs
        )
        response = Response(response)
        return response


class _DESAdapter(HTTPAdapter):

    def __init__(self, *args, **kwargs):
        """
        A TransportAdapter that re-enables 3DES support in Requests.
        """
        ciphers = ":".join(setting.SESSION_REQUEST_CIPHERS).split(":")
        random.shuffle(ciphers)
        ciphers = ":".join(ciphers)
        self.ciphers = ciphers + ":!aNULL:!eNULL:!MD5"
        super().__init__(*args, **kwargs)

    def init_poolmanager(self, *args, **kwargs):
        context = create_urllib3_context(ciphers=self.ciphers)
        kwargs["ssl_context"] = context
        return super(_DESAdapter, self).init_poolmanager(*args, **kwargs)

    def proxy_manager_for(self, *args, **kwargs):
        context = create_urllib3_context(ciphers=self.ciphers)
        kwargs["ssl_context"] = context
        return super(_DESAdapter, self).proxy_manager_for(*args, **kwargs)


class RequestsJa3SessionDownloader(Downloader):
    session = None

    @property
    def _session(self):
        if not self.__class__.session:
            self.__class__.session = requests.Session()
            # pool_connections – 缓存的 urllib3 连接池个数
            # pool_maxsize – 连接池中保存的最大连接数
            des_adapter = _DESAdapter(pool_connections=1000, pool_maxsize=1000)
            # 任何使用该 session会话的 HTTP/HTTPS 请求，
            # 只要其 URL 是以给定的前缀开头，该传输适配器就会被使用到。
            self.__class__.session.mount("https://", des_adapter)
            self.__class__.session.mount("http://", des_adapter)

        return self.__class__.session

    def download(self, request) -> Response:
        response = self._session.request(
            request.method, request.url, **request.requests_kwargs
        )
        response = Response(response)
        return response