# -*- coding: utf-8 -*- """ Created on 2022/4/10 5:57 下午 --------- @summary: --------- @author: Boris @email: boris_liu@foxmail.com """ import random import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.ssl_ import create_urllib3_context import feapder.setting as setting from feapder.network.downloader.base import Downloader from feapder.network.response import Response class RequestsDownloader(Downloader): def download(self, request) -> Response: response = requests.request( request.method, request.url, **request.requests_kwargs ) response = Response(response) return response class RequestsSessionDownloader(Downloader): session = None @property def _session(self): if not self.__class__.session: self.__class__.session = requests.Session() # pool_connections – 缓存的 urllib3 连接池个数 pool_maxsize – 连接池中保存的最大连接数 http_adapter = HTTPAdapter(pool_connections=1000, pool_maxsize=1000) # 任何使用该session会话的 HTTP 请求,只要其 URL 是以给定的前缀开头,该传输适配器就会被使用到。 self.__class__.session.mount("http", http_adapter) return self.__class__.session def download(self, request) -> Response: response = self._session.request( request.method, request.url, **request.requests_kwargs ) response = Response(response) return response class _DESAdapter(HTTPAdapter): def __init__(self, *args, **kwargs): """ A TransportAdapter that re-enables 3DES support in Requests. """ ciphers = ":".join(setting.SESSION_REQUEST_CIPHERS).split(":") random.shuffle(ciphers) ciphers = ":".join(ciphers) self.ciphers = ciphers + ":!aNULL:!eNULL:!MD5" super().__init__(*args, **kwargs) def init_poolmanager(self, *args, **kwargs): context = create_urllib3_context(ciphers=self.ciphers) kwargs["ssl_context"] = context return super(_DESAdapter, self).init_poolmanager(*args, **kwargs) def proxy_manager_for(self, *args, **kwargs): context = create_urllib3_context(ciphers=self.ciphers) kwargs["ssl_context"] = context return super(_DESAdapter, self).proxy_manager_for(*args, **kwargs) class RequestsJa3SessionDownloader(Downloader): session = None @property def _session(self): if not self.__class__.session: self.__class__.session = requests.Session() # pool_connections – 缓存的 urllib3 连接池个数 # pool_maxsize – 连接池中保存的最大连接数 des_adapter = _DESAdapter(pool_connections=1000, pool_maxsize=1000) # 任何使用该 session会话的 HTTP/HTTPS 请求, # 只要其 URL 是以给定的前缀开头,该传输适配器就会被使用到。 self.__class__.session.mount("https://", des_adapter) self.__class__.session.mount("http://", des_adapter) return self.__class__.session def download(self, request) -> Response: response = self._session.request( request.method, request.url, **request.requests_kwargs ) response = Response(response) return response