crawlab_feader/FworkSpider/untils/WebCookiePool.py

import sys
sys.path.append('/app/spiders/sword_feapder/FworkSpider')

from feapder.utils.webdriver import WebDriver
from feapder.utils.log import log
from untils.cookie_pool import PageCookiePool
import feapder.utils.tools as tools


class WebCookiePool(PageCookiePool):

    def __init__(self, redis_key, page_url, cookie_key, **kwargs):
        super(WebCookiePool, self).__init__(redis_key, **kwargs)
        self.page_url = page_url
        self.cookie_key = cookie_key
        self._kwargs = kwargs
        self._kwargs.setdefault("load_images", False)
        self._kwargs.setdefault("headless", True)
        self._kwargs.setdefault("driver_type", "CHROME")

    def create_cookie(self, proxy=None):
        self._kwargs.setdefault("proxy", proxy)
        with WebDriver(**self._kwargs) as browser:
            try:
                browser.get(self.page_url)
                count = 0
                while self.cookie_key not in browser.cookies.keys():
                    tools.delay_time(1)
                    count += 1
                    if count >= 30:
                        return
                cookies = browser.cookies
                return cookies
            except Exception as e:
                log.error(f"获取cookie失败,{e}")