crawlab_feader/FworkSpider/untils/WebCookiePool.py

import json
import sys
import requests
import re,execjs


sys.path.append('C:/Users/topnet/Desktop/crawlab_feader/FworkSpider')
sys.path.append('/app/spiders/sword_feapder/FworkSpider')
# from utils.cookie_pool import PageCookiePool
from feapder.utils.webdriver import WebDriverPool
from feapder.utils.log import log
from FworkSpider.untils.cookie_pool import PageCookiePool

class WebCookiePool(PageCookiePool):
    def __init__(self, redis_key, page_url=None,cookie_key=None,
                 min_cookies=10000, must_contained_keys=(), keep_alive=False, **kwargs):
        super(WebCookiePool, self).__init__(redis_key, page_url=None,
                                           min_cookies=10000, must_contained_keys=(), keep_alive=False, **kwargs)
        self.page_url = page_url
        self.cookie_key = cookie_key
        self._kwargs = kwargs
        self._kwargs.setdefault("load_images", False)
        self._kwargs.setdefault("headless", True)
        self._kwargs.setdefault("executable_path", "D:\\geckodriver.exe")
        self._kwargs.setdefault("driver_type", "FIREFOX")


    def create_cookie(self):
        with WebDriverPool(**self._kwargs).get() as driver_pool:
            # driver = driver_pool.driver
            driver_pool.get(self.page_url)
            import time
            try:
                count = 0
                while self.cookie_key not in driver_pool.cookies.keys():
                    time.sleep(1)
                    count+=1
                    if count>=30:
                        # driver_pool.close()
                        return
                cookies = driver_pool.cookies
                # driver_pool.close()
                return cookies
                # driver_pool.close()
            except Exception as e:
                log.error(f"获取cookie失败,{e}")


if __name__ == '__main__':
    WebCookiePool(redis_key='gdcookie',cookie_key='SUB',page_url="https://weibo.com/p/1005051203448454/home?from=page_100505_profile&wvr=6&mod=data&is_all=1#place").create_cookie()