import json import sys import requests import re,execjs sys.path.append('C:/Users/topnet/Desktop/crawlab_feader/FworkSpider') sys.path.append('/app/spiders/sword_feapder/FworkSpider') # from utils.cookie_pool import PageCookiePool from feapder.utils.webdriver import WebDriverPool from feapder.utils.log import log from FworkSpider.untils.cookie_pool import PageCookiePool class WebCookiePool(PageCookiePool): def __init__(self, redis_key, page_url=None,cookie_key=None, min_cookies=10000, must_contained_keys=(), keep_alive=False, **kwargs): super(WebCookiePool, self).__init__(redis_key, page_url=None, min_cookies=10000, must_contained_keys=(), keep_alive=False, **kwargs) self.page_url = page_url self.cookie_key = cookie_key self._kwargs = kwargs self._kwargs.setdefault("load_images", False) self._kwargs.setdefault("headless", True) self._kwargs.setdefault("executable_path", "D:\\geckodriver.exe") self._kwargs.setdefault("driver_type", "FIREFOX") def create_cookie(self): with WebDriverPool(**self._kwargs).get() as driver_pool: # driver = driver_pool.driver driver_pool.get(self.page_url) import time try: count = 0 while self.cookie_key not in driver_pool.cookies.keys(): time.sleep(1) count+=1 if count>=30: # driver_pool.close() return cookies = driver_pool.cookies # driver_pool.close() return cookies # driver_pool.close() except Exception as e: log.error(f"获取cookie失败,{e}") if __name__ == '__main__': WebCookiePool(redis_key='gdcookie',cookie_key='SUB',page_url="https://weibo.com/p/1005051203448454/home?from=page_100505_profile&wvr=6&mod=data&is_all=1#place").create_cookie()