更新采集工具包环境依赖

FworkSpider/__init__.py

@@ -1,4 +0,0 @@
-__all__ = [
-    "setting",
-    "feapder",
-]

FworkSpider/crawl_func/PYCCS_cookies.py

@@ -6,17 +6,15 @@ Created on 2023-09-08
 ---------
 @author: Lzz
 """
-import requests
 import re
-import execjs
-
 
+import execjs
+import requests
 
-def get_PYCCS_ck(url,headers,proxies=False):
 
+def get_PYCCS_ck(url, headers, proxies=False):
     session = requests.session()
     session.proxies = proxies
-
     ex_js = '''
     function get_ck(a,b,c) {
         var x08c924 = parseInt(a);

FworkSpider/crawl_func/YunSuoAutoJump.py

@@ -6,13 +6,13 @@ Created on 2023-09-12
 ---------
 @author: Lzz
 """
-import requests
-import execjs
 import time
 
+import execjs
+import requests
 
 
-def get_mid_code(security_verify_data_url,proxies=False):
+def get_mid_code(security_verify_data_url, proxies=False):
 
     session = requests.session()
     session.proxies = proxies
@@ -57,4 +57,3 @@ def get_mid_code(security_verify_data_url,proxies=False):
         num += 1
         time.sleep(2)
     return cookies
-

FworkSpider/crawl_func/ali_slide_verify.py

@@ -6,15 +6,14 @@ Created on 2023-08-17
 ---------
 @author: Lzz
 """
-import sys
-from selenium.webdriver import ActionChains
-sys.path.append('/app/spiders/sword_feapder/FworkSpider')
-from feapder.utils.webdriver import WebDriver
-from feapder.utils.log import log
-from untils.cookie_pool import PageCookiePool
 import re
+
 import execjs
+from selenium.webdriver import ActionChains
 
+from feapder.network.cookie_pool import PageCookiePool
+from feapder.utils.log import log
+from feapder.utils.webdriver import WebDriver
 
 
 def get_acw_sc_v2(html):

FworkSpider/crawl_func/jsl_5s.py

@@ -6,21 +6,22 @@ Created on 2023-08-08
 ---------
 @author: jsl、创宇5秒盾
 """
-import re
-import sys
 import json
+import re
+
 import execjs
 import requests
-sys.path.append('/app/spiders/sword_feapder/FworkSpider')
-from untils.cookie_pool import PageCookiePool
 
+from feapder.network.cookie_pool import PageCookiePool
 
 
 class DTCookiePool(PageCookiePool):
-    def __init__(self,redis_key,header,page_url=None,**kwargs):
-        super(DTCookiePool, self).__init__(redis_key,page_url=None,
-        min_cookies=10000,must_contained_keys=(),keep_alive=False,**kwargs)
-        self.headers=header
+    def __init__(self, redis_key, header, page_url=None, **kwargs):
+        super(DTCookiePool, self).__init__(redis_key, page_url=None,
+                                           min_cookies=10000,
+                                           must_contained_keys=(),
+                                           keep_alive=False, **kwargs)
+        self.headers = header
         self.page_url = page_url
         self.proxies = kwargs.get('proxies') or False
 

FworkSpider/crawl_func/jsl_clearance_s.py

@@ -6,32 +6,30 @@ Created on 2023-04-24
 ---------
 @author: jsl
 """
-import re
-import sys
 import json
+import re
+
 import execjs
 import requests
-sys.path.append('/app/spiders/sword_feapder/FworkSpider')
-from untils.cookie_pool import PageCookiePool
 
+from untils.cookie_pool import PageCookiePool
 
 
 class DTCookiePool(PageCookiePool):
-    def __init__(self,redis_key,header,page_url=None,**kwargs):
-        super(DTCookiePool, self).__init__(redis_key,page_url=None,
-        min_cookies=10000,must_contained_keys=(),keep_alive=False,**kwargs)
-        self.headers=header
+    def __init__(self, redis_key, header, page_url=None, **kwargs):
+        super(DTCookiePool, self).__init__(redis_key, page_url=None,
+                                           min_cookies=10000,
+                                           must_contained_keys=(),
+                                           keep_alive=False, **kwargs)
+        self.headers = header
         self.page_url = page_url
         self.proxies = kwargs.get('proxies') or False
 
     def create_cookie(self):
-
-        proxies = self.proxies
-
         session = requests.Session()
-        session.proxies = proxies
+        session.proxies = self.proxies
         start_url = self.page_url
-        res = session.get(start_url, headers=self.headers,timeout=120, verify=False)
+        res = session.get(start_url, headers=self.headers, timeout=120, verify=False)
         js_func = "".join(re.findall("document.cookie=(.*?)location.href", res.text))
         js_func = 'function sd() { return ' + js_func + "}"
         ctx = execjs.compile(js_func)