data_spider
/
crawlab_feader


			
							1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283
							# -*- coding: utf-8 -*-
"""
Created on 2023-04-24 
---------
@summary: jsl通用模板
---------
@author: jsl
"""
import json
import re

import execjs
import requests

from untils.cookie_pool import PageCookiePool


class DTCookiePool(PageCookiePool):
    def __init__(self, redis_key, header, page_url=None, **kwargs):
        super(DTCookiePool, self).__init__(redis_key, page_url=None,
                                           min_cookies=10000,
                                           must_contained_keys=(),
                                           keep_alive=False, **kwargs)
        self.headers = header
        self.page_url = page_url
        self.proxies = kwargs.get('proxies') or False

    def create_cookie(self):
        session = requests.Session()
        session.proxies = self.proxies
        start_url = self.page_url
        res = session.get(start_url, headers=self.headers, timeout=120, verify=False)
        js_func = "".join(re.findall("document.cookie=(.*?)location.href", res.text))
        js_func = 'function sd() { return ' + js_func + "}"
        ctx = execjs.compile(js_func)
        sss = ctx.call("sd")
        cookie = {}
        for temp, index in res.cookies.get_dict().items():
            cookie[temp] = index

        for item in sss.split(";"):
            if '=' in item:
                cookie[item.split("=")[0]] = item.split("=")[-1]

        res = session.get(start_url, cookies=cookie,headers=self.headers,timeout=120,verify=False)
        html_str = res.content.decode()
        js_do_data = "".join(re.findall('};go\((.*?)\)', html_str))
        js_func = re.sub("<(/*?)script>", "", html_str)
        location = re.compile('location(.*?)}}else')
        location2 = re.compile('location(.*?)}else')
        setTimeout = re.compile('setTimeout(.*?)document')
        gox = re.compile('};go(.*?)\)')
        js_func = re.sub(location, "}}else", js_func)
        js_func = re.sub(location2, "}else", js_func)
        js_func = re.sub(setTimeout, "document", js_func)
        js_func = re.sub('0x5dc;}(.*?)\(document', "0x5dc;}document", js_func)
        js_func = re.sub(gox, "return document['cookie']\n};", js_func)
        js_func = '''const jsdom = require("jsdom");
                    const {JSDOM} = jsdom;
                    const dom = new JSDOM(`<!DOCTYPE html><p>Hello world</p>`,
                                        {
                                            url: "https://example.org/",
                                            referrer: "https://example.com/",
                                            contentType: "text/html",
                                        });
                    window = dom.window;
                    document = window.document;
                    location = window.location;
                    ''' + js_func
        ctx = execjs.compile(js_func)
        # with open('wzjyjt_xxgg_pm.js', 'w+', encoding='utf-8') as f:
        #     f.write(js_func)
        try:
            ss = ctx.call("go", json.loads(js_do_data))
            for item in ss.split(";"):
                if '=' in item:
                    session.cookies.setdefault(item.split("=")[0], item.split("=")[-1])
            session.get(start_url,headers=self.headers,timeout=120,verify=False)
            cookies = requests.utils.dict_from_cookiejar(session.cookies)
            return cookies
        except Exception as e:
            pass