清除废弃模块

zgztb_cookie/FworkSpider/feapder/pipelines/swordfish/mongo_pipeline.py

@@ -1,88 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Created on 2021-04-18 14:12:21
----------
-@summary: 导出数据
----------
-@author: 马国鹏
-@email:  305021384@qq.com
-"""
-from typing import Dict, List, Tuple
-
-from feapder.db.mongodb import MongoDB
-from feapder.dedup import Dedup
-from feapder.pipelines import BasePipeline
-from feapder.utils.log import log
-
-
-class MongoPipeline(BasePipeline):
-    def __init__(self):
-        self._to_db = None
-
-    @property
-    def to_db(self):
-        if not self._to_db:
-            self._to_db = MongoDB()
-        return self._to_db
-
-    def save_items(self, table, items: List[Dict]) -> bool:
-        """
-        保存数据
-        Args:
-            table: 表名
-            items: 数据，[{},{},...]
-
-        Returns: 是否保存成功 True / False
-                 若False，不会将本批数据入到去重库，以便再次入库
-        """
-        try:
-            add_count = self.to_db.add_batch(coll_name=table, datas=items)
-            for item in items:
-                dedup = Dedup(Dedup.BloomFilter)
-                dedup.add([item.get("href")])
-
-            datas_size = len(items)
-            log.info(
-                "共导出 %s 条数据到 %s,  新增 %s条, 重复 %s 条"
-                % (datas_size, table, add_count, datas_size - add_count)
-            )
-
-            return True
-        except Exception as e:
-            log.exception(e)
-            return False
-
-    def update_items(self, table, items: List[Dict], update_keys=Tuple) -> bool:
-        """
-        更新数据
-        Args:
-            table: 表名
-            items: 数据，[{},{},...]
-            update_keys: 更新的字段, 如 ("title", "publish_time")
-
-        Returns: 是否更新成功 True / False
-                 若False，不会将本批数据入到去重库，以便再次入库
-
-        """
-        try:
-            add_count = self.to_db.add_batch(
-                coll_name=table,
-                datas=items,
-                update_columns=update_keys or list(items[0].keys()),
-            )
-            datas_size = len(items)
-            update_count = datas_size - add_count
-            msg = "共导出 %s 条数据到 %s,  新增 %s 条, 更新 %s 条" % (
-                datas_size,
-                table,
-                add_count,
-                update_count,
-            )
-            if update_keys:
-                msg += " 更新字段为 {}".format(update_keys)
-            log.info(msg)
-
-            return True
-        except Exception as e:
-            log.exception(e)
-            return False

zgztb_cookie/ces_detail.py

@@ -1,162 +0,0 @@
-import re
-import sys
-
-import execjs
-import requests
-
-sys.path.append('/mnt/topic_spider/zgztb_cookie/FworkSpider')
-from feapder.utils.cleaner import cleaner
-from feapder.db.mongodb import MongoDB
-import feapder.utils.tools as tools
-from encode_info import encode_info
-from cookie_pool import WebCookiePool
-import time
-db_name="zgzb_list"
-cookie_pool = WebCookiePool(redis_key='zgztbcookie',
-                                page_url="http://www.cebpubservice.com/ctpsp_iiss/SecondaryAction/findDetails.do")
-def get_proxy():
-    headers = {
-        "Authorization": "Basic amlhbnl1MDAxOjEyM3F3ZSFB"
-    }
-    proxy = requests.get("http://cc.spdata.jianyu360.com/crawl/proxy/socks5/fetch", headers=headers).json()
-    print("切换代理：", proxy.get("data"))
-    return proxy.get("data")
-def get_acw_sc_v2(arg1):
-    path = 'acw_sc_v2.js'
-    with open(path, encoding='utf-8') as rp:
-        js_script = rp.read()
-        ctx = execjs.compile(js_script)
-        arg2 = ctx.call('l', arg1)
-        print(f'acw_sc_v2 >>> {arg2}')
-        return arg2
-def get_response():
-    session = requests.session()
-    # proxy = get_proxy()
-    session.headers={
-            "Host": "www.cebpubservice.com",
-            "Accept": "application/json, text/javascript, */*; q=0.01",
-            "X-Requested-With": "XMLHttpRequest",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0",
-            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
-            "Origin": "http://www.cebpubservice.com",
-            "Referer": "http://www.cebpubservice.com/ctpsp_iiss/SecondaryAction/findDetails.do",
-            "Accept-Language": "zh-CN,zh;q=0.9"
-        }
-    session.proxies=get_proxy()
-    if True:
-        data_lsit = MongoDB().find("zgzb_list", {"type": "0","timeout":None}, sort={"publishtime": -1}, limit=100)
-        for item in data_lsit:
-            tenderprojectcode = item.get("href").split("&")[1]
-            businessid = item.get("href").split("&")[0]
-            businesskeyword = item.get("businessKeyWord")
-            businessoObjectname = item.get("title")
-            schemaversion = item.pop("schemaVersion")
-            rowguid = item.pop("rowGuid")
-            print(businessoObjectname)
-            detail_data = {
-                "schemaVersion": schemaversion,
-                "businessKeyWord": businesskeyword,
-                "tenderProjectCode": encode_info(tenderprojectcode),
-                "businessObjectName": businessoObjectname,
-                "businessId": encode_info(businessid),
-            }
-            detail_url = 'http://www.cebpubservice.com/ctpsp_iiss/SecondaryAction/findDetails.do'
-            try:
-                response = session.post(url=detail_url,data=detail_data,timeout=5)
-                response.encoding = response.apparent_encoding
-            except :
-                print("timeout")
-                print(detail_data)
-                MongoDB().update(db_name, {"timeout": 1}, {"_id": item["_id"]})
-
-                continue
-            if "arg1" in response.text:
-                print(response.text)
-                arg1 = "".join(re.findall("arg1='(\w+)';", response.text)).strip()
-                if arg1 != '':
-                    acw_sc_v2 = get_acw_sc_v2(arg1)
-                    session.cookies.update({'acw_sc__v2': acw_sc_v2})
-                    response = session.post(url=detail_url,data=detail_data,timeout=5)
-            elif '滑动验证页面' in response.text:
-                cookies = cookie_pool.create_cookies(proxies=session.proxies.get("http"))
-                print("cookies", cookies)
-
-                session.cookies.update(cookies)
-                try:
-                    response = session.post(url=detail_url,data=detail_data,timeout=5)
-                    if '滑动验证页面' in response.text:
-                        continue
-                except:
-                    print("timeout")
-                    print(detail_data)
-                    MongoDB().update(db_name,{"timeout":1}, {"_id": item["_id"]})
-                    continue
-            try:
-                res = response.json()
-            except Exception as e:
-                print(response.text)
-                print(e)
-                continue
-            else:
-                # print(response.json())
-                detail_response = response.json()
-                print('ok 请求成功')
-                tenderprojectcode = item.get("href").split("&")[1]
-                businessid = item.get("href").split("&")[0]
-                businesskeyword = "businessKeyWord"
-                if item.__contains__("businessKeyWord"):
-                    businesskeyword = item.pop("businessKeyWord")
-
-                detail_info = detail_response.get("object").get(businesskeyword)
-                if not detail_info:
-                    businesskeywords = detail_response.get("object").keys()
-                    for businesskeyword in businesskeywords:
-                        businesskeyword = businesskeyword
-                    detail_info = detail_response.get("object").get(businesskeyword)
-
-                area = item.get("area")
-                if area is None:
-                    item["area"] = "全国"
-                    item["city"] = ""
-                elif " " in area:
-                    item["area"] = area.split(" ")[0]
-                    item["city"] = area.split(" ")[-1]
-                else:
-                    item["area"] = "全国"
-                    item["city"] = ""
-                if detail_info is None or detail_info == []:
-                    businessKeyWords = detail_response.get("object").keys()
-                    for key in businessKeyWords:
-                        businesskeyword = key
-                    detail_info = detail_response.get("object").get(businesskeyword)
-                    if detail_info is None or detail_info == []:
-                        item[
-                            "href"] = f"http://www.cebpubservice.com/ctpsp_iiss/searchbusinesstypebeforedooraction/showDetails.do#uuid={businessid + tenderprojectcode}"
-                        item["sendflag"] = "true"
-                        item["comeintime"] = int(time.time())
-                        result = MongoDB().to_db.add("data_bak", item)
-                        MongoDB().delete(db_name, {"_id": item["_id"]})
-
-                        print(f"mongo add _id:{item.get('title')}<空结果")
-                    else:
-                        pass
-                detail_info = detail_info[0]
-                item["contenthtml"] = detail_info.get("bulletinContent")
-                if item["contenthtml"] is None:
-                    item["detail"] = None
-                    item["sendflag"] = "true"
-                else:
-                    item["detail"] = cleaner(item["contenthtml"])
-                item[
-                    "href"] = f"http://www.cebpubservice.com/ctpsp_iiss/searchbusinesstypebeforedooraction/showDetails.do#uuid={businessid + tenderprojectcode}"
-                if tools.chinese_character(item["detail"]).total == 0:
-                    item["sendflag"] = "true"
-                item["comeintime"] = tools.ensure_int64(int(time.time()))
-                ss = {"": ""}
-                result = MongoDB().add("data_bak", item)
-                MongoDB().delete(db_name, {"_id": item["_id"]})
-                print(f"mongo add _id:{item.get('title')}")
-
-if __name__ == '__main__':
-    get_response()
-    #

zgztb_cookie/config/constants.yaml

@@ -1,8 +0,0 @@
-headers:
-  User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36
-  Accept: application/json, text/plain, */*
-
-proxy:
-  socks5:
-    url: http://socks.spdata.jianyu360.com/socks/getips?limit=100
-    decrypt: ABNOPqrceQRSTklmUDEFGXYZabnopfghHVWdijstuvwCIJKLMxyz0123456789+/

zgztb_cookie/config/dev.yaml

@@ -1,37 +0,0 @@
-mongo:
-  host: 172.17.4.87
-  port: !!int 27080
-
-
-redis:
-  host: 172.17.4.232
-  port: !!int 7361
-  pwd: "k5ZJR5KV4q7DRZ92DQ"
-  db: !!int 10
-
-
-redis_cluster:
-  - host: 172.17.4.239
-    port: 2479
-  - host: 172.17.4.240
-    port: 2579
-  - host: 172.17.4.84
-    port: 2379
-
-
-es:
-  host: 172.17.145.178
-  port: !!int 9800
-  db: biddingall # es库别名
-
-
-proxy:
-  socks5:
-    url: http://cc.spdata.jianyu360.com/crawl/proxy/socks5/fetch
-    auth:
-      Authorization: Basic amlhbnl1MDAxOjEyM3F3ZSFB
-
-
-selenium:
-  remote:
-    server_addr: http://172.17.4.232:6666/wd/hub

zgztb_cookie/config/load.py

@@ -1,39 +0,0 @@
-import platform
-from pathlib import Path
-
-import yaml
-
-__all__ = [
-    'mongo_conf',
-    'redis_conf',
-    'redis_startup_nodes',
-    'es_conf',
-    'jy_proxy',
-    'node_module_path',
-    'headers',
-    'selenium_remote_server_addr'
-]
-if platform.system() not in ['Darwin', 'Windows']:
-    ENV = 'dev.yaml'
-else:
-    ENV = 'test.yaml'
-
-_base_path = Path(__file__).parent
-_yaml_conf = (_base_path / ENV).resolve()
-_yaml_constants = (_base_path / 'constants.yaml').resolve()
-_node_modules = (_base_path.parent / 'node_modules').resolve()
-
-with open(_yaml_conf, encoding="utf-8") as f:
-    _conf = yaml.safe_load(f)
-    mongo_conf = _conf['mongo']
-    redis_conf = _conf['redis']
-    redis_startup_nodes = _conf['redis_cluster']
-    es_conf: dict = _conf['es']
-    jy_proxy: dict = _conf['proxy']
-    selenium_remote_server_addr = _conf['selenium']['remote']['server_addr']
-    node_module_path = _node_modules
-
-
-with open(_yaml_constants, encoding="utf-8") as fp:
-    _constants = yaml.safe_load(fp)
-    headers: dict = _constants['headers']

zgztb_cookie/config/test.yaml

@@ -1,37 +0,0 @@
-mongo:
-  host: 192.168.3.182
-  port: !!int 27017
-
-
-redis:
-  host: 192.168.3.182
-  port: !!int 6379
-  pwd: "jianyu@python"
-  db: !!int 10
-
-
-redis_cluster:
-  - host: 192.168.3.207
-    port: 2179
-  - host: 192.168.3.166
-    port: 2379
-  - host: 192.168.3.207
-    port: 2279
-
-
-es:
-  host: 192.168.3.206
-  port: !!int 9800
-  db: biddingall # es库别名
-
-
-proxy:
-  socks5:
-    url: http://cc.spdata.jianyu360.com/crawl/proxy/socks5/fetch
-    auth:
-      Authorization: Basic amlhbnl1MDAxOjEyM3F3ZSFB
-
-
-selenium:
-  remote:
-    server_addr: http://192.168.3.182:9999/wd/hub

zgztb_cookie/utils/databases.py

@@ -1,110 +0,0 @@
-import bson
-import pymongo
-import redis
-from elasticsearch import Elasticsearch
-from pymongo.uri_parser import SCHEME
-from rediscluster import RedisCluster
-
-from config.load import mongo_conf, redis_conf, es_conf, redis_startup_nodes
-
-# ---------------------------------- mongo ----------------------------------
-MONGO_URI_CLIENTS = {}    # a dictionary hold all client with uri as key
-
-
-def mongo_client(host=None, port=None, fork=False, **kwargs):
-    if host is None:
-        host = mongo_conf["host"]
-    if port is None:
-        port = mongo_conf["port"]
-
-    if host is not None and host.startswith(SCHEME):
-        uri = host
-    else:
-        uri = f'mongodb://{host}:{port}'
-
-    if fork:
-        return pymongo.MongoClient(uri, **kwargs)
-
-    global MONGO_URI_CLIENTS
-    matched_client = MONGO_URI_CLIENTS.get(uri)
-    if matched_client is None:
-        new_client = pymongo.MongoClient(uri, **kwargs)
-        if new_client is not None:
-            MONGO_URI_CLIENTS[uri] = new_client
-        return new_client
-    return matched_client
-
-
-def mongo_database(name: str, **kw):
-    client = mongo_client(**kw)
-    return client.get_database(name)
-
-
-def mongo_table(db: str, name: str, **kw):
-    database = mongo_database(db, **kw)
-    return database.get_collection(name)
-
-
-def int2long(param: int):
-    """int 转换成 long """
-    return bson.int64.Int64(param)
-
-
-def object_id(_id: str):
-    return bson.objectid.ObjectId(_id)
-
-
-# ---------------------------------- es ----------------------------------
-def es_client(cfg=None):
-    if cfg is None:
-        cfg = es_conf
-    return Elasticsearch([{"host": cfg['host'], "port": cfg['port']}])
-
-
-def es_query(title: str, publish_time: int):
-    """
-    查询es
-
-    :param title: 标题
-    :param publish_time: 发布时间
-    :return:
-    """
-    client = es_client()
-    stime = publish_time - 432000  # 往前推5天
-    etime = publish_time + 432000
-    # 通过发布标题和发布时间范围查询
-    query = {
-        "query": {
-            "bool": {
-                "must": [
-                    {
-                        "multi_match": {
-                            "query": title,
-                            "type": "phrase",
-                            "fields": ["title"]
-                        }
-                    },
-                    {"range": {'publishtime': {"from": stime, "to": etime}}}
-                ]
-            }
-        }
-    }
-    result = client.search(index=es_conf['db'], body=query, request_timeout=100)
-    total = int(result['hits']['total'])
-    return total
-
-
-# ---------------------------------- redis ----------------------------------
-def redis_client(cfg=None):
-    if cfg is None:
-        cfg = redis_conf
-
-    return redis.StrictRedis(host=cfg['host'],
-                             port=cfg['port'],
-                             password=cfg['pwd'],
-                             db=cfg['db'], decode_responses=True)
-
-
-def redis_cluster():
-    startup_nodes = redis_startup_nodes
-    return RedisCluster(startup_nodes=startup_nodes, decode_responses=True)

zgztb_cookie/utils/encode_info.py

@@ -1,710 +0,0 @@
-import execjs, json
-
-script = '''
-function CurentTime() {
-    var now = new Date();
-    var year = now.getFullYear();
-    //年
-    var month = now.getMonth() + 1;
-    //月
-    var day = now.getDate();
-    //日
-    var hh = now.getHours();
-    //时
-    var mm = now.getMinutes();
-    //分
-    var ss = now.getSeconds();
-    if (ss > 30) {
-        if (mm == 59) {
-            hh = hh + 1;
-            mm = 0;
-        } else {
-            mm = mm + 1;
-        }
-    }
-    var clock = year + "-";
-    if (month < 10)
-        clock += "0";
-    clock += month + "-";
-    if (day < 10)
-        clock += "0";
-    clock += day + " ";
-    if (hh < 10)
-        clock += "0";
-    clock += hh;
-    //clock += hh + ":";
-    //if (mm < 10) clock += '0';
-    //clock += mm;
-    return (clock);
-};
-function getKeyBytes(key){
- var keyBytes = new Array();
- var leng = key.length;
- var iterator = parseInt(leng/4);
- var remainder = leng%4;
- var i = 0;
- for(i = 0;i < iterator; i ++){
-   keyBytes[i] = strToBt(key.substring(i*4+0,i*4+4));
- }
- if(remainder > 0){
-   keyBytes[i] = strToBt(key.substring(i*4+0,leng));
- }    
- return keyBytes;
-};
-function strToBt(str){  
- var leng = str.length;
- var bt = new Array(64);
- if(leng < 4){
-   var i=0,j=0,p=0,q=0;
-   for(i = 0;i<leng;i++){
-     var k = str.charCodeAt(i);
-     for(j=0;j<16;j++){      
-       var pow=1,m=0;
-       for(m=15;m>j;m--){
-         pow *= 2;
-       }        
-       bt[16*i+j]=parseInt(k/pow)%2;
-     }
-   }
-   for(p = leng;p<4;p++){
-     var k = 0;
-     for(q=0;q<16;q++){      
-       var pow=1,m=0;
-       for(m=15;m>q;m--){
-         pow *= 2;
-       }        
-       bt[16*p+q]=parseInt(k/pow)%2;
-     }
-   }  
- }else{
-   for(i = 0;i<4;i++){
-     var k = str.charCodeAt(i);
-     for(j=0;j<16;j++){      
-       var pow=1;
-       for(m=15;m>j;m--){
-         pow *= 2;
-       }        
-       bt[16*i+j]=parseInt(k/pow)%2;
-     }
-   }  
- }
- return bt;
-};
-function enc(dataByte,keyByte){  
- var keys = generateKeys(keyByte);    
- var ipByte   = initPermute(dataByte);  
- var ipLeft   = new Array(32);
- var ipRight  = new Array(32);
- var tempLeft = new Array(32);
- var i = 0,j = 0,k = 0,m = 0, n = 0;
- for(k = 0;k < 32;k ++){
-   ipLeft[k] = ipByte[k];
-   ipRight[k] = ipByte[32+k];
- }    
- for(i = 0;i < 16;i ++){
-   for(j = 0;j < 32;j ++){
-     tempLeft[j] = ipLeft[j];
-     ipLeft[j] = ipRight[j];      
-   }  
-   var key = new Array(48);
-   for(m = 0;m < 48;m ++){
-     key[m] = keys[i][m];
-   }
-   var  tempRight = xor(pPermute(sBoxPermute(xor(expandPermute(ipRight),key))), tempLeft);      
-   for(n = 0;n < 32;n ++){
-     ipRight[n] = tempRight[n];
-   }  
-   
- }  
- 
- 
- var finalData =new Array(64);
- for(i = 0;i < 32;i ++){
-   finalData[i] = ipRight[i];
-   finalData[32+i] = ipLeft[i];
- }
- return finallyPermute(finalData)} 
-;
-function generateKeys(keyByte){    
- var key   = new Array(56);
- var keys = new Array();  
- 
- keys[ 0] = new Array();
- keys[ 1] = new Array();
- keys[ 2] = new Array();
- keys[ 3] = new Array();
- keys[ 4] = new Array();
- keys[ 5] = new Array();
- keys[ 6] = new Array();
- keys[ 7] = new Array();
- keys[ 8] = new Array();
- keys[ 9] = new Array();
- keys[10] = new Array();
- keys[11] = new Array();
- keys[12] = new Array();
- keys[13] = new Array();
- keys[14] = new Array();
- keys[15] = new Array();  
- var loop = [1,1,2,2,2,2,2,2,1,2,2,2,2,2,2,1];
-
- for(i=0;i<7;i++){
-   for(j=0,k=7;j<8;j++,k--){
-     key[i*8+j]=keyByte[8*k+i];
-   }
- }    
- 
- var i = 0;
- for(i = 0;i < 16;i ++){
-   var tempLeft=0;
-   var tempRight=0;
-   for(j = 0; j < loop[i];j ++){          
-     tempLeft = key[0];
-     tempRight = key[28];
-     for(k = 0;k < 27 ;k ++){
-       key[k] = key[k+1];
-       key[28+k] = key[29+k];
-     }  
-     key[27]=tempLeft;
-     key[55]=tempRight;
-   }
-   var tempKey = new Array(48);
-   tempKey[ 0] = key[13];
-   tempKey[ 1] = key[16];
-   tempKey[ 2] = key[10];
-   tempKey[ 3] = key[23];
-   tempKey[ 4] = key[ 0];
-   tempKey[ 5] = key[ 4];
-   tempKey[ 6] = key[ 2];
-   tempKey[ 7] = key[27];
-   tempKey[ 8] = key[14];
-   tempKey[ 9] = key[ 5];
-   tempKey[10] = key[20];
-   tempKey[11] = key[ 9];
-   tempKey[12] = key[22];
-   tempKey[13] = key[18];
-   tempKey[14] = key[11];
-   tempKey[15] = key[ 3];
-   tempKey[16] = key[25];
-   tempKey[17] = key[ 7];
-   tempKey[18] = key[15];
-   tempKey[19] = key[ 6];
-   tempKey[20] = key[26];
-   tempKey[21] = key[19];
-   tempKey[22] = key[12];
-   tempKey[23] = key[ 1];
-   tempKey[24] = key[40];
-   tempKey[25] = key[51];
-   tempKey[26] = key[30];
-   tempKey[27] = key[36];
-   tempKey[28] = key[46];
-   tempKey[29] = key[54];
-   tempKey[30] = key[29];
-   tempKey[31] = key[39];
-   tempKey[32] = key[50];
-   tempKey[33] = key[44];
-   tempKey[34] = key[32];
-   tempKey[35] = key[47];
-   tempKey[36] = key[43];
-   tempKey[37] = key[48];
-   tempKey[38] = key[38];
-   tempKey[39] = key[55];
-   tempKey[40] = key[33];
-   tempKey[41] = key[52];
-   tempKey[42] = key[45];
-   tempKey[43] = key[41];
-   tempKey[44] = key[49];
-   tempKey[45] = key[35];
-   tempKey[46] = key[28];
-   tempKey[47] = key[31];
-   switch(i){
-     case 0: for(m=0;m < 48 ;m++){ keys[ 0][m] = tempKey[m]; } break;
-     case 1: for(m=0;m < 48 ;m++){ keys[ 1][m] = tempKey[m]; } break;
-     case 2: for(m=0;m < 48 ;m++){ keys[ 2][m] = tempKey[m]; } break;
-     case 3: for(m=0;m < 48 ;m++){ keys[ 3][m] = tempKey[m]; } break;
-     case 4: for(m=0;m < 48 ;m++){ keys[ 4][m] = tempKey[m]; } break;
-     case 5: for(m=0;m < 48 ;m++){ keys[ 5][m] = tempKey[m]; } break;
-     case 6: for(m=0;m < 48 ;m++){ keys[ 6][m] = tempKey[m]; } break;
-     case 7: for(m=0;m < 48 ;m++){ keys[ 7][m] = tempKey[m]; } break;
-     case 8: for(m=0;m < 48 ;m++){ keys[ 8][m] = tempKey[m]; } break;
-     case 9: for(m=0;m < 48 ;m++){ keys[ 9][m] = tempKey[m]; } break;
-     case 10: for(m=0;m < 48 ;m++){ keys[10][m] = tempKey[m]; } break;
-     case 11: for(m=0;m < 48 ;m++){ keys[11][m] = tempKey[m]; } break;
-     case 12: for(m=0;m < 48 ;m++){ keys[12][m] = tempKey[m]; } break;
-     case 13: for(m=0;m < 48 ;m++){ keys[13][m] = tempKey[m]; } break;
-     case 14: for(m=0;m < 48 ;m++){ keys[14][m] = tempKey[m]; } break;
-     case 15: for(m=0;m < 48 ;m++){ keys[15][m] = tempKey[m]; } break;
-   }
- }
- return keys;  
-};
-function initPermute(originalData){
- var ipByte = new Array(64);
- for (i = 0, m = 1, n = 0; i < 4; i++, m += 2, n += 2) {
-   for (j = 7, k = 0; j >= 0; j--, k++) {
-     ipByte[i * 8 + k] = originalData[j * 8 + m];
-     ipByte[i * 8 + k + 32] = originalData[j * 8 + n];
-   }
- }    
- return ipByte;
-};
-function xor(byteOne,byteTwo){  
- var xorByte = new Array(byteOne.length);
- for(i = 0;i < byteOne.length; i ++){      
-   xorByte[i] = byteOne[i] ^ byteTwo[i];
- }  
- return xorByte;
-};
-function pPermute(sBoxByte){
- var pBoxPermute = new Array(32);
- pBoxPermute[ 0] = sBoxByte[15]; 
- pBoxPermute[ 1] = sBoxByte[ 6]; 
- pBoxPermute[ 2] = sBoxByte[19]; 
- pBoxPermute[ 3] = sBoxByte[20]; 
- pBoxPermute[ 4] = sBoxByte[28]; 
- pBoxPermute[ 5] = sBoxByte[11]; 
- pBoxPermute[ 6] = sBoxByte[27]; 
- pBoxPermute[ 7] = sBoxByte[16]; 
- pBoxPermute[ 8] = sBoxByte[ 0]; 
- pBoxPermute[ 9] = sBoxByte[14]; 
- pBoxPermute[10] = sBoxByte[22]; 
- pBoxPermute[11] = sBoxByte[25]; 
- pBoxPermute[12] = sBoxByte[ 4]; 
- pBoxPermute[13] = sBoxByte[17]; 
- pBoxPermute[14] = sBoxByte[30]; 
- pBoxPermute[15] = sBoxByte[ 9]; 
- pBoxPermute[16] = sBoxByte[ 1]; 
- pBoxPermute[17] = sBoxByte[ 7]; 
- pBoxPermute[18] = sBoxByte[23]; 
- pBoxPermute[19] = sBoxByte[13]; 
- pBoxPermute[20] = sBoxByte[31]; 
- pBoxPermute[21] = sBoxByte[26]; 
- pBoxPermute[22] = sBoxByte[ 2]; 
- pBoxPermute[23] = sBoxByte[ 8]; 
- pBoxPermute[24] = sBoxByte[18]; 
- pBoxPermute[25] = sBoxByte[12]; 
- pBoxPermute[26] = sBoxByte[29]; 
- pBoxPermute[27] = sBoxByte[ 5]; 
- pBoxPermute[28] = sBoxByte[21]; 
- pBoxPermute[29] = sBoxByte[10]; 
- pBoxPermute[30] = sBoxByte[ 3]; 
- pBoxPermute[31] = sBoxByte[24];    
- return pBoxPermute;
-};
-function sBoxPermute(expandByte){
- 
-   var sBoxByte = new Array(32);
-   var binary = "";
-   var s1 = [
-       [14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7],
-       [0, 15, 7, 4, 14, 2, 13, 1, 10, 6, 12, 11, 9, 5, 3, 8],
-       [4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, 3, 10, 5, 0],
-       [15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13 ]];
-
-       /* Table - s2 */
-   var s2 = [
-       [15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10],
-       [3, 13, 4, 7, 15, 2, 8, 14, 12, 0, 1, 10, 6, 9, 11, 5],
-       [0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, 9, 3, 2, 15],
-       [13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9 ]];
-
-       /* Table - s3 */
-   var s3= [
-       [10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8],
-       [13, 7, 0, 9, 3, 4, 6, 10, 2, 8, 5, 14, 12, 11, 15, 1],
-       [13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, 5, 10, 14, 7],
-       [1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12 ]];
-       /* Table - s4 */
-   var s4 = [
-       [7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15],
-       [13, 8, 11, 5, 6, 15, 0, 3, 4, 7, 2, 12, 1, 10, 14, 9],
-       [10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, 5, 2, 8, 4],
-       [3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14 ]];
-
-       /* Table - s5 */
-   var s5 = [
-       [2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9],
-       [14, 11, 2, 12, 4, 7, 13, 1, 5, 0, 15, 10, 3, 9, 8, 6],
-       [4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, 6, 3, 0, 14],
-       [11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3 ]];
-
-       /* Table - s6 */
-   var s6 = [
-       [12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11],
-       [10, 15, 4, 2, 7, 12, 9, 5, 6, 1, 13, 14, 0, 11, 3, 8],
-       [9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, 1, 13, 11, 6],
-       [4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13 ]];
-
-       /* Table - s7 */
-   var s7 = [
-       [4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1],
-       [13, 0, 11, 7, 4, 9, 1, 10, 14, 3, 5, 12, 2, 15, 8, 6],
-       [1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, 0, 5, 9, 2],
-       [6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12]];
-
-       /* Table - s8 */
-   var s8 = [
-       [13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7],
-       [1, 15, 13, 8, 10, 3, 7, 4, 12, 5, 6, 11, 0, 14, 9, 2],
-       [7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, 15, 3, 5, 8],
-       [2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11]];
-   
-   for(m=0;m<8;m++){
-   var i=0,j=0;
-   i = expandByte[m*6+0]*2+expandByte[m*6+5];
-   j = expandByte[m * 6 + 1] * 2 * 2 * 2 
-     + expandByte[m * 6 + 2] * 2* 2 
-     + expandByte[m * 6 + 3] * 2 
-     + expandByte[m * 6 + 4];
-   switch (m) {
-     case 0 :
-       binary = getBoxBinary(s1[i][j]);
-       break;
-     case 1 :
-       binary = getBoxBinary(s2[i][j]);
-       break;
-     case 2 :
-       binary = getBoxBinary(s3[i][j]);
-       break;
-     case 3 :
-       binary = getBoxBinary(s4[i][j]);
-       break;
-     case 4 :
-       binary = getBoxBinary(s5[i][j]);
-       break;
-     case 5 :
-       binary = getBoxBinary(s6[i][j]);
-       break;
-     case 6 :
-       binary = getBoxBinary(s7[i][j]);
-       break;
-     case 7 :
-       binary = getBoxBinary(s8[i][j]);
-       break;
-   }      
-   sBoxByte[m*4+0] = parseInt(binary.substring(0,1));
-   sBoxByte[m*4+1] = parseInt(binary.substring(1,2));
-   sBoxByte[m*4+2] = parseInt(binary.substring(2,3));
-   sBoxByte[m*4+3] = parseInt(binary.substring(3,4));
- }
- return sBoxByte;
-};
-function expandPermute(rightData){  
- var epByte = new Array(48);
- for (i = 0; i < 8; i++) {
-   if (i == 0) {
-     epByte[i * 6 + 0] = rightData[31];
-   } else {
-     epByte[i * 6 + 0] = rightData[i * 4 - 1];
-   }
-   epByte[i * 6 + 1] = rightData[i * 4 + 0];
-   epByte[i * 6 + 2] = rightData[i * 4 + 1];
-   epByte[i * 6 + 3] = rightData[i * 4 + 2];
-   epByte[i * 6 + 4] = rightData[i * 4 + 3];
-   if (i == 7) {
-     epByte[i * 6 + 5] = rightData[0];
-   } else {
-     epByte[i * 6 + 5] = rightData[i * 4 + 4];
-   }
- }      
- return epByte;
-};
-function getBoxBinary(i) {
- var binary = "";
- switch (i) {
-   case 0 :binary = "0000";break;
-   case 1 :binary = "0001";break;
-   case 2 :binary = "0010";break;
-   case 3 :binary = "0011";break;
-   case 4 :binary = "0100";break;
-   case 5 :binary = "0101";break;
-   case 6 :binary = "0110";break;
-   case 7 :binary = "0111";break;
-   case 8 :binary = "1000";break;
-   case 9 :binary = "1001";break;
-   case 10 :binary = "1010";break;
-   case 11 :binary = "1011";break;
-   case 12 :binary = "1100";break;
-   case 13 :binary = "1101";break;
-   case 14 :binary = "1110";break;
-   case 15 :binary = "1111";break;
- }
- return binary;
-};
-function finallyPermute(endByte){    
- var fpByte = new Array(64);  
- fpByte[ 0] = endByte[39]; 
- fpByte[ 1] = endByte[ 7]; 
- fpByte[ 2] = endByte[47]; 
- fpByte[ 3] = endByte[15]; 
- fpByte[ 4] = endByte[55]; 
- fpByte[ 5] = endByte[23]; 
- fpByte[ 6] = endByte[63]; 
- fpByte[ 7] = endByte[31]; 
- fpByte[ 8] = endByte[38]; 
- fpByte[ 9] = endByte[ 6]; 
- fpByte[10] = endByte[46]; 
- fpByte[11] = endByte[14]; 
- fpByte[12] = endByte[54]; 
- fpByte[13] = endByte[22]; 
- fpByte[14] = endByte[62]; 
- fpByte[15] = endByte[30]; 
- fpByte[16] = endByte[37]; 
- fpByte[17] = endByte[ 5]; 
- fpByte[18] = endByte[45]; 
- fpByte[19] = endByte[13]; 
- fpByte[20] = endByte[53]; 
- fpByte[21] = endByte[21]; 
- fpByte[22] = endByte[61]; 
- fpByte[23] = endByte[29]; 
- fpByte[24] = endByte[36]; 
- fpByte[25] = endByte[ 4]; 
- fpByte[26] = endByte[44]; 
- fpByte[27] = endByte[12]; 
- fpByte[28] = endByte[52]; 
- fpByte[29] = endByte[20]; 
- fpByte[30] = endByte[60]; 
- fpByte[31] = endByte[28]; 
- fpByte[32] = endByte[35]; 
- fpByte[33] = endByte[ 3]; 
- fpByte[34] = endByte[43]; 
- fpByte[35] = endByte[11]; 
- fpByte[36] = endByte[51]; 
- fpByte[37] = endByte[19]; 
- fpByte[38] = endByte[59]; 
- fpByte[39] = endByte[27]; 
- fpByte[40] = endByte[34]; 
- fpByte[41] = endByte[ 2]; 
- fpByte[42] = endByte[42]; 
- fpByte[43] = endByte[10]; 
- fpByte[44] = endByte[50]; 
- fpByte[45] = endByte[18]; 
- fpByte[46] = endByte[58]; 
- fpByte[47] = endByte[26]; 
- fpByte[48] = endByte[33]; 
- fpByte[49] = endByte[ 1]; 
- fpByte[50] = endByte[41]; 
- fpByte[51] = endByte[ 9]; 
- fpByte[52] = endByte[49]; 
- fpByte[53] = endByte[17]; 
- fpByte[54] = endByte[57]; 
- fpByte[55] = endByte[25]; 
- fpByte[56] = endByte[32]; 
- fpByte[57] = endByte[ 0]; 
- fpByte[58] = endByte[40]; 
- fpByte[59] = endByte[ 8]; 
- fpByte[60] = endByte[48]; 
- fpByte[61] = endByte[16]; 
- fpByte[62] = endByte[56]; 
- fpByte[63] = endByte[24];
- return fpByte;
-};
-function bt64ToHex(byteData){
- var hex = "";
- for(i = 0;i<16;i++){
-   var bt = "";
-   for(j=0;j<4;j++){    
-     bt += byteData[i*4+j];
-   }    
-   hex+=bt4ToHex(bt);
- }
- return hex;
-};
-function bt4ToHex(binary) {
- var hex;
- switch (binary) {
-   case "0000" : hex = "0"; break;
-   case "0001" : hex = "1"; break;
-   case "0010" : hex = "2"; break;
-   case "0011" : hex = "3"; break;
-   case "0100" : hex = "4"; break;
-   case "0101" : hex = "5"; break;
-   case "0110" : hex = "6"; break;
-   case "0111" : hex = "7"; break;
-   case "1000" : hex = "8"; break;
-   case "1001" : hex = "9"; break;
-   case "1010" : hex = "A"; break;
-   case "1011" : hex = "B"; break;
-   case "1100" : hex = "C"; break;
-   case "1101" : hex = "D"; break;
-   case "1110" : hex = "E"; break;
-   case "1111" : hex = "F"; break;
- }
- return hex;
-};
-
-
-
-
-
-
-function strEnc(data){
-var firstKey = CurentTime();
-var secondKey = "cebpubservice";
-var thirdKey = "iiss";
-
- var leng = data.length;
- var encData = "";
- var firstKeyBt,secondKeyBt,thirdKeyBt,firstLength,secondLength,thirdLength;
- if(firstKey != null && firstKey != ""){    
-   firstKeyBt = getKeyBytes(firstKey);
-   firstLength = firstKeyBt.length;
- }
- if(secondKey != null && secondKey != ""){
-   secondKeyBt = getKeyBytes(secondKey);
-   secondLength = secondKeyBt.length;
- }
- if(thirdKey != null && thirdKey != ""){
-   thirdKeyBt = getKeyBytes(thirdKey);
-   thirdLength = thirdKeyBt.length;
- }  
- 
- if(leng > 0){
-   if(leng < 4){
-     var bt = strToBt(data);      
-     var encByte ;
-     if(firstKey != null && firstKey !="" && secondKey != null && secondKey != "" && thirdKey != null && thirdKey != ""){
-       var tempBt;
-       var x,y,z;
-       tempBt = bt;        
-       for(x = 0;x < firstLength ;x ++){
-         tempBt = enc(tempBt,firstKeyBt[x]);
-       }
-       for(y = 0;y < secondLength ;y ++){
-         tempBt = enc(tempBt,secondKeyBt[y]);
-       }
-       for(z = 0;z < thirdLength ;z ++){
-         tempBt = enc(tempBt,thirdKeyBt[z]);
-       }        
-       encByte = tempBt;        
-     }else{
-       if(firstKey != null && firstKey !="" && secondKey != null && secondKey != ""){
-         var tempBt;
-         var x,y;
-         tempBt = bt;
-         for(x = 0;x < firstLength ;x ++){
-           tempBt = enc(tempBt,firstKeyBt[x]);
-         }
-         for(y = 0;y < secondLength ;y ++){
-           tempBt = enc(tempBt,secondKeyBt[y]);
-         }
-         encByte = tempBt;
-       }else{
-         if(firstKey != null && firstKey !=""){            
-           var tempBt;
-           var x = 0;
-           tempBt = bt;            
-           for(x = 0;x < firstLength ;x ++){
-             tempBt = enc(tempBt,firstKeyBt[x]);
-           }
-           encByte = tempBt;
-         }
-       }        
-     }
-     encData = bt64ToHex(encByte);
-   }else{
-     var iterator = parseInt(leng/4);
-     var remainder = leng%4;
-     var i=0;      
-     for(i = 0;i < iterator;i++){
-       var tempData = data.substring(i*4+0,i*4+4);
-       var tempByte = strToBt(tempData);
-       var encByte ;
-       if(firstKey != null && firstKey !="" && secondKey != null && secondKey != "" && thirdKey != null && thirdKey != ""){
-         var tempBt;
-         var x,y,z;
-         tempBt = tempByte;
-         for(x = 0;x < firstLength ;x ++){
-           tempBt = enc(tempBt,firstKeyBt[x]);
-         }
-         for(y = 0;y < secondLength ;y ++){
-           tempBt = enc(tempBt,secondKeyBt[y]);
-         }
-         for(z = 0;z < thirdLength ;z ++){
-           tempBt = enc(tempBt,thirdKeyBt[z]);
-         }
-         encByte = tempBt;
-       }else{
-         if(firstKey != null && firstKey !="" && secondKey != null && secondKey != ""){
-           var tempBt;
-           var x,y;
-           tempBt = tempByte;
-           for(x = 0;x < firstLength ;x ++){
-             tempBt = enc(tempBt,firstKeyBt[x]);
-           }
-           for(y = 0;y < secondLength ;y ++){
-             tempBt = enc(tempBt,secondKeyBt[y]);
-           }
-           encByte = tempBt;
-         }else{
-           if(firstKey != null && firstKey !=""){                      
-             var tempBt;
-             var x;
-             tempBt = tempByte;
-             for(x = 0;x < firstLength ;x ++){                
-               tempBt = enc(tempBt,firstKeyBt[x]);
-             }
-             encByte = tempBt;              
-           }
-         }
-       }
-       encData += bt64ToHex(encByte);
-     }      
-     if(remainder > 0){
-       var remainderData = data.substring(iterator*4+0,leng);
-       var tempByte = strToBt(remainderData);
-       var encByte ;
-       if(firstKey != null && firstKey !="" && secondKey != null && secondKey != "" && thirdKey != null && thirdKey != ""){
-         var tempBt;
-         var x,y,z;
-         tempBt = tempByte;
-         for(x = 0;x < firstLength ;x ++){
-           tempBt = enc(tempBt,firstKeyBt[x]);
-         }
-         for(y = 0;y < secondLength ;y ++){
-           tempBt = enc(tempBt,secondKeyBt[y]);
-         }
-         for(z = 0;z < thirdLength ;z ++){
-           tempBt = enc(tempBt,thirdKeyBt[z]);
-         }
-         encByte = tempBt;
-       }else{
-         if(firstKey != null && firstKey !="" && secondKey != null && secondKey != ""){
-           var tempBt;
-           var x,y;
-           tempBt = tempByte;
-           for(x = 0;x < firstLength ;x ++){
-             tempBt = enc(tempBt,firstKeyBt[x]);
-           }
-           for(y = 0;y < secondLength ;y ++){
-             tempBt = enc(tempBt,secondKeyBt[y]);
-           }
-           encByte = tempBt;
-         }else{
-           if(firstKey != null && firstKey !=""){            
-             var tempBt;
-             var x;
-             tempBt = tempByte;
-             for(x = 0;x < firstLength ;x ++){
-               tempBt = enc(tempBt,firstKeyBt[x]);
-             }
-             encByte = tempBt;
-           }
-         }
-       }
-       encData += bt64ToHex(encByte);
-     }                
-   }
- }
- return encData;
-}
-'''
-
-
-def encode_info(plaintext):
-    ctx = execjs.compile(script)
-    res = ctx.call('strEnc', plaintext)
-    return res
-
-
-if __name__ == '__main__':
-    print(encode_info("2447016584bf865cbe60"))

zgztb_cookie/utils/log.py

@@ -1,14 +0,0 @@
-from pathlib import Path
-
-from loguru import logger
-
-_absolute = Path(__file__).absolute().parent.parent
-_log_path = (_absolute / 'logs/crawl-{time:YYYY-MM-DD}.log').resolve()
-logger.add(
-    _log_path,
-    format='{time:YYYY-MM-DD HH:mm:ss} - {level} - {message}',
-    level='INFO',
-    rotation='00:00',
-    retention='1 week',
-    encoding='utf-8',
-)

zgztb_cookie/utils/tools.py

@@ -1,30 +0,0 @@
-import hashlib
-
-
-def encrypt(text: str):
-    """
-    十六进制数字文本摘要值
-
-    @param text: 字符串文本
-    @return: 十六进制编码
-    """
-    _hash = hashlib.sha256()
-    _hash.update(text.encode("utf-8"))
-    return _hash.hexdigest()
-
-
-def redis_exists(href: str, redis_cluster):
-    href_sign = encrypt(href)
-    list_sign = 'pylist_' + href_sign
-    if redis_cluster.exists(href_sign) > 0 or redis_cluster.exists(list_sign) > 0:
-        return True
-    return False
-
-
-def redis_set(href: str, redis_cluster, val=1):
-    if not redis_exists(href, redis_cluster):
-        sign = 'pylist_' + encrypt(href)
-        expiration_time = 86400 * 365 * 2
-        redis_cluster.set(sign, val, ex=expiration_time)
-        return True
-    return False