代码合并6.16

a_gctdzztbjypt_jyxx/gctdzztbjypt.js

@@ -0,0 +1,187 @@
+const wA = 0
+    , bA = 16
+    ,
+    BA = [214, 144, 233, 254, 204, 225, 61, 183, 22, 182, 20, 194, 40, 251, 44, 5, 43, 103, 154, 118, 42, 190, 4, 195, 170, 68, 19, 38, 73, 134, 6, 153, 156, 66, 80, 244, 145, 239, 152, 122, 51, 84, 11, 67, 237, 207, 172, 98, 228, 179, 28, 169, 201, 8, 232, 149, 128, 223, 148, 250, 117, 143, 63, 166, 71, 7, 167, 252, 243, 115, 23, 186, 131, 89, 60, 25, 230, 133, 79, 168, 104, 107, 129, 178, 113, 100, 218, 139, 248, 235, 15, 75, 112, 86, 157, 53, 30, 36, 14, 94, 99, 88, 209, 162, 37, 34, 124, 59, 1, 33, 120, 135, 212, 0, 70, 87, 159, 211, 39, 82, 76, 54, 2, 231, 160, 196, 200, 158, 234, 191, 138, 210, 64, 199, 56, 181, 163, 247, 242, 206, 249, 97, 21, 161, 224, 174, 93, 164, 155, 52, 26, 85, 173, 147, 50, 48, 245, 140, 177, 227, 29, 246, 226, 46, 130, 102, 202, 96, 192, 41, 35, 171, 13, 83, 78, 111, 213, 219, 55, 69, 222, 253, 142, 47, 3, 255, 106, 114, 109, 108, 91, 81, 141, 27, 175, 146, 187, 221, 188, 127, 17, 217, 92, 65, 31, 16, 90, 216, 10, 193, 49, 136, 165, 205, 123, 189, 45, 116, 208, 18, 184, 229, 180, 176, 137, 105, 151, 74, 12, 150, 119, 126, 101, 185, 241, 9, 197, 110, 198, 132, 24, 240, 125, 236, 58, 220, 77, 32, 121, 238, 95, 62, 215, 203, 57, 72]
+    ,
+    CA = [462357, 472066609, 943670861, 1415275113, 1886879365, 2358483617, 2830087869, 3301692121, 3773296373, 4228057617, 404694573, 876298825, 1347903077, 1819507329, 2291111581, 2762715833, 3234320085, 3705924337, 4177462797, 337322537, 808926789, 1280531041, 1752135293, 2223739545, 2695343797, 3166948049, 3638552301, 4110090761, 269950501, 741554753, 1213159005, 1684763257];
+
+
+function SA(e, t) {
+    const n = 31 & t;
+    return e << n | e >>> 32 - n
+}
+
+
+function FA(e) {
+    return (255 & BA[e >>> 24 & 255]) << 24 | (255 & BA[e >>> 16 & 255]) << 16 | (255 & BA[e >>> 8 & 255]) << 8 | 255 & BA[255 & e]
+}
+
+
+function EA(e) {
+    return e ^ SA(e, 2) ^ SA(e, 10) ^ SA(e, 18) ^ SA(e, 24)
+}
+
+
+function kA(e) {
+    return e ^ SA(e, 13) ^ SA(e, 23)
+}
+
+
+function xA(e) {
+    const t = [];
+    for (let n = 0, r = e.length; n < r; n += 2)
+        t.push(parseInt(e.substr(n, 2), 16));
+    return t
+}
+
+
+function IA(e, t, n) {
+    const r = new Array(4)
+        , o = new Array(4);
+    for (let a = 0; a < 4; a++)
+        o[0] = 255 & e[4 * a],
+            o[1] = 255 & e[4 * a + 1],
+            o[2] = 255 & e[4 * a + 2],
+            o[3] = 255 & e[4 * a + 3],
+            r[a] = o[0] << 24 | o[1] << 16 | o[2] << 8 | o[3];
+    for (let a, i = 0; i < 32; i += 4)
+        a = r[1] ^ r[2] ^ r[3] ^ n[i + 0],
+            r[0] ^= EA(FA(a)),
+            a = r[2] ^ r[3] ^ r[0] ^ n[i + 1],
+            r[1] ^= EA(FA(a)),
+            a = r[3] ^ r[0] ^ r[1] ^ n[i + 2],
+            r[2] ^= EA(FA(a)),
+            a = r[0] ^ r[1] ^ r[2] ^ n[i + 3],
+            r[3] ^= EA(FA(a));
+    for (let a = 0; a < 16; a += 4)
+        t[a] = r[3 - a / 4] >>> 24 & 255,
+            t[a + 1] = r[3 - a / 4] >>> 16 & 255,
+            t[a + 2] = r[3 - a / 4] >>> 8 & 255,
+            t[a + 3] = 255 & r[3 - a / 4]
+}
+
+
+function UA(e, t, n, {padding: r = "pkcs#7", mode: o, iv: a = [], output: i = "string"} = {}) {
+    if ("cbc" === o && ("string" == typeof a && (a = xA(a)), 16 !== a.length))
+        throw new Error("iv is invalid");
+    if ("string" == typeof t && (t = xA(t)),
+    16 !== t.length)
+        throw new Error("key is invalid");
+    if (e = "string" == typeof e ? n !== wA ? function (e) {
+        const t = [];
+        for (let n = 0, r = e.length; n < r; n++) {
+            const r = e.codePointAt(n);
+            if (r <= 127)
+                t.push(r);
+            else if (r <= 2047)
+                t.push(192 | r >>> 6),
+                    t.push(128 | 63 & r);
+            else if (r <= 55295 || r >= 57344 && r <= 65535)
+                t.push(224 | r >>> 12),
+                    t.push(128 | r >>> 6 & 63),
+                    t.push(128 | 63 & r);
+            else {
+                if (!(r >= 65536 && r <= 1114111))
+                    throw t.push(r),
+                        new Error("input is not supported");
+                n++,
+                    t.push(240 | r >>> 18 & 28),
+                    t.push(128 | r >>> 12 & 63),
+                    t.push(128 | r >>> 6 & 63),
+                    t.push(128 | 63 & r)
+            }
+        }
+        return t
+    }(e) : xA(e) : [...e],
+    ("pkcs#5" === r || "pkcs#7" === r) && n !== wA) {
+        const t = bA - e.length % bA;
+        for (let n = 0; n < t; n++)
+            e.push(t)
+    }
+    const s = new Array(32);
+    !function (e, t, n) {
+        const r = new Array(4)
+            , o = new Array(4);
+        for (let a = 0; a < 4; a++)
+            o[0] = 255 & e[0 + 4 * a],
+                o[1] = 255 & e[1 + 4 * a],
+                o[2] = 255 & e[2 + 4 * a],
+                o[3] = 255 & e[3 + 4 * a],
+                r[a] = o[0] << 24 | o[1] << 16 | o[2] << 8 | o[3];
+        r[0] ^= 2746333894,
+            r[1] ^= 1453994832,
+            r[2] ^= 1736282519,
+            r[3] ^= 2993693404;
+        for (let a, i = 0; i < 32; i += 4)
+            a = r[1] ^ r[2] ^ r[3] ^ CA[i + 0],
+                t[i + 0] = r[0] ^= kA(FA(a)),
+                a = r[2] ^ r[3] ^ r[0] ^ CA[i + 1],
+                t[i + 1] = r[1] ^= kA(FA(a)),
+                a = r[3] ^ r[0] ^ r[1] ^ CA[i + 2],
+                t[i + 2] = r[2] ^= kA(FA(a)),
+                a = r[0] ^ r[1] ^ r[2] ^ CA[i + 3],
+                t[i + 3] = r[3] ^= kA(FA(a));
+        if (n === wA)
+            for (let a, i = 0; i < 16; i++)
+                a = t[i],
+                    t[i] = t[31 - i],
+                    t[31 - i] = a
+    }(t, s, n);
+    const l = [];
+    let A = a
+        , u = e.length
+        , c = 0;
+    for (; u >= bA;) {
+        const t = e.slice(c, c + 16)
+            , r = new Array(16);
+        if ("cbc" === o)
+            for (let e = 0; e < bA; e++)
+                n !== wA && (t[e] ^= A[e]);
+        IA(t, r, s);
+        for (let e = 0; e < bA; e++)
+            "cbc" === o && n === wA && (r[e] ^= A[e]),
+                l[c + e] = r[e];
+        "cbc" === o && (A = n !== wA ? r : t),
+            u -= bA,
+            c += bA
+    }
+    if (("pkcs#5" === r || "pkcs#7" === r) && n === wA) {
+        const e = l.length
+            , t = l[e - 1];
+        for (let n = 1; n <= t; n++)
+            if (l[e - n] !== t)
+                throw new Error("padding is invalid");
+        l.splice(e - t, t)
+    }
+    return "array" !== i ? n !== wA ? l.map((e => 1 === (e = e.toString(16)).length ? "0" + e : e)).join("") : function (e) {
+        const t = [];
+        for (let n = 0, r = e.length; n < r; n++)
+            e[n] >= 240 && e[n] <= 247 ? (t.push(String.fromCodePoint(((7 & e[n]) << 18) + ((63 & e[n + 1]) << 12) + ((63 & e[n + 2]) << 6) + (63 & e[n + 3]))),
+                n += 3) : e[n] >= 224 && e[n] <= 239 ? (t.push(String.fromCodePoint(((15 & e[n]) << 12) + ((63 & e[n + 1]) << 6) + (63 & e[n + 2]))),
+                n += 2) : e[n] >= 192 && e[n] <= 223 ? (t.push(String.fromCodePoint(((31 & e[n]) << 6) + (63 & e[n + 1]))),
+                n++) : t.push(String.fromCodePoint(e[n]));
+        return t.join("")
+    }(l) : l
+}
+
+
+var QA = {
+    encrypt: (e, t, n) => UA(e, t, 1, n),
+    decrypt: (e, t, n) => UA(e, t, 0, n)
+};
+
+
+var iie = "90bdd291004611ef87fc52540023e781"
+
+function sm4Decrypt(data) {
+    return QA.decrypt(xA(data), iie)
+}
+
+function sm4Encrypt(requestParams) {
+    requestParams = JSON.stringify(requestParams)
+    return QA.encrypt(requestParams, iie)
+}
+
+
+
+

a_gctdzztbjypt_jyxx/交易信息-列表页.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-30
+---------
+@summary: 工采通电子招投标交易平台
+---------
+@author: lzz
+"""
+import random
+import time
+
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+import execjs
+import json
+
+anty = {"01": "招标公告", "02": "答疑公告", "03": "变更公告", "04": "候选人公示", "05": "结果公示",}
+
+def ctx():
+    with open('./gctdzztbjypt.js','r') as fr:
+        ex_js = fr.read()
+    xx = execjs.compile(ex_js)
+    return xx
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+
+    def start_callback(self):
+
+        self.site = "工采通电子招投标交易平台"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('交易信息', 'a_gctdzztbjypt_jyxx', 10),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json;",
+            "Origin": "https://easy-prt.com",
+            "Pragma": "no-cache",
+            "Referer": "https://easy-prt.com/trading",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://easy-prt.com/gykjbid/gykj-bid/bid/portal/getProjectList"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        data = {
+            "announcementType": "",
+            "areaCode": "",
+            "bidStatus": "",
+            "cityCode": "",
+            "industryCategories": "",
+            "industrySegmentation": "",
+            "keyword": "",
+            "provinceCode": "",
+            "releaseTimeType": "lastAMonth",
+            "showTop": 0,
+            "pageNo": page,
+            "pageSize": 10,
+            "t": int(time.time()*1000)
+        }
+        request.data = ctx().call('sm4Encrypt',data)
+        request.headers = self.headers
+
+    def parse(self, request, response):
+
+        menu = request.item
+        info_list = json.loads(ctx().call('sm4Decrypt',response.text)).get('result').get('records')
+        for info in info_list:
+            hid = info.get('id')
+            href = f"https://easy-prt.com/project?id={hid}&type=01&t={int(time.time()*1000)}"
+            title = info.get('projectName').strip()
+            publish_time = info.get('releaseTime')
+
+            area = info.get('provinceName').replace('省','').replace('市','')
+            city = info.get('cityName')
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+
+            list_item.publishtime = publish_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area or "全国"  # 省份 默认:全国
+            list_item.city = city  # 城市 默认 为空
+
+            list_item.unique_key = ('title', publish_time)
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            announcementList = info.get('announcementList')
+            list_item.parse_url = "https://easy-prt.com/gykjbid/gykj-bid/bid/portal/getProjectContent"
+            for ann in announcementList:
+                contentId = ann.get('contentId')
+                announcementType = anty.get(ann.get('announcementType',''),'')
+                list_item.title = title + f"_{announcementType}" # 标题
+                params = {"contentId":contentId}
+                list_item.request_params = {"params":params}
+                list_item.is_check_spider = False
+
+                yield list_item
+                time.sleep(random.random())
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:gctdzztbjypt_jyxx").start()

a_gctdzztbjypt_jyxx/交易信息-详情页.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-30
+---------
+@summary: 工采通电子招投标交易平台
+---------
+@author: lzz
+"""
+import time
+
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+import execjs
+import json
+
+headers = {
+    "Accept": "application/json, text/plain, */*",
+    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "Cache-Control": "no-cache",
+    "Connection": "keep-alive",
+    "Pragma": "no-cache",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+}
+
+
+def ctx():
+    with open('./gctdzztbjypt.js','r') as fr:
+        ex_js = fr.read()
+    xx = execjs.compile(ex_js)
+    return xx
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=30)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 20)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout, proxies=False)
+
+    def download_midware(self, request):
+        pm = request.params
+        pm['t'] = int(time.time()*1000)
+        new_pm = ctx().call('sm4Encrypt',pm)
+        params = {
+            "encryptParams": new_pm
+        }
+        request.params = params
+        request.headers = headers
+
+    def detail_get(self, request, response):
+        items = request.item
+        list_item = DataBakItem(**items)
+        dt = json.loads(ctx().call('sm4Decrypt', response.text)).get('result')
+
+        html = dt.get('content')
+
+        list_item.contenthtml = html
+
+        attachments = {}
+        file_list = dt.get('annexes')
+        if file_list:
+            for info in file_list:
+                file_name = info.get('fileName','').strip()
+                file_url = "https://upy.easy-prt.com" + info.get('fileUrl','')
+                file_type = extract_file_type(file_name, file_url)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:gctdzztbjypt_jyxx").start()

a_gzsyyjzcgpt_tzgg_yyhc/通知公告-列表页.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 贵州省医药集中采购平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+
+    def start_callback(self):
+
+        self.site = "贵州省医药集中采购平台"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('通知公告-药品', 'a_gzsyyjzcgpt_tzgg_yp', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "https://gzyycg.ggzy.guizhou.gov.cn:8001/Home/Notice",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://gzyycg.ggzy.guizhou.gov.cn:8001/Home/GetList"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        params = {
+            "sysid": "",
+            "pagenum": f"{page}",
+            "typeid": "6BBB08AA-8E77-4E1B-8374-11151FAB926D"
+        }
+        request.params = params
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json
+        for info in info_list:
+            hid = info.get('DetailID')
+            href = f"https://gzyycg.ggzy.guizhou.gov.cn:8001/Home/Detail?id={hid}"
+            title = info.get('Title').strip()
+            create_time = info.get('AddTime').replace('T',' ').split('.')[0]
+
+            area = "贵州"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.proxies = False
+            list_item.deal_detail = ['//div[@class="main"]']
+            list_item.parse_url = href
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:gzsyyjzcgpt_tzgg_yp").start()

a_gzsyyjzcgpt_tzgg_yyhc/通知公告-详情页.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 贵州省医药集中采购平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+from feapder.network.selector import Selector
+import execjs
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=10)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout, proxies=False,)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        ex_js = '''
+        function de_str(value) {
+            return unescape(value)
+        }
+        '''
+        ctx = execjs.compile(ex_js)
+
+        html = ctx.call('de_str',response.xpath('//input[@id="CONTENT"]/@value').extract_first(""))
+
+        list_item.contenthtml = html
+
+        attachments = {}
+        file_list = Selector(html).xpath('//a')
+        if file_list:
+            for info in file_list:
+                file_url = "https://gzyycg.ggzy.guizhou.gov.cn:8001" + info.xpath('./@href').extract_first("")
+                file_name = info.xpath('./@title').extract_first("").strip()
+                file_type = extract_file_type(file_name, file_url)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+                list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:gzsyyjzcgpt_tzgg_yp").start()

a_hnrljywz_ggl/公告-详情页.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from feapder.utils.log import log
+from feapder.utils.tools import get_current_date
+
+
+headers = {
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "priority": "u=0, i",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+}
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),
+                                  callback="detail_get", **request_params, proxies=False)
+
+    def download_midware(self, request):
+        request.headers = headers
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+        if list_item.spidercode == "a_hnrljywz_xsgg":
+            list_item.contenthtml = "详情请访问原网页！"
+
+            furl = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
+            fid = furl.split('guid%3d')[-1]
+            attachments = {}
+
+            file_url = f'https://fec.hpi.com.cn/attachfile/FileDownLoad?guid={fid}'
+            fn = response.xpath('//a')
+            file_name = list_item.title
+            for n in fn:
+                if fid in n.xpath('./@href').extract_first():
+                    file_name = n.xpath('./text()').extract_first().strip()
+                    break
+            file_type = 'pdf'
+
+            attachment = AttachmentDownloader().fetch_attachment(
+                file_name=file_name, file_type=file_type, download_url=file_url,headers=headers)
+            attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+               list_item.projectinfo = {"attachments": attachments}
+
+            yield list_item
+        else:
+            detail_xpath = ['//div[@class="contain"]/div[@class="contain"]',
+                            '//div[@class="contain"]/div[@class="contain_bj"]']
+            html = ''
+            for xpath in detail_xpath:
+                html = response.xpath(xpath).extract_first()  # 标书详细内容
+                if html is not None:
+                    break
+            if '...' in list_item.title:
+                title = "".join(response.xpath(
+                    '//div[@class="contain"]/div[@class="contain_bj"]/div[@class="zwym"]/h2/text()').extract())
+                if title:
+                    list_item.title = title.strip()
+            pub_time1 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][1]/tr[2]/td[2]/text()").extract()).strip()
+            if pub_time1 and '/' in pub_time1:
+                list_item.publishtime = pub_time1.strip().replace('/', '-')
+            pub_time2 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][3]/tr[6]/td[2]/text()").extract()).strip()
+            if pub_time2 and '/' in pub_time2:
+                list_item.publishtime = pub_time2.strip().replace('/', '-')
+            extra_html = '<a style="cursor:pointer;" onclick="javascript:window.history.go(-1);"> 返回</a>'
+            list_item.contenthtml = html.replace(extra_html, '')
+            files = response.xpath('//div[@class="contain"]//a[@href]')
+            if len(files) > 0:
+                attachments = {}
+                for info in files:
+                    file_url = info.xpath('./@href').extract_first()
+                    if info.xpath('./@title').extract_first():
+                        file_name = info.xpath('./@title').extract_first()
+                    else:
+                        file_name = info.xpath('./text()').extract_first()
+                    if file_name:
+                        file_type = file_name.split(".")[-1].lower()
+                        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls',
+                                      'xlsx', 'swp', 'rtf']
+                        if file_type in file_types and "AttachFile" in file_url:
+                            attachment = AttachmentDownloader().fetch_attachment(
+                                file_name=file_name, file_type=file_type, download_url=file_url)
+                            attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+            if len(list_item.publishtime) == 0:
+                list_item.publishtime = get_current_date()
+            yield list_item
+
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_ggl/公告栏-列表页.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import requests
+
+
+def get_cookeis(proxies=False):
+    headers = {
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "priority": "u=0, i",
+        "referer": "https://fec.hpi.com.cn/Home/Login",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+    }
+
+    url = "https://fec.hpi.com.cn/Home/Login"
+    res = requests.get(url, headers=headers, timeout=30, proxies=proxies)
+    cookies = res.cookies.get_dict()
+
+    url1 = "https://fec.hpi.com.cn/Topic/Index?currmenuId=100&topiccode=BULLETIN"
+    url2 = "https://fec.hpi.com.cn/Topic/Search?topiccode=BULLETIN"
+    requests.get(url1, headers=headers, cookies=cookies, timeout=30, proxies=proxies)
+    requests.get(url2, headers=headers, cookies=cookies, timeout=30, proxies=proxies)
+    return cookies
+
+
+class Hnrljywz(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'typeone', 'order', 'crawl_page'])
+        self.site = "华能燃料交易网站"
+
+        self.menus = [
+            Menu('公告栏', 'a_hnrljywz_ggl', 'Topic', '2', 1)
+            # Menu('销售结果', 'a_hnrljywz_xsjg', 'Evaluate/search', '3', 1)
+        ]
+        self.count = 0
+
+        self.headers = {
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "priority": "u=0, i",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+        }
+        self.cookies= get_cookeis()
+
+    def start_requests(self):
+        for menu in self.menus:
+            yield feapder.Request(item=menu._asdict(), headers=self.headers, proxies=False, page=1)
+
+    def download_midware(self, request):
+        menu = request.item
+        if menu["code"] == 'a_hnrljywz_zdzbjg':
+            url = "https://fec.hpi.com.cn/Evaluate/Index?currmenuId=100"
+        else:
+            url = f"https://fec.hpi.com.cn/{menu['typeone']}?page={request.page}"
+
+        request.url = url
+        request.headers = self.headers
+        request.cookies = self.cookies
+
+
+    def parse(self, request, response):
+        if self.count > 5:
+            return
+        if response.status_code == 500:
+            self.count += 1
+            self.cookies = get_cookeis()
+            yield request
+        else:
+            self.count = 0
+            menu = request.item
+            info_list = response.xpath('//table[@class="contain_table"]/tr')
+            order = menu.get('order')
+            for info in info_list[-20:]:
+                title = info.xpath('./td[1]/a/text()').extract_first().strip()
+                href = info.xpath('./td[1]/a/@href').extract_first()
+                zbgs = "".join(info.xpath(f'./td[{order}]/text()').extract()).strip()
+                create_time_org = "".join(info.xpath('./td[last()]/text()').extract())
+                if '/' in create_time_org:
+                    create_time = create_time_org.replace('/', '-').strip()
+                    if len(create_time) == 16:
+                        create_time = create_time + ":00"
+                elif '-' in create_time_org:
+                    create_time = create_time_org.strip()
+                    if len(create_time) == 16:
+                        create_time = create_time + ":00"
+                else:
+                    create_time = ""
+
+                area = "全国"  # 省份
+                city = ""  # 城市
+
+                list_item = MgpListItem()  # 存储数据的管道
+                list_item.href = href  # 标书链接
+                list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+                list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+                list_item.title = title  # 标题
+                list_item.publishtime = create_time  # 标书发布时间
+                list_item.site = self.site
+                list_item.area = area  # 城市默认:全国
+                list_item.city = city  # 城市 默认为空
+
+                list_item.unique_key = ("href", zbgs)
+                list_item.parse = "self.detail_get"
+                list_item.proxies = False
+                list_item.parse_url = href
+
+                yield list_item
+
+            # 无限翻页设置
+            request = self.infinite_pages(request, response)
+            yield request
+
+
+if __name__ == "__main__":
+    Hnrljywz(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_xsgg/公告-详情页.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from feapder.utils.log import log
+from feapder.utils.tools import get_current_date
+
+
+headers = {
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "priority": "u=0, i",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+}
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),
+                                  callback="detail_get", **request_params, proxies=False)
+
+    def download_midware(self, request):
+        request.headers = headers
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+        if list_item.spidercode == "a_hnrljywz_xsgg":
+            list_item.contenthtml = "详情请访问原网页！"
+
+            furl = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
+            fid = furl.split('guid%3d')[-1]
+            attachments = {}
+
+            file_url = f'https://fec.hpi.com.cn/attachfile/FileDownLoad?guid={fid}'
+            fn = response.xpath('//a')
+            file_name = list_item.title
+            for n in fn:
+                if fid in n.xpath('./@href').extract_first():
+                    file_name = n.xpath('./text()').extract_first().strip()
+                    break
+            file_type = 'pdf'
+
+            attachment = AttachmentDownloader().fetch_attachment(
+                file_name=file_name, file_type=file_type, download_url=file_url,headers=headers)
+            attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+               list_item.projectinfo = {"attachments": attachments}
+
+            yield list_item
+        else:
+            detail_xpath = ['//div[@class="contain"]/div[@class="contain"]',
+                            '//div[@class="contain"]/div[@class="contain_bj"]']
+            html = ''
+            for xpath in detail_xpath:
+                html = response.xpath(xpath).extract_first()  # 标书详细内容
+                if html is not None:
+                    break
+            if '...' in list_item.title:
+                title = "".join(response.xpath(
+                    '//div[@class="contain"]/div[@class="contain_bj"]/div[@class="zwym"]/h2/text()').extract())
+                if title:
+                    list_item.title = title.strip()
+            pub_time1 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][1]/tr[2]/td[2]/text()").extract()).strip()
+            if pub_time1 and '/' in pub_time1:
+                list_item.publishtime = pub_time1.strip().replace('/', '-')
+            pub_time2 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][3]/tr[6]/td[2]/text()").extract()).strip()
+            if pub_time2 and '/' in pub_time2:
+                list_item.publishtime = pub_time2.strip().replace('/', '-')
+            extra_html = '<a style="cursor:pointer;" onclick="javascript:window.history.go(-1);"> 返回</a>'
+            list_item.contenthtml = html.replace(extra_html, '')
+            files = response.xpath('//div[@class="contain"]//a[@href]')
+            if len(files) > 0:
+                attachments = {}
+                for info in files:
+                    file_url = info.xpath('./@href').extract_first()
+                    if info.xpath('./@title').extract_first():
+                        file_name = info.xpath('./@title').extract_first()
+                    else:
+                        file_name = info.xpath('./text()').extract_first()
+                    if file_name:
+                        file_type = file_name.split(".")[-1].lower()
+                        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls',
+                                      'xlsx', 'swp', 'rtf']
+                        if file_type in file_types and "AttachFile" in file_url:
+                            attachment = AttachmentDownloader().fetch_attachment(
+                                file_name=file_name, file_type=file_type, download_url=file_url)
+                            attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+            if len(list_item.publishtime) == 0:
+                list_item.publishtime = get_current_date()
+            yield list_item
+
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_xsgg/销售公告-列表页.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+class Feapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+        self.site = "华能燃料交易网站"
+
+        self.menus = [
+            Menu('销售公告', 'a_hnrljywz_xsgg', 1),
+        ]
+
+        self.headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://fec.hpi.com.cn/Home/Login"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1,  proxies=False)
+
+    def parse(self, request, response):
+
+        menu = request.item
+        info_list = response.xpath("//ul[@class='tab-bd_xg']/li[3]/ul[@class='tab-bd']/li/div[@class='jrgz sy_list']/table[@class='sy_table']/tr")
+        for info in info_list[1:]:
+            title = info.xpath('./td[1]/@title').extract_first().strip()
+            href = info.xpath('./td[1]/a/@href').extract_first()
+            create_time = "2021-"+ info.xpath('./td[7]/text()').extract_first().strip()
+
+            area = "北京"   # 省份
+            city = "北京市"  # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = create_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"
+
+            list_item.proxies = False
+            list_item.deal_detail = []
+            list_item.parse_url = href
+
+            yield list_item
+
+
+    def download_midware(self, request):
+        page = request.page
+        request.headers = self.headers
+
+
+
+if __name__ == "__main__":
+    Feapder(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_xsmcgxx/下水煤采购信息-列表页.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import requests
+
+
+def get_cookeis(proxies=False):
+    headers = {
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "priority": "u=0, i",
+        "referer": "https://fec.hpi.com.cn/Home/Login",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+    }
+
+    url = "https://fec.hpi.com.cn/Home/Login"
+    res = requests.get(url, headers=headers, timeout=30, proxies=proxies)
+    cookies = res.cookies.get_dict()
+
+    url1 = "https://fec.hpi.com.cn/Topic/Index?currmenuId=100&topiccode=BULLETIN"
+    url2 = "https://fec.hpi.com.cn/Topic/Search?topiccode=BULLETIN"
+    requests.get(url1, headers=headers, cookies=cookies, timeout=30, proxies=proxies)
+    requests.get(url2, headers=headers, cookies=cookies, timeout=30, proxies=proxies)
+    return cookies
+
+
+class Hnrljywz(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'typeone', 'order', 'crawl_page'])
+        self.site = "华能燃料交易网站"
+
+        self.menus = [
+            Menu('下水煤采购信息', 'a_hnrljywz_xsmcgxx', 'JTOrder/search', '3', 1),
+        ]
+        self.count = 0
+
+        self.headers = {
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "priority": "u=0, i",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+        }
+        self.cookies= get_cookeis()
+
+    def start_requests(self):
+        for menu in self.menus:
+            yield feapder.Request(item=menu._asdict(), headers=self.headers, proxies=False, page=1)
+
+    def download_midware(self, request):
+        menu = request.item
+        if menu["code"] == 'a_hnrljywz_zdzbjg':
+            url = "https://fec.hpi.com.cn/Evaluate/Index?currmenuId=100"
+        else:
+            url = f"https://fec.hpi.com.cn/{menu['typeone']}?page={request.page}"
+
+        request.url = url
+        request.headers = self.headers
+        request.cookies = self.cookies
+
+
+    def parse(self, request, response):
+        if self.count > 5:
+            return
+        if response.status_code == 500:
+            self.count += 1
+            self.cookies = get_cookeis()
+            yield request
+        else:
+            self.count = 0
+            menu = request.item
+            info_list = response.xpath('//table[@class="contain_table"]/tr')
+            order = menu.get('order')
+            for info in info_list[-20:]:
+                title = info.xpath('./td[1]/a/text()').extract_first().strip()
+                href = info.xpath('./td[1]/a/@href').extract_first()
+                zbgs = "".join(info.xpath(f'./td[{order}]/text()').extract()).strip()
+                create_time_org = "".join(info.xpath('./td[last()]/text()').extract())
+                if '/' in create_time_org:
+                    create_time = create_time_org.replace('/', '-').strip()
+                    if len(create_time) == 16:
+                        create_time = create_time + ":00"
+                elif '-' in create_time_org:
+                    create_time = create_time_org.strip()
+                    if len(create_time) == 16:
+                        create_time = create_time + ":00"
+                else:
+                    create_time = ""
+
+                area = "全国"  # 省份
+                city = ""  # 城市
+
+                list_item = MgpListItem()  # 存储数据的管道
+                list_item.href = href  # 标书链接
+                list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+                list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+                list_item.title = title  # 标题
+                list_item.publishtime = create_time  # 标书发布时间
+                list_item.site = self.site
+                list_item.area = area  # 城市默认:全国
+                list_item.city = city  # 城市 默认为空
+
+                list_item.unique_key = ("href", zbgs)
+                list_item.parse = "self.detail_get"
+                list_item.proxies = False
+                list_item.parse_url = href
+
+                yield list_item
+
+            # 无限翻页设置
+            request = self.infinite_pages(request, response)
+            yield request
+
+
+if __name__ == "__main__":
+    Hnrljywz(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_xsmcgxx/公告-详情页.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from feapder.utils.log import log
+from feapder.utils.tools import get_current_date
+
+
+headers = {
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "priority": "u=0, i",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+}
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),
+                                  callback="detail_get", **request_params, proxies=False)
+
+    def download_midware(self, request):
+        request.headers = headers
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+        if list_item.spidercode == "a_hnrljywz_xsgg":
+            list_item.contenthtml = "详情请访问原网页！"
+
+            furl = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
+            fid = furl.split('guid%3d')[-1]
+            attachments = {}
+
+            file_url = f'https://fec.hpi.com.cn/attachfile/FileDownLoad?guid={fid}'
+            fn = response.xpath('//a')
+            file_name = list_item.title
+            for n in fn:
+                if fid in n.xpath('./@href').extract_first():
+                    file_name = n.xpath('./text()').extract_first().strip()
+                    break
+            file_type = 'pdf'
+
+            attachment = AttachmentDownloader().fetch_attachment(
+                file_name=file_name, file_type=file_type, download_url=file_url,headers=headers)
+            attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+               list_item.projectinfo = {"attachments": attachments}
+
+            yield list_item
+        else:
+            detail_xpath = ['//div[@class="contain"]/div[@class="contain"]',
+                            '//div[@class="contain"]/div[@class="contain_bj"]']
+            html = ''
+            for xpath in detail_xpath:
+                html = response.xpath(xpath).extract_first()  # 标书详细内容
+                if html is not None:
+                    break
+            if '...' in list_item.title:
+                title = "".join(response.xpath(
+                    '//div[@class="contain"]/div[@class="contain_bj"]/div[@class="zwym"]/h2/text()').extract())
+                if title:
+                    list_item.title = title.strip()
+            pub_time1 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][1]/tr[2]/td[2]/text()").extract()).strip()
+            if pub_time1 and '/' in pub_time1:
+                list_item.publishtime = pub_time1.strip().replace('/', '-')
+            pub_time2 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][3]/tr[6]/td[2]/text()").extract()).strip()
+            if pub_time2 and '/' in pub_time2:
+                list_item.publishtime = pub_time2.strip().replace('/', '-')
+            extra_html = '<a style="cursor:pointer;" onclick="javascript:window.history.go(-1);"> 返回</a>'
+            list_item.contenthtml = html.replace(extra_html, '')
+            files = response.xpath('//div[@class="contain"]//a[@href]')
+            if len(files) > 0:
+                attachments = {}
+                for info in files:
+                    file_url = info.xpath('./@href').extract_first()
+                    if info.xpath('./@title').extract_first():
+                        file_name = info.xpath('./@title').extract_first()
+                    else:
+                        file_name = info.xpath('./text()').extract_first()
+                    if file_name:
+                        file_type = file_name.split(".")[-1].lower()
+                        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls',
+                                      'xlsx', 'swp', 'rtf']
+                        if file_type in file_types and "AttachFile" in file_url:
+                            attachment = AttachmentDownloader().fetch_attachment(
+                                file_name=file_name, file_type=file_type, download_url=file_url)
+                            attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+            if len(list_item.publishtime) == 0:
+                list_item.publishtime = get_current_date()
+            yield list_item
+
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_xsmzbjg/下水煤中标结果-列表页.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import requests
+
+
+def get_cookeis(proxies=False):
+    headers = {
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "priority": "u=0, i",
+        "referer": "https://fec.hpi.com.cn/Home/Login",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+    }
+
+    url = "https://fec.hpi.com.cn/Home/Login"
+    res = requests.get(url, headers=headers, timeout=30, proxies=proxies)
+    cookies = res.cookies.get_dict()
+
+    url1 = "https://fec.hpi.com.cn/Topic/Index?currmenuId=100&topiccode=BULLETIN"
+    url2 = "https://fec.hpi.com.cn/Topic/Search?topiccode=BULLETIN"
+    requests.get(url1, headers=headers, cookies=cookies, timeout=30, proxies=proxies)
+    requests.get(url2, headers=headers, cookies=cookies, timeout=30, proxies=proxies)
+    return cookies
+
+
+class Hnrljywz(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'typeone', 'order', 'crawl_page'])
+        self.site = "华能燃料交易网站"
+
+        self.menus = [
+            Menu('下水煤中标结果', 'a_hnrljywz_xsmzbjg', 'JTEvaluate/search', '2', 1),
+        ]
+        self.count = 0
+
+        self.headers = {
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "priority": "u=0, i",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+        }
+        self.cookies= get_cookeis()
+
+    def start_requests(self):
+        for menu in self.menus:
+            yield feapder.Request(item=menu._asdict(), headers=self.headers, proxies=False, page=1)
+
+    def download_midware(self, request):
+        menu = request.item
+        if menu["code"] == 'a_hnrljywz_zdzbjg':
+            url = "https://fec.hpi.com.cn/Evaluate/Index?currmenuId=100"
+        else:
+            url = f"https://fec.hpi.com.cn/{menu['typeone']}?page={request.page}"
+
+        request.url = url
+        request.headers = self.headers
+        request.cookies = self.cookies
+
+
+    def parse(self, request, response):
+        if self.count > 5:
+            return
+        if response.status_code == 500:
+            self.count += 1
+            self.cookies = get_cookeis()
+            yield request
+        else:
+            self.count = 0
+            menu = request.item
+            info_list = response.xpath('//table[@class="contain_table"]/tr')
+            order = menu.get('order')
+            for info in info_list[-20:]:
+                title = info.xpath('./td[1]/a/text()').extract_first().strip()
+                href = info.xpath('./td[1]/a/@href').extract_first()
+                zbgs = "".join(info.xpath(f'./td[{order}]/text()').extract()).strip()
+                create_time_org = "".join(info.xpath('./td[last()]/text()').extract())
+                if '/' in create_time_org:
+                    create_time = create_time_org.replace('/', '-').strip()
+                    if len(create_time) == 16:
+                        create_time = create_time + ":00"
+                elif '-' in create_time_org:
+                    create_time = create_time_org.strip()
+                    if len(create_time) == 16:
+                        create_time = create_time + ":00"
+                else:
+                    create_time = ""
+
+                area = "全国"  # 省份
+                city = ""  # 城市
+
+                list_item = MgpListItem()  # 存储数据的管道
+                list_item.href = href  # 标书链接
+                list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+                list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+                list_item.title = title  # 标题
+                list_item.publishtime = create_time  # 标书发布时间
+                list_item.site = self.site
+                list_item.area = area  # 城市默认:全国
+                list_item.city = city  # 城市 默认为空
+
+                list_item.unique_key = ("href", zbgs)
+                list_item.parse = "self.detail_get"
+                list_item.proxies = False
+                list_item.parse_url = href
+
+                yield list_item
+
+            # 无限翻页设置
+            request = self.infinite_pages(request, response)
+            yield request
+
+
+if __name__ == "__main__":
+    Hnrljywz(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_xsmzbjg/公告-详情页.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from feapder.utils.log import log
+from feapder.utils.tools import get_current_date
+
+
+headers = {
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "priority": "u=0, i",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+}
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),
+                                  callback="detail_get", **request_params, proxies=False)
+
+    def download_midware(self, request):
+        request.headers = headers
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+        if list_item.spidercode == "a_hnrljywz_xsgg":
+            list_item.contenthtml = "详情请访问原网页！"
+
+            furl = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
+            fid = furl.split('guid%3d')[-1]
+            attachments = {}
+
+            file_url = f'https://fec.hpi.com.cn/attachfile/FileDownLoad?guid={fid}'
+            fn = response.xpath('//a')
+            file_name = list_item.title
+            for n in fn:
+                if fid in n.xpath('./@href').extract_first():
+                    file_name = n.xpath('./text()').extract_first().strip()
+                    break
+            file_type = 'pdf'
+
+            attachment = AttachmentDownloader().fetch_attachment(
+                file_name=file_name, file_type=file_type, download_url=file_url,headers=headers)
+            attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+               list_item.projectinfo = {"attachments": attachments}
+
+            yield list_item
+        else:
+            detail_xpath = ['//div[@class="contain"]/div[@class="contain"]',
+                            '//div[@class="contain"]/div[@class="contain_bj"]']
+            html = ''
+            for xpath in detail_xpath:
+                html = response.xpath(xpath).extract_first()  # 标书详细内容
+                if html is not None:
+                    break
+            if '...' in list_item.title:
+                title = "".join(response.xpath(
+                    '//div[@class="contain"]/div[@class="contain_bj"]/div[@class="zwym"]/h2/text()').extract())
+                if title:
+                    list_item.title = title.strip()
+            pub_time1 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][1]/tr[2]/td[2]/text()").extract()).strip()
+            if pub_time1 and '/' in pub_time1:
+                list_item.publishtime = pub_time1.strip().replace('/', '-')
+            pub_time2 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][3]/tr[6]/td[2]/text()").extract()).strip()
+            if pub_time2 and '/' in pub_time2:
+                list_item.publishtime = pub_time2.strip().replace('/', '-')
+            extra_html = '<a style="cursor:pointer;" onclick="javascript:window.history.go(-1);"> 返回</a>'
+            list_item.contenthtml = html.replace(extra_html, '')
+            files = response.xpath('//div[@class="contain"]//a[@href]')
+            if len(files) > 0:
+                attachments = {}
+                for info in files:
+                    file_url = info.xpath('./@href').extract_first()
+                    if info.xpath('./@title').extract_first():
+                        file_name = info.xpath('./@title').extract_first()
+                    else:
+                        file_name = info.xpath('./text()').extract_first()
+                    if file_name:
+                        file_type = file_name.split(".")[-1].lower()
+                        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls',
+                                      'xlsx', 'swp', 'rtf']
+                        if file_type in file_types and "AttachFile" in file_url:
+                            attachment = AttachmentDownloader().fetch_attachment(
+                                file_name=file_name, file_type=file_type, download_url=file_url)
+                            attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+            if len(list_item.publishtime) == 0:
+                list_item.publishtime = get_current_date()
+            yield list_item
+
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_zdcgxx/公告-详情页.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from feapder.utils.log import log
+from feapder.utils.tools import get_current_date
+
+
+headers = {
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "priority": "u=0, i",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+}
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),
+                                  callback="detail_get", **request_params, proxies=False)
+
+    def download_midware(self, request):
+        request.headers = headers
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+        if list_item.spidercode == "a_hnrljywz_xsgg":
+            list_item.contenthtml = "详情请访问原网页！"
+
+            furl = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
+            fid = furl.split('guid%3d')[-1]
+            attachments = {}
+
+            file_url = f'https://fec.hpi.com.cn/attachfile/FileDownLoad?guid={fid}'
+            fn = response.xpath('//a')
+            file_name = list_item.title
+            for n in fn:
+                if fid in n.xpath('./@href').extract_first():
+                    file_name = n.xpath('./text()').extract_first().strip()
+                    break
+            file_type = 'pdf'
+
+            attachment = AttachmentDownloader().fetch_attachment(
+                file_name=file_name, file_type=file_type, download_url=file_url,headers=headers)
+            attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+               list_item.projectinfo = {"attachments": attachments}
+
+            yield list_item
+        else:
+            detail_xpath = ['//div[@class="contain"]/div[@class="contain"]',
+                            '//div[@class="contain"]/div[@class="contain_bj"]']
+            html = ''
+            for xpath in detail_xpath:
+                html = response.xpath(xpath).extract_first()  # 标书详细内容
+                if html is not None:
+                    break
+            if '...' in list_item.title:
+                title = "".join(response.xpath(
+                    '//div[@class="contain"]/div[@class="contain_bj"]/div[@class="zwym"]/h2/text()').extract())
+                if title:
+                    list_item.title = title.strip()
+            pub_time1 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][1]/tr[2]/td[2]/text()").extract()).strip()
+            if pub_time1 and '/' in pub_time1:
+                list_item.publishtime = pub_time1.strip().replace('/', '-')
+            pub_time2 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][3]/tr[6]/td[2]/text()").extract()).strip()
+            if pub_time2 and '/' in pub_time2:
+                list_item.publishtime = pub_time2.strip().replace('/', '-')
+            extra_html = '<a style="cursor:pointer;" onclick="javascript:window.history.go(-1);"> 返回</a>'
+            list_item.contenthtml = html.replace(extra_html, '')
+            files = response.xpath('//div[@class="contain"]//a[@href]')
+            if len(files) > 0:
+                attachments = {}
+                for info in files:
+                    file_url = info.xpath('./@href').extract_first()
+                    if info.xpath('./@title').extract_first():
+                        file_name = info.xpath('./@title').extract_first()
+                    else:
+                        file_name = info.xpath('./text()').extract_first()
+                    if file_name:
+                        file_type = file_name.split(".")[-1].lower()
+                        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls',
+                                      'xlsx', 'swp', 'rtf']
+                        if file_type in file_types and "AttachFile" in file_url:
+                            attachment = AttachmentDownloader().fetch_attachment(
+                                file_name=file_name, file_type=file_type, download_url=file_url)
+                            attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+            if len(list_item.publishtime) == 0:
+                list_item.publishtime = get_current_date()
+            yield list_item
+
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_zdcgxx/直达采购信息-列表页.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import requests
+
+
+def get_cookeis(proxies=False):
+    headers = {
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "priority": "u=0, i",
+        "referer": "https://fec.hpi.com.cn/Home/Login",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+    }
+
+    url = "https://fec.hpi.com.cn/Home/Login"
+    res = requests.get(url, headers=headers, timeout=30, proxies=proxies)
+    cookies = res.cookies.get_dict()
+
+    url1 = "https://fec.hpi.com.cn/Topic/Index?currmenuId=100&topiccode=BULLETIN"
+    url2 = "https://fec.hpi.com.cn/Topic/Search?topiccode=BULLETIN"
+    requests.get(url1, headers=headers, cookies=cookies, timeout=30, proxies=proxies)
+    requests.get(url2, headers=headers, cookies=cookies, timeout=30, proxies=proxies)
+    return cookies
+
+
+class Hnrljywz(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'typeone', 'order', 'crawl_page'])
+        self.site = "华能燃料交易网站"
+
+        self.menus = [
+            Menu('直达采购信息', 'a_hnrljywz_zdcgxx', 'Order/MoreSearch', '3', 1),
+        ]
+        self.count = 0
+
+        self.headers = {
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "priority": "u=0, i",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+        }
+        self.cookies= get_cookeis()
+
+    def start_requests(self):
+        for menu in self.menus:
+            yield feapder.Request(item=menu._asdict(), headers=self.headers, proxies=False, page=1)
+
+    def download_midware(self, request):
+        menu = request.item
+        if menu["code"] == 'a_hnrljywz_zdzbjg':
+            url = "https://fec.hpi.com.cn/Evaluate/Index?currmenuId=100"
+        else:
+            url = f"https://fec.hpi.com.cn/{menu['typeone']}?page={request.page}"
+
+        request.url = url
+        request.headers = self.headers
+        request.cookies = self.cookies
+
+
+    def parse(self, request, response):
+        if self.count > 5:
+            return
+        if response.status_code == 500:
+            self.count += 1
+            self.cookies = get_cookeis()
+            yield request
+        else:
+            self.count = 0
+            menu = request.item
+            info_list = response.xpath('//table[@class="contain_table"]/tr')
+            order = menu.get('order')
+            for info in info_list[-20:]:
+                title = info.xpath('./td[1]/a/text()').extract_first().strip()
+                href = info.xpath('./td[1]/a/@href').extract_first()
+                zbgs = "".join(info.xpath(f'./td[{order}]/text()').extract()).strip()
+                create_time_org = "".join(info.xpath('./td[last()]/text()').extract())
+                if '/' in create_time_org:
+                    create_time = create_time_org.replace('/', '-').strip()
+                    if len(create_time) == 16:
+                        create_time = create_time + ":00"
+                elif '-' in create_time_org:
+                    create_time = create_time_org.strip()
+                    if len(create_time) == 16:
+                        create_time = create_time + ":00"
+                else:
+                    create_time = ""
+
+                area = "全国"  # 省份
+                city = ""  # 城市
+
+                list_item = MgpListItem()  # 存储数据的管道
+                list_item.href = href  # 标书链接
+                list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+                list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+                list_item.title = title  # 标题
+                list_item.publishtime = create_time  # 标书发布时间
+                list_item.site = self.site
+                list_item.area = area  # 城市默认:全国
+                list_item.city = city  # 城市 默认为空
+
+                list_item.unique_key = ("href", zbgs)
+                list_item.parse = "self.detail_get"
+                list_item.proxies = False
+                list_item.parse_url = href
+
+                yield list_item
+
+            # 无限翻页设置
+            request = self.infinite_pages(request, response)
+            yield request
+
+
+if __name__ == "__main__":
+    Hnrljywz(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_zdzbjg/公告-详情页.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from feapder.utils.log import log
+from feapder.utils.tools import get_current_date
+
+
+headers = {
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "priority": "u=0, i",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+}
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),
+                                  callback="detail_get", **request_params, proxies=False)
+
+    def download_midware(self, request):
+        request.headers = headers
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+        if list_item.spidercode == "a_hnrljywz_xsgg":
+            list_item.contenthtml = "详情请访问原网页！"
+
+            furl = response.xpath('//iframe[@id="pdfContainer"]/@src').extract_first()
+            fid = furl.split('guid%3d')[-1]
+            attachments = {}
+
+            file_url = f'https://fec.hpi.com.cn/attachfile/FileDownLoad?guid={fid}'
+            fn = response.xpath('//a')
+            file_name = list_item.title
+            for n in fn:
+                if fid in n.xpath('./@href').extract_first():
+                    file_name = n.xpath('./text()').extract_first().strip()
+                    break
+            file_type = 'pdf'
+
+            attachment = AttachmentDownloader().fetch_attachment(
+                file_name=file_name, file_type=file_type, download_url=file_url,headers=headers)
+            attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+               list_item.projectinfo = {"attachments": attachments}
+
+            yield list_item
+        else:
+            detail_xpath = ['//div[@class="contain"]/div[@class="contain"]',
+                            '//div[@class="contain"]/div[@class="contain_bj"]']
+            html = ''
+            for xpath in detail_xpath:
+                html = response.xpath(xpath).extract_first()  # 标书详细内容
+                if html is not None:
+                    break
+            if '...' in list_item.title:
+                title = "".join(response.xpath(
+                    '//div[@class="contain"]/div[@class="contain_bj"]/div[@class="zwym"]/h2/text()').extract())
+                if title:
+                    list_item.title = title.strip()
+            pub_time1 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][1]/tr[2]/td[2]/text()").extract()).strip()
+            if pub_time1 and '/' in pub_time1:
+                list_item.publishtime = pub_time1.strip().replace('/', '-')
+            pub_time2 = "".join(response.xpath(
+                "//div[@class='contain']/div[@class='contain']/table[@class='contain_form contain_form_bd'][3]/tr[6]/td[2]/text()").extract()).strip()
+            if pub_time2 and '/' in pub_time2:
+                list_item.publishtime = pub_time2.strip().replace('/', '-')
+            extra_html = '<a style="cursor:pointer;" onclick="javascript:window.history.go(-1);"> 返回</a>'
+            list_item.contenthtml = html.replace(extra_html, '')
+            files = response.xpath('//div[@class="contain"]//a[@href]')
+            if len(files) > 0:
+                attachments = {}
+                for info in files:
+                    file_url = info.xpath('./@href').extract_first()
+                    if info.xpath('./@title').extract_first():
+                        file_name = info.xpath('./@title').extract_first()
+                    else:
+                        file_name = info.xpath('./text()').extract_first()
+                    if file_name:
+                        file_type = file_name.split(".")[-1].lower()
+                        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls',
+                                      'xlsx', 'swp', 'rtf']
+                        if file_type in file_types and "AttachFile" in file_url:
+                            attachment = AttachmentDownloader().fetch_attachment(
+                                file_name=file_name, file_type=file_type, download_url=file_url)
+                            attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+            if len(list_item.publishtime) == 0:
+                list_item.publishtime = get_current_date()
+            yield list_item
+
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:hnrljywz_xsgg").start()

a_hnrljywz_zdzbjg/直达中标结果-列表页.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 华能燃料交易网站
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import requests
+
+
+def get_cookeis(proxies=False):
+    headers = {
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "priority": "u=0, i",
+        "referer": "https://fec.hpi.com.cn/Home/Login",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+    }
+
+    url = "https://fec.hpi.com.cn/Home/Login"
+    res = requests.get(url, headers=headers, timeout=30, proxies=proxies)
+    cookies = res.cookies.get_dict()
+
+    url1 = "https://fec.hpi.com.cn/Topic/Index?currmenuId=100&topiccode=BULLETIN"
+    url2 = "https://fec.hpi.com.cn/Topic/Search?topiccode=BULLETIN"
+    requests.get(url1, headers=headers, cookies=cookies, timeout=30, proxies=proxies)
+    requests.get(url2, headers=headers, cookies=cookies, timeout=30, proxies=proxies)
+    return cookies
+
+
+class Hnrljywz(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'typeone', 'order', 'crawl_page'])
+        self.site = "华能燃料交易网站"
+
+        self.menus = [
+            Menu('直达中标结果', 'a_hnrljywz_zdzbjg', '', '3', 1),
+        ]
+        self.count = 0
+
+        self.headers = {
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "priority": "u=0, i",
+            "upgrade-insecure-requests": "1",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
+        }
+        self.cookies= get_cookeis()
+
+    def start_requests(self):
+        for menu in self.menus:
+            yield feapder.Request(item=menu._asdict(), headers=self.headers, proxies=False, page=1)
+
+    def download_midware(self, request):
+        menu = request.item
+        if menu["code"] == 'a_hnrljywz_zdzbjg':
+            url = "https://fec.hpi.com.cn/Evaluate/Index?currmenuId=100"
+        else:
+            url = f"https://fec.hpi.com.cn/{menu['typeone']}?page={request.page}"
+
+        request.url = url
+        request.headers = self.headers
+        request.cookies = self.cookies
+
+
+    def parse(self, request, response):
+        if self.count > 5:
+            return
+        if response.status_code == 500:
+            self.count += 1
+            self.cookies = get_cookeis()
+            yield request
+        else:
+            self.count = 0
+            menu = request.item
+            info_list = response.xpath('//table[@class="contain_table"]/tr')
+            order = menu.get('order')
+            for info in info_list[-20:]:
+                title = info.xpath('./td[1]/a/text()').extract_first().strip()
+                href = info.xpath('./td[1]/a/@href').extract_first()
+                zbgs = "".join(info.xpath(f'./td[{order}]/text()').extract()).strip()
+                create_time_org = "".join(info.xpath('./td[last()]/text()').extract())
+                if '/' in create_time_org:
+                    create_time = create_time_org.replace('/', '-').strip()
+                    if len(create_time) == 16:
+                        create_time = create_time + ":00"
+                elif '-' in create_time_org:
+                    create_time = create_time_org.strip()
+                    if len(create_time) == 16:
+                        create_time = create_time + ":00"
+                else:
+                    create_time = ""
+
+                area = "全国"  # 省份
+                city = ""  # 城市
+
+                list_item = MgpListItem()  # 存储数据的管道
+                list_item.href = href  # 标书链接
+                list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+                list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+                list_item.title = title  # 标题
+                list_item.publishtime = create_time  # 标书发布时间
+                list_item.site = self.site
+                list_item.area = area  # 城市默认:全国
+                list_item.city = city  # 城市 默认为空
+
+                list_item.unique_key = ("href", zbgs)
+                list_item.parse = "self.detail_get"
+                list_item.proxies = False
+                list_item.parse_url = href
+
+                yield list_item
+
+            # 无限翻页设置
+            request = self.infinite_pages(request, response)
+            yield request
+
+
+if __name__ == "__main__":
+    Hnrljywz(redis_key="lzz:hnrljywz_xsgg").start()

a_jstdscw_cjgg/交易大厅-详情页.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 江苏土地市场网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.tools import extract_file_type
+from untils.attachment import AttachmentDownloader
+import re
+
+
+def fileDown(hid):
+    return hid
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=10)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+            yield feapder.Request(url=item.get("parse_url"), item=item, proxies=False,render_time=5,render=True,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html1 = response.xpath('//div[@id="resultInfo"]').extract_first("")
+        html2 = response.xpath('//div[@id="bargainInfo"]').extract_first("")
+        html3 = response.xpath('//div[@id="formInfo"]').extract_first("")
+        html4 = response.xpath('//div[@id="afficheInfo"]').extract_first("")
+
+        s_title = "".join("".join(response.xpath('//div[@class="mainTwo-middle notice"]/ul[1]/li[1]//text()').extract()).split())
+        if s_title:
+            list_item.title = s_title
+            list_item.s_title = s_title
+
+        list_item.contenthtml = html1 + html2 + html3 + html4
+
+        attachments = {}
+        file_list = response.xpath('//div[@class="mainContent"]//a[contains(@href, "downLoadAttch")]')
+        if file_list:
+            for info in file_list:
+                fid = "".join(re.findall("javascript:downLoadAttch\('(.*?)'",info.xpath('./@href').extract_first("")))
+                file_name = info.xpath('./text()').extract_first("").strip()
+                file_url = f"http://www.landjs.com/tAfficheParcel/fileDownLoad/{fid}"
+                file_type = extract_file_type(file_name=file_name, file_url=file_url)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url, )
+                    attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:jstdscw_jydt").start()

a_jstdscw_cjgg/成交公告-列表页.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 江苏土地市场网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+import json
+
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "江苏土地市场网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('成交公告', 'a_jstdscw_cjgg', 1),
+        ]
+
+        self.headers = {
+            "Accept": "*/*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json",
+            "Origin": "http://www.landjs.com",
+            "Pragma": "no-cache",
+            "Referer": "http://www.landjs.com/tAfficheParcel/bargainParcel",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "http://www.landjs.com/tAfficheParcel/searchBargainParcel"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        data = {
+            "index": page,
+            "size": 10,
+            "mrFlag": 2
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('bargainParcelList')
+        for info in info_list:
+            href = "http://www.landjs.com/tAfficheParcel/detail/bargain/" + info.get('cjgsGuid')
+            title = info.get('xmMc')
+            create_time = timestamp_to_date(int(str(info.get('createDate'))[:10]))
+
+            area = "江苏"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="mainContent"]']
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:jstdscw_jydt").start()

a_jstdscw_crgg/出让公告-列表页.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 江苏土地市场网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+import json
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "江苏土地市场网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('出让公告', 'a_jstdscw_crgg', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json",
+            "Origin": "http://www.landjs.com",
+            "Pragma": "no-cache",
+            "Referer": "http://www.landjs.com/affiche/index/5",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "http://www.landjs.com/affiche/information"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        data = {
+            "index": page,
+            "size": 10,
+            "mrFlag": 2
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('list')
+        for info in info_list:
+            href = "http://www.landjs.com/affiche/" + info.get('gyggGuid')
+            title = info.get('afficheName')
+            create_time = timestamp_to_date(int(str(info.get('createDate'))[:10]))
+
+            area = "江苏"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="mainContent"]']
+            list_item.parse_url = href  # 详情页请求地址
+
+            list_item.files = {  # 附件采集规则
+                "list_xpath": '//div[@class="mainContent"]//a[@href]',
+                "url_xpath": './@href',
+                "name_xpath": './/text()',
+                "file_type":'pdf',                  # 默认的附件类型，用于url中未带附件类型的
+                "url_key": 'download',  # 用于区别连接是否为正常附件连接的url关键词 必须携带，如无可填http
+                "host": '',  # 需要拼接url的host
+            }
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="detail:chrome").start()

a_jstdscw_gyjsydgyjh/国有建设用地供应计划-列表页.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 江苏土地市场网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+import json
+
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "江苏土地市场网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('国有建设用地供应计划', 'a_jstdscw_gyjsydgyjh', 1),
+        ]
+
+        self.headers = {
+            "Accept": "*/*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json",
+            "Origin": "http://www.landjs.com",
+            "Pragma": "no-cache",
+            "Referer": "http://www.landjs.com/gyjh/index",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "http://www.landjs.com/gyjh/information"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        data = {
+            "index": page,
+            "size": 10,
+            "nf": "全部",
+            "mrFlag": 2
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('list')
+        for info in info_list:
+            href = "http://www.landjs.com/gyjh/" + info.get('gyjhGuid')
+            title = info.get('biaoti')
+            create_time = timestamp_to_date(int(str(info.get('fbSj'))[:10]))
+
+            area = "江苏"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="mainContent"]']
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="detail:chrome").start()

a_jstdscw_jggg/交易大厅-详情页.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 江苏土地市场网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.tools import extract_file_type
+from untils.attachment import AttachmentDownloader
+import re
+
+
+def fileDown(hid):
+    return hid
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=10)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+            yield feapder.Request(url=item.get("parse_url"), item=item, proxies=False,render_time=5,render=True,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html1 = response.xpath('//div[@id="resultInfo"]').extract_first("")
+        html2 = response.xpath('//div[@id="bargainInfo"]').extract_first("")
+        html3 = response.xpath('//div[@id="formInfo"]').extract_first("")
+        html4 = response.xpath('//div[@id="afficheInfo"]').extract_first("")
+
+        s_title = "".join("".join(response.xpath('//div[@class="mainTwo-middle notice"]/ul[1]/li[1]//text()').extract()).split())
+        if s_title:
+            list_item.title = s_title
+            list_item.s_title = s_title
+
+        list_item.contenthtml = html1 + html2 + html3 + html4
+
+        attachments = {}
+        file_list = response.xpath('//div[@class="mainContent"]//a[contains(@href, "downLoadAttch")]')
+        if file_list:
+            for info in file_list:
+                fid = "".join(re.findall("javascript:downLoadAttch\('(.*?)'",info.xpath('./@href').extract_first("")))
+                file_name = info.xpath('./text()').extract_first("").strip()
+                file_url = f"http://www.landjs.com/tAfficheParcel/fileDownLoad/{fid}"
+                file_type = extract_file_type(file_name=file_name, file_url=file_url)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url, )
+                    attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:jstdscw_jydt").start()

a_jstdscw_jggg/结果公告-列表页.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 江苏土地市场网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+import json
+
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "江苏土地市场网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('结果公告', 'a_jstdscw_jggg', 2),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/Json",
+            "Origin": "http://www.landjs.com",
+            "Pragma": "no-cache",
+            "Referer": "http://www.landjs.com/tAfficheParcel/bulletin",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "http://www.landjs.com/tAfficheParcel/bulletinInfo"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        data = {
+            "index": page,
+            "size": 10,
+            "mrFlag": 2
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('list')
+        for info in info_list:
+            href = "http://www.landjs.com/tAfficheParcel/detail/gdxm/" + info.get('gdGuid')
+            title = info.get('xmMc')
+            create_time = timestamp_to_date(int(str(info.get('createDate'))[:10]))
+
+            area = "江苏"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="mainContent"]']
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:jstdscw_jydt").start()

a_jstdscw_ncrdkqd/拟出让地块清单-列表页.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 江苏土地市场网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+import json
+
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "江苏土地市场网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('拟出让地块清单', 'a_jstdscw_ncrdkqd', 1),
+        ]
+
+        self.headers = {
+            "Accept": "*/*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json",
+            "Origin": "http://www.landjs.com",
+            "Pragma": "no-cache",
+            "Referer": "http://www.landjs.com/gyjh/ncrydIndex",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "http://www.landjs.com/gyjh/getNcrzdlist"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        data = {
+            "index": page,
+            "size": 10,
+            "nf": "全部"
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('list')
+        for info in info_list:
+            href = "http://www.landjs.com/gyjh/toDetail/" + info.get('ncrzdGuid')
+            title = info.get('bt')
+            create_time = timestamp_to_date(int(str(info.get('fbSj'))[:10]))
+
+            area = "江苏"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="main-middle"]']
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="detail:chrome").start()

a_sthjgsw_fbgs_njpc/发布公示-列表页.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-30
+---------
+@summary: 生态环境公示网
+---------
+@author: lzz
+"""
+import feapder
+from items.njpc_item import NjpcListItem
+from collections import namedtuple
+
+
+
+class Sthjgsw_Fbgs(feapder.PlanToBuildListSpider):
+
+    def start_callback(self):
+
+        self.site = "生态环境公示网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('发布公示', 'a_sthjgsw_fbgs_njpc',  1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Authorization": "Bearer",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Origin": "https://gongshi.qsyhbgj.com",
+            "Pragma": "no-cache",
+            "Referer": "https://gongshi.qsyhbgj.com/",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36",
+            "appId": "p_web",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://api.qsyhbgj.com/publicity/publicityList"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=10)
+
+    def parse(self, request, response):
+
+        menu = request.item
+        info_list = response.json.get('data').get('list')
+        for info in info_list:
+            projectname = info.get('name').strip()
+            Id = info.get('id')
+            regionOne = info.get('regionOne').strip()
+            detail_href = f"https://gongshi.qsyhbgj.com/h5public-detail?id={Id}"
+            publish_time = info.get('creationTime').strip()
+
+            area = f"{regionOne}"  # 省份
+            city = ""
+            district = ""
+
+            data_item = NjpcListItem()  # 存储数据的管道
+            data_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            data_item.unique_key = ('href', publish_time)
+            data_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            data_item.projectname = projectname      # 项目名称
+            data_item.publishtime = publish_time     # 发布时间
+
+            data_item.site = self.site
+            data_item.area = area or "全国"  # 城市默认:全国
+            data_item.city = city  # 城市 默认为空
+            data_item.district = district  # 城市 默认为空
+            data_item.parser_url = f"https://api.qsyhbgj.com/publicity/publicity?id={Id}"  # 详情页数据链接
+            data_item.href = detail_href  # 详情链接
+            data_item.request_params = {"headers":self.headers}
+            yield data_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+    def download_midware(self, request):
+        page = request.page
+        params = {
+            "pageNo": f"{page}",
+            "pages": "20"
+        }
+        request.params = params
+        request.headers = self.headers
+
+
+if __name__ == "__main__":
+    Sthjgsw_Fbgs(redis_key="lzz:Sthjgsw_Fbgs").start()

a_sthjgsw_fbgs_njpc/发布公示-详情页.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-30
+---------
+@summary: 生态环境公示网
+---------
+@author: lzz
+"""
+import feapder
+from items.njpc_item import DataNjpcItem
+from untils.attachment import AttachmentDownloader
+from feapder.utils.log import log
+from lxml.html import fromstring
+import re
+
+
+
+class Details(feapder.PlanToBuildDetailSpider):
+
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=50)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get('request_params')
+            yield feapder.Request(url=item.get("parser_url"),item=item,**request_params)
+
+
+    def parse(self,request,response):
+
+        items = request.item
+        data_item = DataNjpcItem(**items)
+
+        html = response.json.get('data').get('content').replace('<o:p>','').replace('</o:p>','')
+        data_item.title = data_item.projectname
+
+        projectname = "".join(re.findall('项目名称(?:<[^>]+>|)[：|:](?:<[^>]+>|)(.*?)<',html,re.S))
+        projectaddr = "".join(re.findall('建设(?:地点|地址)(?:<[^>]+>|)[：|:](?:<[^>]+>|)(.*?)<',html,re.S))
+        owner = "".join(re.findall('建设(?:单位|单位名称)(?:<[^>]+>|)[:|：](?:<[^>]+>|)(.*?)<',html,re.S))
+        project_person = "".join(re.findall('联系人(?:<[^>]+>|)[：|:](?:<[^>]+>|)(.*?)<',html,re.S))
+        if project_person:
+            phone = "".join(re.findall('\d{8,12}',project_person))
+            if phone:
+                data_item.project_person = project_person.replace(phone,'').replace('&nbsp;','').strip()
+                data_item.project_phone = phone
+            else:
+                data_item.project_person = project_person
+        if projectname:
+            data_item.projectname = projectname
+        if projectaddr:
+            data_item.projectaddr = projectaddr
+        if owner:
+            data_item.owner = owner
+
+        root = fromstring(html)
+        file_info = root.xpath('//a[@href]')
+        if file_info:
+            attachments = {}
+            for info in file_info:
+                file_url = "".join(info.xpath('./@href'))
+                file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg',
+                              'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg', 'wps']
+                file_name = "".join("".join(info.xpath('.//text()')).split()).strip()
+                file_type = file_name.split(".")[-1].lower()
+                if file_type in file_types:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+                data_item.projectinfo = {"attachments": attachments}
+
+        data_item.contenthtml = html
+
+        yield data_item
+
+
+
+if __name__ == '__main__':
+    Details(redis_key="lzz:Sthjgsw_Fbgs").start()
+

a_sylhcqjys_cgqzr/sylhcqjys_details_firefox.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+import time
+from feapder.utils.log import log
+from lxml.html import fromstring,tostring
+from untils.attachment import AttachmentDownloader
+import re
+import requests
+import json
+
+
+
+
+class FirefoxDetails(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),item=item,files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),**request_params,
+                                  callback=eval(item.get("parse")),render=True,
+                                  render_time=item.get("render_time"),proxies=False)
+
+    def detail_get(self,request,response):
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        driver = response.browser
+        driver.switch_to.frame('iframe')
+        driver.switch_to.frame('project-page')
+        pub_time = re.findall('\d{4}-\d{1,2}-\d{1,2}',driver.page_source,re.S)
+        if pub_time:
+            list_item.publishtime = pub_time[0]
+        root = fromstring(driver.page_source)
+        iframe_list = root.xpath('//iframe[@class="idt-subForm-iframe innerFrame"]/@id')
+        html_str = root.xpath('//div[@id="formApp"]/div[@id="formScrollWrap"]')
+        if html_str != None and len(html_str) > 0:
+            html_str = tostring(html_str[0],encoding='utf-8').decode()
+        else:
+            html_str = ''
+        for iframe in iframe_list:
+            driver.switch_to.frame(f'{iframe}')
+            res = fromstring(driver.page_source)
+            html = res.xpath(request.deal_detail[0])  # 标书详细内容
+            html = tostring(html[0],encoding='utf-8').decode()
+            html_str += html
+            if res.xpath('//iframe/@id'):
+                for four_iframe in res.xpath('//iframe/@id'):
+                    driver.switch_to.frame(f'{four_iframe}')
+                    driver.switch_to.frame('project-page')
+                    f_res = fromstring(driver.page_source)
+                    html_f = f_res.xpath('//div[@id="formApp"]/div[@class="formScrollBar"]')  # 标书详细内容
+                    html_f = tostring(html_f[0], encoding='utf-8').decode()
+                    html_str += html_f
+                    driver.switch_to.parent_frame()
+                    driver.switch_to.parent_frame()
+            driver.switch_to.parent_frame()
+
+
+        list_item.contenthtml = html_str
+
+        driver.switch_to.frame(f'{iframe_list[-1]}')
+        time.sleep(2)
+        fujian_res = driver.page_source
+        if re.findall('"tableId":"(.*?)"',fujian_res,re.S):
+            fujian_tableId = re.findall('"tableId":"(.*?)"',fujian_res,re.S)[-1].strip()
+            id = "".join(re.findall('id=(.*?)&',list_item.href))
+            f_name = "".join(re.findall('data-filedname="(.*?)"',fujian_res,re.S))
+            data = {
+                "tableId": f"{fujian_tableId}",
+                "uuid": f"{id}",
+                "fieldName": f"{f_name}",
+                "developer": "sjsxm",
+            }
+            url = 'https://www.sprtc.com/idtAppServiceV6/exclude/control/getFiles'
+            f_res = requests.post(url, data=data)
+            file_dict = json.loads(f_res.content.decode(),strict=False).get('data')
+            if file_dict:
+                attachments = {}
+                for info in file_dict:
+                    uuid = info.get('UUID')
+                    file_url = f"https://www.sprtc.com/idtAppServiceV6/oApp/downloadLargeFile?tableId={fujian_tableId}&uuid={uuid}&developer=sjsxm&authToken=null"
+                    file_name = info.get('FILENAME').strip()
+                    file_type = file_name.split(".")[-1].lower()
+                    file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls', 'xlsx', 'swp']
+                    if file_type in file_types:
+                        if file_type in file_name:
+                            file_name = file_name.replace(f'.{file_type}', '')
+                        attachment = AttachmentDownloader().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url)
+                        attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    FirefoxDetails(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_cgqzr/产股权转让-列表页.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+
+class Sylhcqjys(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code','typeone', 'typetwo','typethree', 'crawl_page'])
+        self.site = "沈阳联合产权交易所"
+
+        self.menus = [
+            Menu('产股权转让', 'a_sylhcqjys_cgqzr','cgqzfxmGetList', '', '2', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            "Origin": "https://test.sprtc.com",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+
+        for menu in self.menus:
+            start_url = 'https://test.sprtc.com/idtAppServiceV6/exclude/control/getGrid'
+            yield feapder.Request(url=start_url,item=menu._asdict(),page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "script": f"{menu.get('typeone')}",
+            "args": "[]",
+            "keys": f"[\"{menu.get('typetwo')}\"]",
+            "replaceMap": "{\"developer\":\"sjsxm\"}",
+            "pageSize": "20",
+            "nowPage": f"{page}"
+        }
+        request.data = data
+        request.headers=self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('DataList')
+        for info in info_list:
+            href_id = info.get('UUID').strip()
+            href = f"https://www.sprtc.com/index/qrtwo.htm?id={href_id}"
+            title = info.get('BDMC').strip()
+            create_time = info.get('GPJSSJ').strip()
+
+            area = "辽宁"  # 省份
+            city = "沈阳市"  # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = create_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"
+            list_item.deal_detail = ['//div[@id="formApp"]/div[@id="formScrollWrap"]']
+            list_item.proxies = False
+            list_item.parse_url = href
+            list_item.render_time = 5
+
+            yield list_item
+
+
+if __name__ == "__main__":
+    Sylhcqjys(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_jrzc/sylhcqjys_details_firefox.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+import time
+from feapder.utils.log import log
+from lxml.html import fromstring,tostring
+from untils.attachment import AttachmentDownloader
+import re
+import requests
+import json
+
+
+
+
+class FirefoxDetails(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),item=item,files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),**request_params,
+                                  callback=eval(item.get("parse")),render=True,
+                                  render_time=item.get("render_time"),proxies=False)
+
+    def detail_get(self,request,response):
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        driver = response.browser
+        driver.switch_to.frame('iframe')
+        driver.switch_to.frame('project-page')
+        pub_time = re.findall('\d{4}-\d{1,2}-\d{1,2}',driver.page_source,re.S)
+        if pub_time:
+            list_item.publishtime = pub_time[0]
+        root = fromstring(driver.page_source)
+        iframe_list = root.xpath('//iframe[@class="idt-subForm-iframe innerFrame"]/@id')
+        html_str = root.xpath('//div[@id="formApp"]/div[@id="formScrollWrap"]')
+        if html_str != None and len(html_str) > 0:
+            html_str = tostring(html_str[0],encoding='utf-8').decode()
+        else:
+            html_str = ''
+        for iframe in iframe_list:
+            driver.switch_to.frame(f'{iframe}')
+            res = fromstring(driver.page_source)
+            html = res.xpath(request.deal_detail[0])  # 标书详细内容
+            html = tostring(html[0],encoding='utf-8').decode()
+            html_str += html
+            if res.xpath('//iframe/@id'):
+                for four_iframe in res.xpath('//iframe/@id'):
+                    driver.switch_to.frame(f'{four_iframe}')
+                    driver.switch_to.frame('project-page')
+                    f_res = fromstring(driver.page_source)
+                    html_f = f_res.xpath('//div[@id="formApp"]/div[@class="formScrollBar"]')  # 标书详细内容
+                    html_f = tostring(html_f[0], encoding='utf-8').decode()
+                    html_str += html_f
+                    driver.switch_to.parent_frame()
+                    driver.switch_to.parent_frame()
+            driver.switch_to.parent_frame()
+
+
+        list_item.contenthtml = html_str
+
+        driver.switch_to.frame(f'{iframe_list[-1]}')
+        time.sleep(2)
+        fujian_res = driver.page_source
+        if re.findall('"tableId":"(.*?)"',fujian_res,re.S):
+            fujian_tableId = re.findall('"tableId":"(.*?)"',fujian_res,re.S)[-1].strip()
+            id = "".join(re.findall('id=(.*?)&',list_item.href))
+            f_name = "".join(re.findall('data-filedname="(.*?)"',fujian_res,re.S))
+            data = {
+                "tableId": f"{fujian_tableId}",
+                "uuid": f"{id}",
+                "fieldName": f"{f_name}",
+                "developer": "sjsxm",
+            }
+            url = 'https://www.sprtc.com/idtAppServiceV6/exclude/control/getFiles'
+            f_res = requests.post(url, data=data)
+            file_dict = json.loads(f_res.content.decode(),strict=False).get('data')
+            if file_dict:
+                attachments = {}
+                for info in file_dict:
+                    uuid = info.get('UUID')
+                    file_url = f"https://www.sprtc.com/idtAppServiceV6/oApp/downloadLargeFile?tableId={fujian_tableId}&uuid={uuid}&developer=sjsxm&authToken=null"
+                    file_name = info.get('FILENAME').strip()
+                    file_type = file_name.split(".")[-1].lower()
+                    file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls', 'xlsx', 'swp']
+                    if file_type in file_types:
+                        if file_type in file_name:
+                            file_name = file_name.replace(f'.{file_type}', '')
+                        attachment = AttachmentDownloader().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url)
+                        attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    FirefoxDetails(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_jrzc/金融资产-列表页.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+
+class Sylhcqjys(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code','typeone', 'typetwo','typethree', 'crawl_page'])
+        self.site = "沈阳联合产权交易所"
+
+        self.menus = [
+            Menu('金融资产', 'a_sylhcqjys_jrzc','jrzcGetList', '', '', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            "Origin": "https://test.sprtc.com",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+
+        for menu in self.menus:
+            start_url = 'https://test.sprtc.com/idtAppServiceV6/exclude/control/getGrid'
+            yield feapder.Request(url=start_url,item=menu._asdict(),page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "script": f"{menu.get('typeone')}",
+            "args": "[]",
+            "keys": f"[\"{menu.get('typetwo')}\"]",
+            "replaceMap": "{\"developer\":\"sjsxm\"}",
+            "pageSize": "20",
+            "nowPage": f"{page}"
+        }
+        request.data = data
+        request.headers=self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('DataList')
+        for info in info_list:
+            href_id = info.get('UUID').strip()
+            href = f"https://www.sprtc.com/index/qrtwo.htm?id={href_id}"
+            title = info.get('BDMC').strip()
+            create_time = info.get('GPJSSJ').strip()
+
+            area = "辽宁"  # 省份
+            city = "沈阳市"  # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = create_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"
+            list_item.deal_detail = ['//div[@id="formApp"]/div[@id="formScrollWrap"]']
+            list_item.proxies = False
+            list_item.parse_url = href
+            list_item.render_time = 5
+
+            yield list_item
+
+
+if __name__ == "__main__":
+    Sylhcqjys(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_jscq/sylhcqjys_details_firefox.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+import time
+from feapder.utils.log import log
+from lxml.html import fromstring,tostring
+from untils.attachment import AttachmentDownloader
+import re
+import requests
+import json
+
+
+
+
+class FirefoxDetails(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),item=item,files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),**request_params,
+                                  callback=eval(item.get("parse")),render=True,
+                                  render_time=item.get("render_time"),proxies=False)
+
+    def detail_get(self,request,response):
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        driver = response.browser
+        driver.switch_to.frame('iframe')
+        driver.switch_to.frame('project-page')
+        pub_time = re.findall('\d{4}-\d{1,2}-\d{1,2}',driver.page_source,re.S)
+        if pub_time:
+            list_item.publishtime = pub_time[0]
+        root = fromstring(driver.page_source)
+        iframe_list = root.xpath('//iframe[@class="idt-subForm-iframe innerFrame"]/@id')
+        html_str = root.xpath('//div[@id="formApp"]/div[@id="formScrollWrap"]')
+        if html_str != None and len(html_str) > 0:
+            html_str = tostring(html_str[0],encoding='utf-8').decode()
+        else:
+            html_str = ''
+        for iframe in iframe_list:
+            driver.switch_to.frame(f'{iframe}')
+            res = fromstring(driver.page_source)
+            html = res.xpath(request.deal_detail[0])  # 标书详细内容
+            html = tostring(html[0],encoding='utf-8').decode()
+            html_str += html
+            if res.xpath('//iframe/@id'):
+                for four_iframe in res.xpath('//iframe/@id'):
+                    driver.switch_to.frame(f'{four_iframe}')
+                    driver.switch_to.frame('project-page')
+                    f_res = fromstring(driver.page_source)
+                    html_f = f_res.xpath('//div[@id="formApp"]/div[@class="formScrollBar"]')  # 标书详细内容
+                    html_f = tostring(html_f[0], encoding='utf-8').decode()
+                    html_str += html_f
+                    driver.switch_to.parent_frame()
+                    driver.switch_to.parent_frame()
+            driver.switch_to.parent_frame()
+
+
+        list_item.contenthtml = html_str
+
+        driver.switch_to.frame(f'{iframe_list[-1]}')
+        time.sleep(2)
+        fujian_res = driver.page_source
+        if re.findall('"tableId":"(.*?)"',fujian_res,re.S):
+            fujian_tableId = re.findall('"tableId":"(.*?)"',fujian_res,re.S)[-1].strip()
+            id = "".join(re.findall('id=(.*?)&',list_item.href))
+            f_name = "".join(re.findall('data-filedname="(.*?)"',fujian_res,re.S))
+            data = {
+                "tableId": f"{fujian_tableId}",
+                "uuid": f"{id}",
+                "fieldName": f"{f_name}",
+                "developer": "sjsxm",
+            }
+            url = 'https://www.sprtc.com/idtAppServiceV6/exclude/control/getFiles'
+            f_res = requests.post(url, data=data)
+            file_dict = json.loads(f_res.content.decode(),strict=False).get('data')
+            if file_dict:
+                attachments = {}
+                for info in file_dict:
+                    uuid = info.get('UUID')
+                    file_url = f"https://www.sprtc.com/idtAppServiceV6/oApp/downloadLargeFile?tableId={fujian_tableId}&uuid={uuid}&developer=sjsxm&authToken=null"
+                    file_name = info.get('FILENAME').strip()
+                    file_type = file_name.split(".")[-1].lower()
+                    file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls', 'xlsx', 'swp']
+                    if file_type in file_types:
+                        if file_type in file_name:
+                            file_name = file_name.replace(f'.{file_type}', '')
+                        attachment = AttachmentDownloader().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url)
+                        attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    FirefoxDetails(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_jscq/技术产权-列表页.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+
+class Sylhcqjys(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code','typeone', 'typetwo','typethree', 'crawl_page'])
+        self.site = "沈阳联合产权交易所"
+
+        self.menus = [
+            Menu('技术产权', 'a_sylhcqjys_jscq','jslxmGetList', '', '', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            "Origin": "https://test.sprtc.com",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+
+        for menu in self.menus:
+            start_url = 'https://test.sprtc.com/idtAppServiceV6/exclude/control/getGrid'
+            yield feapder.Request(url=start_url,item=menu._asdict(),page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "script": f"{menu.get('typeone')}",
+            "args": "[]",
+            "keys": f"[\"{menu.get('typetwo')}\"]",
+            "replaceMap": "{\"developer\":\"sjsxm\"}",
+            "pageSize": "20",
+            "nowPage": f"{page}"
+        }
+        request.data = data
+        request.headers=self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('DataList')
+        for info in info_list:
+            href_id = info.get('UUID').strip()
+            href = f"https://www.sprtc.com/index/qrtwo.htm?id={href_id}"
+            title = info.get('BDMC').strip()
+            create_time = info.get('GPJSSJ').strip()
+
+            area = "辽宁"  # 省份
+            city = "沈阳市"  # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = create_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"
+            list_item.deal_detail = ['//div[@id="formApp"]/div[@id="formScrollWrap"]']
+            list_item.proxies = False
+            list_item.parse_url = href
+            list_item.render_time = 5
+
+            yield list_item
+
+
+if __name__ == "__main__":
+    Sylhcqjys(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_zccz/sylhcqjys_details_firefox.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+import time
+from feapder.utils.log import log
+from lxml.html import fromstring,tostring
+from untils.attachment import AttachmentDownloader
+import re
+import requests
+import json
+
+
+
+
+class FirefoxDetails(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),item=item,files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),**request_params,
+                                  callback=eval(item.get("parse")),render=True,
+                                  render_time=item.get("render_time"),proxies=False)
+
+    def detail_get(self,request,response):
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        driver = response.browser
+        driver.switch_to.frame('iframe')
+        driver.switch_to.frame('project-page')
+        pub_time = re.findall('\d{4}-\d{1,2}-\d{1,2}',driver.page_source,re.S)
+        if pub_time:
+            list_item.publishtime = pub_time[0]
+        root = fromstring(driver.page_source)
+        iframe_list = root.xpath('//iframe[@class="idt-subForm-iframe innerFrame"]/@id')
+        html_str = root.xpath('//div[@id="formApp"]/div[@id="formScrollWrap"]')
+        if html_str != None and len(html_str) > 0:
+            html_str = tostring(html_str[0],encoding='utf-8').decode()
+        else:
+            html_str = ''
+        for iframe in iframe_list:
+            driver.switch_to.frame(f'{iframe}')
+            res = fromstring(driver.page_source)
+            html = res.xpath(request.deal_detail[0])  # 标书详细内容
+            html = tostring(html[0],encoding='utf-8').decode()
+            html_str += html
+            if res.xpath('//iframe/@id'):
+                for four_iframe in res.xpath('//iframe/@id'):
+                    driver.switch_to.frame(f'{four_iframe}')
+                    driver.switch_to.frame('project-page')
+                    f_res = fromstring(driver.page_source)
+                    html_f = f_res.xpath('//div[@id="formApp"]/div[@class="formScrollBar"]')  # 标书详细内容
+                    html_f = tostring(html_f[0], encoding='utf-8').decode()
+                    html_str += html_f
+                    driver.switch_to.parent_frame()
+                    driver.switch_to.parent_frame()
+            driver.switch_to.parent_frame()
+
+
+        list_item.contenthtml = html_str
+
+        driver.switch_to.frame(f'{iframe_list[-1]}')
+        time.sleep(2)
+        fujian_res = driver.page_source
+        if re.findall('"tableId":"(.*?)"',fujian_res,re.S):
+            fujian_tableId = re.findall('"tableId":"(.*?)"',fujian_res,re.S)[-1].strip()
+            id = "".join(re.findall('id=(.*?)&',list_item.href))
+            f_name = "".join(re.findall('data-filedname="(.*?)"',fujian_res,re.S))
+            data = {
+                "tableId": f"{fujian_tableId}",
+                "uuid": f"{id}",
+                "fieldName": f"{f_name}",
+                "developer": "sjsxm",
+            }
+            url = 'https://www.sprtc.com/idtAppServiceV6/exclude/control/getFiles'
+            f_res = requests.post(url, data=data)
+            file_dict = json.loads(f_res.content.decode(),strict=False).get('data')
+            if file_dict:
+                attachments = {}
+                for info in file_dict:
+                    uuid = info.get('UUID')
+                    file_url = f"https://www.sprtc.com/idtAppServiceV6/oApp/downloadLargeFile?tableId={fujian_tableId}&uuid={uuid}&developer=sjsxm&authToken=null"
+                    file_name = info.get('FILENAME').strip()
+                    file_type = file_name.split(".")[-1].lower()
+                    file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls', 'xlsx', 'swp']
+                    if file_type in file_types:
+                        if file_type in file_name:
+                            file_name = file_name.replace(f'.{file_type}', '')
+                        attachment = AttachmentDownloader().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url)
+                        attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    FirefoxDetails(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_zccz/资产出租-列表页.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+
+class Sylhcqjys(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code','typeone', 'typetwo','typethree', 'crawl_page'])
+        self.site = "沈阳联合产权交易所"
+
+        self.menus = [
+            Menu('资产出租', 'a_sylhcqjys_zccz','qyxzcxmGetList', 'zc', '7', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            "Origin": "https://test.sprtc.com",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+
+        for menu in self.menus:
+            start_url = 'https://test.sprtc.com/idtAppServiceV6/exclude/control/getGrid'
+            yield feapder.Request(url=start_url,item=menu._asdict(),page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "script": f"{menu.get('typeone')}",
+            "args": "[]",
+            "keys": f"[\"{menu.get('typetwo')}\"]",
+            "replaceMap": "{\"developer\":\"sjsxm\"}",
+            "pageSize": "20",
+            "nowPage": f"{page}"
+        }
+        request.data = data
+        request.headers=self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('DataList')
+        for info in info_list:
+            href_id = info.get('UUID').strip()
+            href = f"https://www.sprtc.com/index/qrtwo.htm?id={href_id}"
+            title = info.get('BDMC').strip()
+            create_time = info.get('GPJSSJ').strip()
+
+            area = "辽宁"  # 省份
+            city = "沈阳市"  # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = create_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"
+            list_item.deal_detail = ['//div[@id="formApp"]/div[@id="formScrollWrap"]']
+            list_item.proxies = False
+            list_item.parse_url = href
+            list_item.render_time = 5
+
+            yield list_item
+
+
+if __name__ == "__main__":
+    Sylhcqjys(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_zczr/sylhcqjys_details_firefox.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+import time
+from feapder.utils.log import log
+from lxml.html import fromstring,tostring
+from untils.attachment import AttachmentDownloader
+import re
+import requests
+import json
+
+
+
+
+class FirefoxDetails(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),item=item,files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),**request_params,
+                                  callback=eval(item.get("parse")),render=True,
+                                  render_time=item.get("render_time"),proxies=False)
+
+    def detail_get(self,request,response):
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        driver = response.browser
+        driver.switch_to.frame('iframe')
+        driver.switch_to.frame('project-page')
+        pub_time = re.findall('\d{4}-\d{1,2}-\d{1,2}',driver.page_source,re.S)
+        if pub_time:
+            list_item.publishtime = pub_time[0]
+        root = fromstring(driver.page_source)
+        iframe_list = root.xpath('//iframe[@class="idt-subForm-iframe innerFrame"]/@id')
+        html_str = root.xpath('//div[@id="formApp"]/div[@id="formScrollWrap"]')
+        if html_str != None and len(html_str) > 0:
+            html_str = tostring(html_str[0],encoding='utf-8').decode()
+        else:
+            html_str = ''
+        for iframe in iframe_list:
+            driver.switch_to.frame(f'{iframe}')
+            res = fromstring(driver.page_source)
+            html = res.xpath(request.deal_detail[0])  # 标书详细内容
+            html = tostring(html[0],encoding='utf-8').decode()
+            html_str += html
+            if res.xpath('//iframe/@id'):
+                for four_iframe in res.xpath('//iframe/@id'):
+                    driver.switch_to.frame(f'{four_iframe}')
+                    driver.switch_to.frame('project-page')
+                    f_res = fromstring(driver.page_source)
+                    html_f = f_res.xpath('//div[@id="formApp"]/div[@class="formScrollBar"]')  # 标书详细内容
+                    html_f = tostring(html_f[0], encoding='utf-8').decode()
+                    html_str += html_f
+                    driver.switch_to.parent_frame()
+                    driver.switch_to.parent_frame()
+            driver.switch_to.parent_frame()
+
+
+        list_item.contenthtml = html_str
+
+        driver.switch_to.frame(f'{iframe_list[-1]}')
+        time.sleep(2)
+        fujian_res = driver.page_source
+        if re.findall('"tableId":"(.*?)"',fujian_res,re.S):
+            fujian_tableId = re.findall('"tableId":"(.*?)"',fujian_res,re.S)[-1].strip()
+            id = "".join(re.findall('id=(.*?)&',list_item.href))
+            f_name = "".join(re.findall('data-filedname="(.*?)"',fujian_res,re.S))
+            data = {
+                "tableId": f"{fujian_tableId}",
+                "uuid": f"{id}",
+                "fieldName": f"{f_name}",
+                "developer": "sjsxm",
+            }
+            url = 'https://www.sprtc.com/idtAppServiceV6/exclude/control/getFiles'
+            f_res = requests.post(url, data=data)
+            file_dict = json.loads(f_res.content.decode(),strict=False).get('data')
+            if file_dict:
+                attachments = {}
+                for info in file_dict:
+                    uuid = info.get('UUID')
+                    file_url = f"https://www.sprtc.com/idtAppServiceV6/oApp/downloadLargeFile?tableId={fujian_tableId}&uuid={uuid}&developer=sjsxm&authToken=null"
+                    file_name = info.get('FILENAME').strip()
+                    file_type = file_name.split(".")[-1].lower()
+                    file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls', 'xlsx', 'swp']
+                    if file_type in file_types:
+                        if file_type in file_name:
+                            file_name = file_name.replace(f'.{file_type}', '')
+                        attachment = AttachmentDownloader().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url)
+                        attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    FirefoxDetails(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_zczr/资产转让-列表页.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+
+class Sylhcqjys(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code','typeone', 'typetwo','typethree', 'crawl_page'])
+        self.site = "沈阳联合产权交易所"
+
+        self.menus = [
+            Menu('资产转让', 'a_sylhcqjys_zczr','wqzrxmGetList', 'zc', '1', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            "Origin": "https://test.sprtc.com",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+
+        for menu in self.menus:
+            start_url = 'https://test.sprtc.com/idtAppServiceV6/exclude/control/getGrid'
+            yield feapder.Request(url=start_url,item=menu._asdict(),page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "script": f"{menu.get('typeone')}",
+            "args": "[]",
+            "keys": f"[\"{menu.get('typetwo')}\"]",
+            "replaceMap": "{\"developer\":\"sjsxm\"}",
+            "pageSize": "20",
+            "nowPage": f"{page}"
+        }
+        request.data = data
+        request.headers=self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('DataList')
+        for info in info_list:
+            href_id = info.get('UUID').strip()
+            href = f"https://www.sprtc.com/index/qrtwo.htm?id={href_id}"
+            title = info.get('BDMC').strip()
+            create_time = info.get('GPJSSJ').strip()
+
+            area = "辽宁"  # 省份
+            city = "沈阳市"  # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = create_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"
+            list_item.deal_detail = ['//div[@id="formApp"]/div[@id="formScrollWrap"]']
+            list_item.proxies = False
+            list_item.parse_url = href
+            list_item.render_time = 5
+
+            yield list_item
+
+
+if __name__ == "__main__":
+    Sylhcqjys(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_zzkg/sylhcqjys_details_firefox.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+import time
+from feapder.utils.log import log
+from lxml.html import fromstring,tostring
+from untils.attachment import AttachmentDownloader
+import re
+import requests
+import json
+
+
+
+
+class FirefoxDetails(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"),item=item,files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"),**request_params,
+                                  callback=eval(item.get("parse")),render=True,
+                                  render_time=item.get("render_time"),proxies=False)
+
+    def detail_get(self,request,response):
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        driver = response.browser
+        driver.switch_to.frame('iframe')
+        driver.switch_to.frame('project-page')
+        pub_time = re.findall('\d{4}-\d{1,2}-\d{1,2}',driver.page_source,re.S)
+        if pub_time:
+            list_item.publishtime = pub_time[0]
+        root = fromstring(driver.page_source)
+        iframe_list = root.xpath('//iframe[@class="idt-subForm-iframe innerFrame"]/@id')
+        html_str = root.xpath('//div[@id="formApp"]/div[@id="formScrollWrap"]')
+        if html_str != None and len(html_str) > 0:
+            html_str = tostring(html_str[0],encoding='utf-8').decode()
+        else:
+            html_str = ''
+        for iframe in iframe_list:
+            driver.switch_to.frame(f'{iframe}')
+            res = fromstring(driver.page_source)
+            html = res.xpath(request.deal_detail[0])  # 标书详细内容
+            html = tostring(html[0],encoding='utf-8').decode()
+            html_str += html
+            if res.xpath('//iframe/@id'):
+                for four_iframe in res.xpath('//iframe/@id'):
+                    driver.switch_to.frame(f'{four_iframe}')
+                    driver.switch_to.frame('project-page')
+                    f_res = fromstring(driver.page_source)
+                    html_f = f_res.xpath('//div[@id="formApp"]/div[@class="formScrollBar"]')  # 标书详细内容
+                    html_f = tostring(html_f[0], encoding='utf-8').decode()
+                    html_str += html_f
+                    driver.switch_to.parent_frame()
+                    driver.switch_to.parent_frame()
+            driver.switch_to.parent_frame()
+
+
+        list_item.contenthtml = html_str
+
+        driver.switch_to.frame(f'{iframe_list[-1]}')
+        time.sleep(2)
+        fujian_res = driver.page_source
+        if re.findall('"tableId":"(.*?)"',fujian_res,re.S):
+            fujian_tableId = re.findall('"tableId":"(.*?)"',fujian_res,re.S)[-1].strip()
+            id = "".join(re.findall('id=(.*?)&',list_item.href))
+            f_name = "".join(re.findall('data-filedname="(.*?)"',fujian_res,re.S))
+            data = {
+                "tableId": f"{fujian_tableId}",
+                "uuid": f"{id}",
+                "fieldName": f"{f_name}",
+                "developer": "sjsxm",
+            }
+            url = 'https://www.sprtc.com/idtAppServiceV6/exclude/control/getFiles'
+            f_res = requests.post(url, data=data)
+            file_dict = json.loads(f_res.content.decode(),strict=False).get('data')
+            if file_dict:
+                attachments = {}
+                for info in file_dict:
+                    uuid = info.get('UUID')
+                    file_url = f"https://www.sprtc.com/idtAppServiceV6/oApp/downloadLargeFile?tableId={fujian_tableId}&uuid={uuid}&developer=sjsxm&authToken=null"
+                    file_name = info.get('FILENAME').strip()
+                    file_type = file_name.split(".")[-1].lower()
+                    file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg', 'png', 'zbid', 'xls', 'xlsx', 'swp']
+                    if file_type in file_types:
+                        if file_type in file_name:
+                            file_name = file_name.replace(f'.{file_type}', '')
+                        attachment = AttachmentDownloader().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url)
+                        attachments[str(len(attachments) + 1)] = attachment
+                if attachments:
+                    list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    FirefoxDetails(redis_key="lzz:Sylhcqjys").start()

a_sylhcqjys_zzkg/增资扩股-列表页.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-28
+---------
+@summary: 沈阳联合产权交易所
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+
+class Sylhcqjys(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code','typeone', 'typetwo','typethree', 'crawl_page'])
+        self.site = "沈阳联合产权交易所"
+
+        self.menus = [
+            Menu('增资扩股', 'a_sylhcqjys_zzkg','zckgxmGetList', '', '9', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+            "Origin": "https://test.sprtc.com",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+
+        for menu in self.menus:
+            start_url = 'https://test.sprtc.com/idtAppServiceV6/exclude/control/getGrid'
+            yield feapder.Request(url=start_url,item=menu._asdict(),page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "script": f"{menu.get('typeone')}",
+            "args": "[]",
+            "keys": f"[\"{menu.get('typetwo')}\"]",
+            "replaceMap": "{\"developer\":\"sjsxm\"}",
+            "pageSize": "20",
+            "nowPage": f"{page}"
+        }
+        request.data = data
+        request.headers=self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('DataList')
+        for info in info_list:
+            href_id = info.get('UUID').strip()
+            href = f"https://www.sprtc.com/index/qrtwo.htm?id={href_id}"
+            title = info.get('BDMC').strip()
+            create_time = info.get('GPJSSJ').strip()
+
+            area = "辽宁"  # 省份
+            city = "沈阳市"  # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = create_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"
+            list_item.deal_detail = ['//div[@id="formApp"]/div[@id="formScrollWrap"]']
+            list_item.proxies = False
+            list_item.parse_url = href
+            list_item.render_time = 5
+
+            yield list_item
+
+
+if __name__ == "__main__":
+    Sylhcqjys(redis_key="lzz:Sylhcqjys").start()

a_whswszjcs_cggg/交易信息-详情页.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 芜湖市网上中介超市
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+import requests
+
+
+
+
+def get_file(did):
+    headers = {
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "Content-Type": "application/x-www-form-urlencoded",
+        "Origin": "https://wh.ahzwfw.gov.cn",
+        "Pragma": "no-cache",
+        "Referer": "https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+    }
+
+    url = "https://wh.ahzwfw.gov.cn/wszjcs-web/tradingInformation/getTradingInformationDetails.do"
+    data = {
+        "id": did,
+    }
+    response = requests.post(url, headers=headers, data=data, timeout=30, verify=False)
+    return response.json().get('data').get('attachmentDtos')
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=10)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+            did = item.get('did')
+            yield feapder.Request(url=item.get("parse_url"), item=item, proxies=False,render=True,render_time=5,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout,did=did)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="detail"]').extract_first()
+
+        attachments = {}
+        file_list = get_file(request.did)
+        if file_list:
+            for info in file_list:
+                file_name = info.get('fileName').strip()
+                fid = info.get('filePath')
+                file_url = f"https://wh.ahzwfw.gov.cn/wszjcs-web/file/downloadFile.do?fileId={fid}&fileName={file_name}"
+                file_type = extract_file_type(file_name=file_name, file_url=fid)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url,)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        list_item.contenthtml = html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:wnjtdzzbcgpt_jggs").start()

a_whswszjcs_cggg/采购公告-列表页.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 芜湖市网上中介超市
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "芜湖市网上中介超市"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.menus = [
+            Menu('采购公告', 'a_whswszjcs_cggg', '1', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://wh.ahzwfw.gov.cn",
+            "Pragma": "no-cache",
+            "Referer": "https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://wh.ahzwfw.gov.cn/wszjcs-web/tradingInformation/getTradingInformationPage.do"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "currentPageNo": f"{page}",
+            "pageSize": "10",
+            "filterType": menu.get('tid'),
+            "projectName": "",
+            "state": "",
+            "isRelease": "",
+            "uscCode": "",
+            "winningAnnouncementStatus": "",
+            "publicityStatus": "",
+        }
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('rows')
+        for info in info_list:
+            hid = info.get('id')
+            t = int(menu.get('tid'))-1
+            href = f"https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html#/projectDetail?id={hid}&type=-1&usc_code=&title={t}&selectAgain=false&mark=1"
+            title = info.get('projectName').strip()
+            create_time = timestamp_to_date(int(str(info.get('creationTime'))[:10]))
+
+            area = "安徽"
+            city = "芜湖市"
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="detail"]']
+            list_item.proxies = False
+            list_item.did = hid
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:wnjtdzzbcgpt_jggs").start()

a_whswszjcs_jyyc/交易信息-详情页.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 芜湖市网上中介超市
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+import requests
+
+
+
+
+def get_file(did):
+    headers = {
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "Content-Type": "application/x-www-form-urlencoded",
+        "Origin": "https://wh.ahzwfw.gov.cn",
+        "Pragma": "no-cache",
+        "Referer": "https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+    }
+
+    url = "https://wh.ahzwfw.gov.cn/wszjcs-web/tradingInformation/getTradingInformationDetails.do"
+    data = {
+        "id": did,
+    }
+    response = requests.post(url, headers=headers, data=data, timeout=30, verify=False)
+    return response.json().get('data').get('attachmentDtos')
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=10)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+            did = item.get('did')
+            yield feapder.Request(url=item.get("parse_url"), item=item, proxies=False,render=True,render_time=5,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout,did=did)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="detail"]').extract_first()
+
+        attachments = {}
+        file_list = get_file(request.did)
+        if file_list:
+            for info in file_list:
+                file_name = info.get('fileName').strip()
+                fid = info.get('filePath')
+                file_url = f"https://wh.ahzwfw.gov.cn/wszjcs-web/file/downloadFile.do?fileId={fid}&fileName={file_name}"
+                file_type = extract_file_type(file_name=file_name, file_url=fid)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url,)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        list_item.contenthtml = html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:wnjtdzzbcgpt_jggs").start()

a_whswszjcs_jyyc/交易异常-列表页.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 芜湖市网上中介超市
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "芜湖市网上中介超市"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.menus = [
+            Menu('交易异常', 'a_whswszjcs_jyyc', '5', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://wh.ahzwfw.gov.cn",
+            "Pragma": "no-cache",
+            "Referer": "https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://wh.ahzwfw.gov.cn/wszjcs-web/tradingInformation/getTradingInformationPage.do"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "currentPageNo": f"{page}",
+            "pageSize": "10",
+            "filterType": menu.get('tid'),
+            "projectName": "",
+            "state": "",
+            "isRelease": "",
+            "uscCode": "",
+            "winningAnnouncementStatus": "",
+            "publicityStatus": "",
+        }
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('rows')
+        for info in info_list:
+            hid = info.get('id')
+            t = int(menu.get('tid'))-1
+            href = f"https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html#/projectDetail?id={hid}&type=-1&usc_code=&title={t}&selectAgain=false&mark=1"
+            title = info.get('projectName').strip()
+            create_time = timestamp_to_date(int(str(info.get('creationTime'))[:10]))
+
+            area = "安徽"
+            city = "芜湖市"
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="detail"]']
+            list_item.proxies = False
+            list_item.did = hid
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:wnjtdzzbcgpt_jggs").start()

a_whswszjcs_lyxx/交易信息-详情页.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 芜湖市网上中介超市
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+import requests
+
+
+
+
+def get_file(did):
+    headers = {
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "Content-Type": "application/x-www-form-urlencoded",
+        "Origin": "https://wh.ahzwfw.gov.cn",
+        "Pragma": "no-cache",
+        "Referer": "https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+    }
+
+    url = "https://wh.ahzwfw.gov.cn/wszjcs-web/tradingInformation/getTradingInformationDetails.do"
+    data = {
+        "id": did,
+    }
+    response = requests.post(url, headers=headers, data=data, timeout=30, verify=False)
+    return response.json().get('data').get('attachmentDtos')
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=10)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+            did = item.get('did')
+            yield feapder.Request(url=item.get("parse_url"), item=item, proxies=False,render=True,render_time=5,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout,did=did)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="detail"]').extract_first()
+
+        attachments = {}
+        file_list = get_file(request.did)
+        if file_list:
+            for info in file_list:
+                file_name = info.get('fileName').strip()
+                fid = info.get('filePath')
+                file_url = f"https://wh.ahzwfw.gov.cn/wszjcs-web/file/downloadFile.do?fileId={fid}&fileName={file_name}"
+                file_type = extract_file_type(file_name=file_name, file_url=fid)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url,)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        list_item.contenthtml = html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:wnjtdzzbcgpt_jggs").start()

a_whswszjcs_lyxx/履约信息-列表页.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 芜湖市网上中介超市
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "芜湖市网上中介超市"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.menus = [
+            Menu('履约信息', 'a_whswszjcs_lyxx', '4', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://wh.ahzwfw.gov.cn",
+            "Pragma": "no-cache",
+            "Referer": "https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://wh.ahzwfw.gov.cn/wszjcs-web/tradingInformation/getTradingInformationPage.do"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "currentPageNo": f"{page}",
+            "pageSize": "10",
+            "filterType": menu.get('tid'),
+            "projectName": "",
+            "state": "",
+            "isRelease": "",
+            "uscCode": "",
+            "winningAnnouncementStatus": "",
+            "publicityStatus": "",
+        }
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('rows')
+        for info in info_list:
+            hid = info.get('id')
+            t = int(menu.get('tid'))-1
+            href = f"https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html#/projectDetail?id={hid}&type=-1&usc_code=&title={t}&selectAgain=false&mark=1"
+            title = info.get('projectName').strip()
+            create_time = timestamp_to_date(int(str(info.get('creationTime'))[:10]))
+
+            area = "安徽"
+            city = "芜湖市"
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="detail"]']
+            list_item.proxies = False
+            list_item.did = hid
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:wnjtdzzbcgpt_jggs").start()

a_whswszjcs_zxgg/中选公告-列表页.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 芜湖市网上中介超市
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "芜湖市网上中介超市"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.menus = [
+            Menu('中选公告', 'a_whswszjcs_zxgg', '2', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://wh.ahzwfw.gov.cn",
+            "Pragma": "no-cache",
+            "Referer": "https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://wh.ahzwfw.gov.cn/wszjcs-web/tradingInformation/getTradingInformationPage.do"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "currentPageNo": f"{page}",
+            "pageSize": "10",
+            "filterType": menu.get('tid'),
+            "projectName": "",
+            "state": "",
+            "isRelease": "",
+            "uscCode": "",
+            "winningAnnouncementStatus": "",
+            "publicityStatus": "",
+        }
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('rows')
+        for info in info_list:
+            hid = info.get('id')
+            t = int(menu.get('tid'))-1
+            href = f"https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html#/projectDetail?id={hid}&type=-1&usc_code=&title={t}&selectAgain=false&mark=1"
+            title = info.get('projectName').strip()
+            create_time = timestamp_to_date(int(str(info.get('creationTime'))[:10]))
+
+            area = "安徽"
+            city = "芜湖市"
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="detail"]']
+            list_item.proxies = False
+            list_item.did = hid
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:wnjtdzzbcgpt_jggs").start()

a_whswszjcs_zxgg/交易信息-详情页.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-06-02
+---------
+@summary: 芜湖市网上中介超市
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+import requests
+
+
+
+
+def get_file(did):
+    headers = {
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "Content-Type": "application/x-www-form-urlencoded",
+        "Origin": "https://wh.ahzwfw.gov.cn",
+        "Pragma": "no-cache",
+        "Referer": "https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+    }
+
+    url = "https://wh.ahzwfw.gov.cn/wszjcs-web/tradingInformation/getTradingInformationDetails.do"
+    data = {
+        "id": did,
+    }
+    response = requests.post(url, headers=headers, data=data, timeout=30, verify=False)
+    return response.json().get('data').get('attachmentDtos')
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=10)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+            did = item.get('did')
+            yield feapder.Request(url=item.get("parse_url"), item=item, proxies=False,render=True,render_time=5,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout,did=did)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="detail"]').extract_first()
+
+        attachments = {}
+        file_list = get_file(request.did)
+        if file_list:
+            for info in file_list:
+                file_name = info.get('fileName').strip()
+                fid = info.get('filePath')
+                file_url = f"https://wh.ahzwfw.gov.cn/wszjcs-web/file/downloadFile.do?fileId={fid}&fileName={file_name}"
+                file_type = extract_file_type(file_name=file_name, file_url=fid)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url,)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        list_item.contenthtml = html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:wnjtdzzbcgpt_jggs").start()

a_ygzlpt_gggs_hfgg/公告公示-恢复公告-列表页.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import json
+
+
+class Feapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.site = "阳光租赁平台"
+
+        self.menus = [
+            Menu('公告公示-恢复公告', 'a_ygzlpt_gggs_hfgg', '恢复公告', 1),
+        ]
+
+        self.headers = {
+            "accept": "application/json, text/javascript, */*; q=0.01",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "content-type": "application/json",
+            "origin": "https://rent.szexgrp.com",
+            "priority": "u=1, i",
+            "referer": "https://rent.szexgrp.com/page/notice/notice.html",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+            "x-requested-with": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://rent.szexgrp.com/cms/api/v1/lease/notice/page"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1,  proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "subClassifyType": "",
+            "noticeTypeName": menu.get('tid'),
+            "publishTimeStartTime": "",
+            "publishTimeEndTime": "",
+            "noticeTitle": "",
+            "statusOrder": "",
+            "orderByParam": "noticeStartTime",
+            "orderByDirection": "DESC",
+            "pageNum": page-1,
+            "pageSize": 10
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('content')
+        for info in info_list:
+            hid = info.get('projectContentId')
+            href = f'https://rent.szexgrp.com/page/projectDetail/projectDetail.html?contentId={hid}'
+            title = info.get('noticeTitle').strip()
+            create_time = info.get('publishTime')
+
+            area = "广东"      # 省份
+            city = "深圳市"    # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('title','href')
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.infoformat = 3
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="rent-width "]']
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 无限翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    Feapder(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_hfgg/公告公示-详情页.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.tools import log
+from untils.attachment import AttachmentDownloader as AD
+from untils.tools import remove_htmldata
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item,render=True,render_time=5,
+                                          deal_detail=item.get("deal_detail"),proxies=False,
+                                          callback=eval(item.get("parse")), **request_params)
+            
+
+    def download_midware(self, request):
+        request.headers = {
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+    def detail_get(self,request,response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="rent-width "]').extract_first()
+        html = remove_htmldata(['<re>data:image/(.*?)"'],html,response)
+        list_item.contenthtml = html
+
+        file_list = response.xpath('//div[@class="rent-width "]//a[@href]')
+
+        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg',
+                      'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg', 'wps', 'ofd']
+        attachments = {}
+        if file_list:
+            for info in file_list:
+                file_url = info.xpath('./@href').extract_first()
+                file_name = info.xpath('./text()').extract_first()
+                if file_name:
+                    file_name =file_name.strip()
+                    file_type = file_name.split('.')[-1].lower()
+
+                    if file_type in file_types and "file" in file_url:
+                        attachment = AD().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url,
+                            enable_proxy=False)
+                        attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_jggg/公告公示-结果公告-列表页.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import json
+
+
+class Feapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.site = "阳光租赁平台"
+
+        self.menus = [
+            Menu('公告公示-结果公告', 'a_ygzlpt_gggs_jggg', '结果公告', 2),
+        ]
+
+        self.headers = {
+            "accept": "application/json, text/javascript, */*; q=0.01",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "content-type": "application/json",
+            "origin": "https://rent.szexgrp.com",
+            "priority": "u=1, i",
+            "referer": "https://rent.szexgrp.com/page/notice/notice.html",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+            "x-requested-with": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://rent.szexgrp.com/cms/api/v1/lease/notice/page"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1,  proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "subClassifyType": "",
+            "noticeTypeName": menu.get('tid'),
+            "publishTimeStartTime": "",
+            "publishTimeEndTime": "",
+            "noticeTitle": "",
+            "statusOrder": "",
+            "orderByParam": "noticeStartTime",
+            "orderByDirection": "DESC",
+            "pageNum": page-1,
+            "pageSize": 10
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('content')
+        for info in info_list:
+            hid = info.get('projectContentId')
+            href = f'https://rent.szexgrp.com/page/projectDetail/projectDetail.html?contentId={hid}'
+            title = info.get('noticeTitle').strip()
+            create_time = info.get('publishTime')
+
+            area = "广东"      # 省份
+            city = "深圳市"    # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('title','href')
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.infoformat = 3
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="rent-width "]']
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 无限翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    Feapder(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_jggg/公告公示-详情页.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.tools import log
+from untils.attachment import AttachmentDownloader as AD
+from untils.tools import remove_htmldata
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item,render=True,render_time=5,
+                                          deal_detail=item.get("deal_detail"),proxies=False,
+                                          callback=eval(item.get("parse")), **request_params)
+            
+
+    def download_midware(self, request):
+        request.headers = {
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+    def detail_get(self,request,response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="rent-width "]').extract_first()
+        html = remove_htmldata(['<re>data:image/(.*?)"'],html,response)
+        list_item.contenthtml = html
+
+        file_list = response.xpath('//div[@class="rent-width "]//a[@href]')
+
+        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg',
+                      'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg', 'wps', 'ofd']
+        attachments = {}
+        if file_list:
+            for info in file_list:
+                file_url = info.xpath('./@href').extract_first()
+                file_name = info.xpath('./text()').extract_first()
+                if file_name:
+                    file_name =file_name.strip()
+                    file_type = file_name.split('.')[-1].lower()
+
+                    if file_type in file_types and "file" in file_url:
+                        attachment = AD().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url,
+                            enable_proxy=False)
+                        attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_jggs/公告公示-结果公示-列表页.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import json
+
+
+class Feapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.site = "阳光租赁平台"
+
+        self.menus = [
+            Menu('公告公示-结果公示', 'a_ygzlpt_gggs_jggs', '结果公示', 3),
+        ]
+
+        self.headers = {
+            "accept": "application/json, text/javascript, */*; q=0.01",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "content-type": "application/json",
+            "origin": "https://rent.szexgrp.com",
+            "priority": "u=1, i",
+            "referer": "https://rent.szexgrp.com/page/notice/notice.html",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+            "x-requested-with": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://rent.szexgrp.com/cms/api/v1/lease/notice/page"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1,  proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "subClassifyType": "",
+            "noticeTypeName": menu.get('tid'),
+            "publishTimeStartTime": "",
+            "publishTimeEndTime": "",
+            "noticeTitle": "",
+            "statusOrder": "",
+            "orderByParam": "noticeStartTime",
+            "orderByDirection": "DESC",
+            "pageNum": page-1,
+            "pageSize": 10
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('content')
+        for info in info_list:
+            hid = info.get('projectContentId')
+            href = f'https://rent.szexgrp.com/page/projectDetail/projectDetail.html?contentId={hid}'
+            title = info.get('noticeTitle').strip()
+            create_time = info.get('publishTime')
+
+            area = "广东"      # 省份
+            city = "深圳市"    # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('title','href')
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.infoformat = 3
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="rent-width "]']
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 无限翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    Feapder(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_jggs/公告公示-详情页.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.tools import log
+from untils.attachment import AttachmentDownloader as AD
+from untils.tools import remove_htmldata
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item,render=True,render_time=5,
+                                          deal_detail=item.get("deal_detail"),proxies=False,
+                                          callback=eval(item.get("parse")), **request_params)
+            
+
+    def download_midware(self, request):
+        request.headers = {
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+    def detail_get(self,request,response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="rent-width "]').extract_first()
+        html = remove_htmldata(['<re>data:image/(.*?)"'],html,response)
+        list_item.contenthtml = html
+
+        file_list = response.xpath('//div[@class="rent-width "]//a[@href]')
+
+        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg',
+                      'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg', 'wps', 'ofd']
+        attachments = {}
+        if file_list:
+            for info in file_list:
+                file_url = info.xpath('./@href').extract_first()
+                file_name = info.xpath('./text()').extract_first()
+                if file_name:
+                    file_name =file_name.strip()
+                    file_type = file_name.split('.')[-1].lower()
+
+                    if file_type in file_types and "file" in file_url:
+                        attachment = AD().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url,
+                            enable_proxy=False)
+                        attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_jygg/公告公示-交易公告-列表页.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import json
+
+
+class Feapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.site = "阳光租赁平台"
+
+        self.menus = [
+            Menu('公告公示-交易公告', 'a_ygzlpt_gggs_jygg', '交易公告', 3),
+        ]
+
+        self.headers = {
+            "accept": "application/json, text/javascript, */*; q=0.01",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "content-type": "application/json",
+            "origin": "https://rent.szexgrp.com",
+            "priority": "u=1, i",
+            "referer": "https://rent.szexgrp.com/page/notice/notice.html",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+            "x-requested-with": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://rent.szexgrp.com/cms/api/v1/lease/notice/page"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1,  proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "subClassifyType": "",
+            "noticeTypeName": menu.get('tid'),
+            "publishTimeStartTime": "",
+            "publishTimeEndTime": "",
+            "noticeTitle": "",
+            "statusOrder": "",
+            "orderByParam": "noticeStartTime",
+            "orderByDirection": "DESC",
+            "pageNum": page-1,
+            "pageSize": 10
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('content')
+        for info in info_list:
+            hid = info.get('projectContentId')
+            href = f'https://rent.szexgrp.com/page/projectDetail/projectDetail.html?contentId={hid}'
+            title = info.get('noticeTitle').strip()
+            create_time = info.get('publishTime')
+
+            area = "广东"      # 省份
+            city = "深圳市"    # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('title','href')
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.infoformat = 3
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="rent-width "]']
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 无限翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    Feapder(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_jygg/公告公示-详情页.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.tools import log
+from untils.attachment import AttachmentDownloader as AD
+from untils.tools import remove_htmldata
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item,render=True,render_time=5,
+                                          deal_detail=item.get("deal_detail"),proxies=False,
+                                          callback=eval(item.get("parse")), **request_params)
+            
+
+    def download_midware(self, request):
+        request.headers = {
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+    def detail_get(self,request,response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="rent-width "]').extract_first()
+        html = remove_htmldata(['<re>data:image/(.*?)"'],html,response)
+        list_item.contenthtml = html
+
+        file_list = response.xpath('//div[@class="rent-width "]//a[@href]')
+
+        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg',
+                      'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg', 'wps', 'ofd']
+        attachments = {}
+        if file_list:
+            for info in file_list:
+                file_url = info.xpath('./@href').extract_first()
+                file_name = info.xpath('./text()').extract_first()
+                if file_name:
+                    file_name =file_name.strip()
+                    file_type = file_name.split('.')[-1].lower()
+
+                    if file_type in file_types and "file" in file_url:
+                        attachment = AD().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url,
+                            enable_proxy=False)
+                        attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_lbgg/公告公示-流标公告-列表页.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import json
+
+
+class Feapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.site = "阳光租赁平台"
+
+        self.menus = [
+            Menu('公告公示-流标公告', 'a_ygzlpt_gggs_lbgg', '流标公告', 1),
+        ]
+
+        self.headers = {
+            "accept": "application/json, text/javascript, */*; q=0.01",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "content-type": "application/json",
+            "origin": "https://rent.szexgrp.com",
+            "priority": "u=1, i",
+            "referer": "https://rent.szexgrp.com/page/notice/notice.html",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+            "x-requested-with": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://rent.szexgrp.com/cms/api/v1/lease/notice/page"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1,  proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "subClassifyType": "",
+            "noticeTypeName": menu.get('tid'),
+            "publishTimeStartTime": "",
+            "publishTimeEndTime": "",
+            "noticeTitle": "",
+            "statusOrder": "",
+            "orderByParam": "noticeStartTime",
+            "orderByDirection": "DESC",
+            "pageNum": page-1,
+            "pageSize": 10
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('content')
+        for info in info_list:
+            hid = info.get('projectContentId')
+            href = f'https://rent.szexgrp.com/page/projectDetail/projectDetail.html?contentId={hid}'
+            title = info.get('noticeTitle').strip()
+            create_time = info.get('publishTime')
+
+            area = "广东"      # 省份
+            city = "深圳市"    # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('title','href')
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.infoformat = 3
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="rent-width "]']
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 无限翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    Feapder(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_lbgg/公告公示-详情页.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.tools import log
+from untils.attachment import AttachmentDownloader as AD
+from untils.tools import remove_htmldata
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item,render=True,render_time=5,
+                                          deal_detail=item.get("deal_detail"),proxies=False,
+                                          callback=eval(item.get("parse")), **request_params)
+            
+
+    def download_midware(self, request):
+        request.headers = {
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+    def detail_get(self,request,response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="rent-width "]').extract_first()
+        html = remove_htmldata(['<re>data:image/(.*?)"'],html,response)
+        list_item.contenthtml = html
+
+        file_list = response.xpath('//div[@class="rent-width "]//a[@href]')
+
+        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg',
+                      'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg', 'wps', 'ofd']
+        attachments = {}
+        if file_list:
+            for info in file_list:
+                file_url = info.xpath('./@href').extract_first()
+                file_name = info.xpath('./text()').extract_first()
+                if file_name:
+                    file_name =file_name.strip()
+                    file_type = file_name.split('.')[-1].lower()
+
+                    if file_type in file_types and "file" in file_url:
+                        attachment = AD().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url,
+                            enable_proxy=False)
+                        attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_ygp/公告公示-详情页.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.tools import log
+from untils.attachment import AttachmentDownloader as AD
+from untils.tools import remove_htmldata
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item,render=True,render_time=5,
+                                          deal_detail=item.get("deal_detail"),proxies=False,
+                                          callback=eval(item.get("parse")), **request_params)
+            
+
+    def download_midware(self, request):
+        request.headers = {
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+    def detail_get(self,request,response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="rent-width "]').extract_first()
+        html = remove_htmldata(['<re>data:image/(.*?)"'],html,response)
+        list_item.contenthtml = html
+
+        file_list = response.xpath('//div[@class="rent-width "]//a[@href]')
+
+        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg',
+                      'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg', 'wps', 'ofd']
+        attachments = {}
+        if file_list:
+            for info in file_list:
+                file_url = info.xpath('./@href').extract_first()
+                file_name = info.xpath('./text()').extract_first()
+                if file_name:
+                    file_name =file_name.strip()
+                    file_type = file_name.split('.')[-1].lower()
+
+                    if file_type in file_types and "file" in file_url:
+                        attachment = AD().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url,
+                            enable_proxy=False)
+                        attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_ygp/公告公示-预挂牌-列表页.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import json
+
+
+class Feapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.site = "阳光租赁平台"
+
+        self.menus = [
+            Menu('公告公示-预挂牌', 'a_ygzlpt_gggs_ygp', '预挂牌', 1),
+        ]
+
+        self.headers = {
+            "accept": "application/json, text/javascript, */*; q=0.01",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "content-type": "application/json",
+            "origin": "https://rent.szexgrp.com",
+            "priority": "u=1, i",
+            "referer": "https://rent.szexgrp.com/page/notice/notice.html",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+            "x-requested-with": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://rent.szexgrp.com/cms/api/v1/lease/notice/page"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1,  proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "subClassifyType": "",
+            "noticeTypeName": menu.get('tid'),
+            "publishTimeStartTime": "",
+            "publishTimeEndTime": "",
+            "noticeTitle": "",
+            "statusOrder": "",
+            "orderByParam": "noticeStartTime",
+            "orderByDirection": "DESC",
+            "pageNum": page-1,
+            "pageSize": 10
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('content')
+        for info in info_list:
+            hid = info.get('projectContentId')
+            href = f'https://rent.szexgrp.com/page/projectDetail/projectDetail.html?contentId={hid}'
+            title = info.get('noticeTitle').strip()
+            create_time = info.get('publishTime')
+
+            area = "广东"      # 省份
+            city = "深圳市"    # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('title','href')
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.infoformat = 3
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="rent-width "]']
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 无限翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    Feapder(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_zhzgg/公告公示-中止公告-列表页.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import json
+
+
+class Feapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.site = "阳光租赁平台"
+
+        self.menus = [
+            Menu('公告公示-中止公告', 'a_ygzlpt_gggs_zhzgg', '中止公告', 1),
+        ]
+
+        self.headers = {
+            "accept": "application/json, text/javascript, */*; q=0.01",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "content-type": "application/json",
+            "origin": "https://rent.szexgrp.com",
+            "priority": "u=1, i",
+            "referer": "https://rent.szexgrp.com/page/notice/notice.html",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+            "x-requested-with": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://rent.szexgrp.com/cms/api/v1/lease/notice/page"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1,  proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "subClassifyType": "",
+            "noticeTypeName": menu.get('tid'),
+            "publishTimeStartTime": "",
+            "publishTimeEndTime": "",
+            "noticeTitle": "",
+            "statusOrder": "",
+            "orderByParam": "noticeStartTime",
+            "orderByDirection": "DESC",
+            "pageNum": page-1,
+            "pageSize": 10
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('content')
+        for info in info_list:
+            hid = info.get('projectContentId')
+            href = f'https://rent.szexgrp.com/page/projectDetail/projectDetail.html?contentId={hid}'
+            title = info.get('noticeTitle').strip()
+            create_time = info.get('publishTime')
+
+            area = "广东"      # 省份
+            city = "深圳市"    # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('title','href')
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.infoformat = 3
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="rent-width "]']
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 无限翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    Feapder(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_zhzgg/公告公示-详情页.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.tools import log
+from untils.attachment import AttachmentDownloader as AD
+from untils.tools import remove_htmldata
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item,render=True,render_time=5,
+                                          deal_detail=item.get("deal_detail"),proxies=False,
+                                          callback=eval(item.get("parse")), **request_params)
+            
+
+    def download_midware(self, request):
+        request.headers = {
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+    def detail_get(self,request,response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="rent-width "]').extract_first()
+        html = remove_htmldata(['<re>data:image/(.*?)"'],html,response)
+        list_item.contenthtml = html
+
+        file_list = response.xpath('//div[@class="rent-width "]//a[@href]')
+
+        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg',
+                      'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg', 'wps', 'ofd']
+        attachments = {}
+        if file_list:
+            for info in file_list:
+                file_url = info.xpath('./@href').extract_first()
+                file_name = info.xpath('./text()').extract_first()
+                if file_name:
+                    file_name =file_name.strip()
+                    file_type = file_name.split('.')[-1].lower()
+
+                    if file_type in file_types and "file" in file_url:
+                        attachment = AD().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url,
+                            enable_proxy=False)
+                        attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_zzgg/公告公示-终止公告-列表页.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+import json
+
+
+class Feapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
+
+        self.site = "阳光租赁平台"
+
+        self.menus = [
+            Menu('公告公示-终止公告', 'a_ygzlpt_gggs_zzgg', '终止公告', 1),
+        ]
+
+        self.headers = {
+            "accept": "application/json, text/javascript, */*; q=0.01",
+            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "content-type": "application/json",
+            "origin": "https://rent.szexgrp.com",
+            "priority": "u=1, i",
+            "referer": "https://rent.szexgrp.com/page/notice/notice.html",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+            "x-requested-with": "XMLHttpRequest"
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://rent.szexgrp.com/cms/api/v1/lease/notice/page"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1,  proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "subClassifyType": "",
+            "noticeTypeName": menu.get('tid'),
+            "publishTimeStartTime": "",
+            "publishTimeEndTime": "",
+            "noticeTitle": "",
+            "statusOrder": "",
+            "orderByParam": "noticeStartTime",
+            "orderByDirection": "DESC",
+            "pageNum": page-1,
+            "pageSize": 10
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.json.get('data').get('content')
+        for info in info_list:
+            hid = info.get('projectContentId')
+            href = f'https://rent.szexgrp.com/page/projectDetail/projectDetail.html?contentId={hid}'
+            title = info.get('noticeTitle').strip()
+            create_time = info.get('publishTime')
+
+            area = "广东"      # 省份
+            city = "深圳市"    # 城市
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('title','href')
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.infoformat = 3
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = ['//div[@class="rent-width "]']
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 无限翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    Feapder(redis_key="lzz:ygzlpt_gggs").start()

a_ygzlpt_gggs_zzgg/公告公示-详情页.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-23
+---------
+@summary: 阳光租赁平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.tools import log
+from untils.attachment import AttachmentDownloader as AD
+from untils.tools import remove_htmldata
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            yield feapder.Request(url=item.get("parse_url"), item=item,render=True,render_time=5,
+                                          deal_detail=item.get("deal_detail"),proxies=False,
+                                          callback=eval(item.get("parse")), **request_params)
+            
+
+    def download_midware(self, request):
+        request.headers = {
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+    def detail_get(self,request,response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="rent-width "]').extract_first()
+        html = remove_htmldata(['<re>data:image/(.*?)"'],html,response)
+        list_item.contenthtml = html
+
+        file_list = response.xpath('//div[@class="rent-width "]//a[@href]')
+
+        file_types = ['zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'jpg',
+                      'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg', 'wps', 'ofd']
+        attachments = {}
+        if file_list:
+            for info in file_list:
+                file_url = info.xpath('./@href').extract_first()
+                file_name = info.xpath('./text()').extract_first()
+                if file_name:
+                    file_name =file_name.strip()
+                    file_type = file_name.split('.')[-1].lower()
+
+                    if file_type in file_types and "file" in file_url:
+                        attachment = AD().fetch_attachment(
+                            file_name=file_name, file_type=file_type, download_url=file_url,
+                            enable_proxy=False)
+                        attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:ygzlpt_gggs").start()

a_yyc_jpcg/yyc_headers.js

@@ -0,0 +1,163 @@
+
+var jokecode = (function() {
+    var OooooOOOOOoo = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/']
+      , encode = o0o0oooOO
+      , handleFormat = {
+        'utf-8': toUTF8Binary
+    };
+    function stringToBinary(str, size, encodeType) {
+        var i, len, binary = '';
+        for (i = 0,
+        len = str.length; i < len; i++) {
+            binary = binary + handleFormat[encodeType.toLowerCase()](str.charCodeAt(i))
+        }
+        return binary
+    }
+    function toUTF8Binary(unicode) {
+        var len, binary = '', star = 0, bitStream = unicode.toString(2), bitLen = bitStream.length, i;
+        if (unicode >= 0x000000 && unicode <= 0x00007F) {
+            binary = bitStream;
+            for (i = 0,
+            len = 8; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+        } else if (unicode >= 0x000080 && unicode <= 0x0007FF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 11; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            binary = '110' + binary.substr(0, 5) + '10' + binary.substr(5, 6)
+        } else if (unicode >= 0x000800 && unicode <= 0x00FFFF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 16; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            ;binary = '1110' + binary.substr(0, 4) + '10' + binary.substr(4, 6) + '10' + binary.substr(10, 6)
+        } else if (unicode >= 0x010000 && unicode <= 0x10FFFF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 21; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            binary = '11110' + binary.substr(0, 3) + '10' + binary.substr(3, 6) + '10' + binary.substr(9, 6) + '10' + binary.substr(15, 6)
+        }
+        return binary
+    }
+    function O000o9o00O(binary24, flag) {
+        var i, len, result = '', decode;
+        if (flag == 1) {
+            for (i = 0; i < 4; i++) {
+                decode = parseInt(binary24.substr(i * 6, 6), 2);
+                result = result + OooooOOOOOoo[decode]
+            }
+        } else {
+            for (i = 0,
+            len = Math.floor(flag / 6); i < len + 1; i++) {
+                decode = parseInt(binary24.substr(i * 6, 6), 2);
+                result = result + OooooOOOOOoo[decode]
+            }
+            for (i = 0; i < 3 - len; i++) {
+                result = result + '='
+            }
+        }
+        return result
+    }
+    function o0o0oooOO(str) {
+        var i, len, rem, mer, result = '', strBinaryAry = [], binary = stringToBinary(str, 8, 'utf-8');
+        len = binary.length;
+        mer = Math.floor(len / 24);
+        rem = len % 24;
+        for (i = 0; i < mer; i++) {
+            result = result + O000o9o00O(binary.substr(i * 24, 24), 1)
+        }
+        remCode = binary.substr(len - rem, rem);
+        if (rem > 0) {
+            for (i = 0; i < 24 - rem; i++) {
+                remCode = remCode + 0
+            }
+            result = result + O000o9o00O(remCode, rem)
+        }
+        return result
+    }
+    return encode
+}
+)();
+function OooOoo0(str) {
+    var str = str.toString();
+    var i = str.length;
+    i = i - 1;
+    var result = '';
+    for (var x = i; x >= 0; x--) {
+        result += str.charAt(x)
+    }
+    return result
+}
+function Oo0OoO(str) {
+    return jokecode(str)
+}
+function oOoOoO(dataParams) {
+    var ts = new Date().getTime();
+    // var ts = 1657346648661
+    var key = OooOoo0(ts + dataParams + 'yuncai');
+    var token1 = Oo0OoO(key);
+    var token2 = OooOoo0(token1);
+    var token3 = Oo0OoO(token2);
+    return {
+        ts: ts,
+        token: token3
+    }
+}
+
+function RQ(e, t) {
+    return String(e).padStart(t, "0")
+}
+
+function rid() {
+    var e = new Date
+        , t = (0,
+        RQ)(e.getHours(), 2)
+        , n = (0,
+        RQ)(e.getMinutes(), 2)
+        , r = (0,
+        RQ)(e.getSeconds(), 2);
+    return parseInt(t + n + r).toString(32)
+}
+
+function create_href(t, e, i, n, s) {
+    let a = "/workbench/index-zh_CN.html#/ifr/" + "%2fcpu-fe-tender%2fdist%2fsupplyenlistshowother%2findex.html%3fshow%3d1%26id%3d"
+    let r = "/cpu-lawbid-fe/portalcas.html#/pages/bid_section/biddingDetail"
+    let o = "/workbench/index-zh_CN.html#/ifr/" + "%252fcpu-fe-bid%252fdist%252fsubmitbidnotice%252findex.html%253ftype%253d3%2526id%253d";
+
+    switch (e) {
+        case "0":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fprice%252fbuyOfferDetail%253fid%253d" + t;
+        case "1":
+            return "/workbench/index-zh_CN.html#/ifr/%2fcpu-fe-bid%2fdist%2fbidddocread%2findex.html%3ftype%3d1%26id%3d" + t;
+        case "4":
+        case "6":
+        case "11":
+        case "12":
+            return "lawnotice" == n ? r + "?ispur=false&from=hnfz&endTime=" + s + "&purNoticeId=" + t : a + t;
+        case "5":
+        case "7":
+        case "13":
+        case "14":
+            return o + t;
+        case "8":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidtrade-fe%252fbidtradesupdetail%252findex.zh_CN.html%253ftype%253d1%2526id%253d" + t;
+        case "10":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fpricedecision%252fpricedecisionnotice_publish%253fid%253d" + t;
+        case "15":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidauction-fe%252fbidding_Apply%252findex.zh_CN.html%253ffrom%253dnfsn%2526id%253d" + t;
+        case "17":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidauction-fe%252fbidNotificationPublic%252findex.zh_CN.html%253fid%253d" + t;
+        case "18":
+            return "/cpu-fe-bid/dist/lbnotice/index.html?id=" + t + "&enterpriseId=" + i;
+        case "19":
+            return "/cpu-fe-bid/dist/clarifynotice/index.html?id=" + t + "&enterpriseId=" + i;
+        default:
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fhome%252fnewestprojectmore%253fstatus%253d0"
+    }
+}

a_yyc_jpcg/中标公告-详情页.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-19
+---------
+@summary: 友云采
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.network.selector import Selector
+from untils.tools import remove_htmldata
+import time
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            timeout = request_params.pop('timeout', 30)
+            yield feapder.Request(url=item.get("parse_url"),item=item,
+                              deal_detail=item.get("deal_detail"),**request_params,timeout=timeout,
+                              callback=eval(item.get("parse")),render=True,
+                              render_time=item.get("render_time",5))
+
+    def detail_get(self,request,response):
+
+        driver = response.browser
+        items = request.item
+        list_item = DataBakItem(**items)
+        xpath_list = ['//div[@class="minW"]','//div[@class="minW index_body__6hbOr"]']
+        html = ""
+        for xpath in xpath_list:
+            html = response.xpath(xpath).extract_first("")  # 标书详细内容
+            if html:
+                break
+        try:
+            driver.switch_to.frame(0)
+            time.sleep(3)
+            notice_html = Selector(driver.page_source).xpath('//body').extract_first("")
+        except:
+            notice_html = ""
+
+        rm_list = ['//div[contains(@class,"index_zhongbiaoheadtop_")]', '//span[@class="pointer fcsb"]',
+                   '//div[@class="index_right__v3vAu"]','//ul[@class="index_recommend__kXvr7"]']
+        html = remove_htmldata(rm_list, html, response)
+
+        list_item.contenthtml = html + notice_html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:yyc_zhbgg").start()

a_yyc_jpcg/竞拍采购-列表页.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-19
+---------
+@summary: 友云采
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+import execjs
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "友云采"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('竞拍采购', 'a_yyc_jpcg', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Connection": "keep-alive",
+            "Referer": "https://yc.yonyoucloud.com/opportunity/opportunity?type=1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://yc.yonyoucloud.com/trade-opportunity/logout/opportunity/queryNoLoginOpportunityList"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+
+    def download_midware(self, request):
+
+        with open('./yyc_headers.js','r') as fr:
+            ex_js = fr.read()
+        ctx = execjs.compile(ex_js)
+        rid = ctx.call('rid')
+        page = request.page
+        params = {
+            "comprehensive": "1",
+            "type": "4",
+            "pageSize": "20",
+            "pageIndex": f"{page}",
+            "_rid": f"{rid}"
+        }
+
+        request.headers = self.headers
+        request.params = params
+
+    def parse(self, request, response):
+
+        menu = request.item
+        info_list = response.json.get('data').get('list')
+        for info in info_list:
+            hid = info.get('idStr')
+            href = f"https://yc.yonyoucloud.com/opportunity/buyOfferDetail/{hid}"
+            title = info.get('billName').strip()
+            publish_time = timestamp_to_date(int(str(info.get('publishTime'))[:10]))
+
+            area = "全国"  # 省份
+            city = ""  # 城市
+            district = ""  # 区县
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = publish_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area or "全国"  # 省份 默认:全国
+            list_item.city = city  # 城市 默认 为空
+            list_item.district = district  # 区县 默认 为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = []  # 抽取正文xpath
+            list_item.proxies = False
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        # 翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:yyc_zhbgg").start()

a_yyc_jzxtp/yyc_headers.js

@@ -0,0 +1,163 @@
+
+var jokecode = (function() {
+    var OooooOOOOOoo = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/']
+      , encode = o0o0oooOO
+      , handleFormat = {
+        'utf-8': toUTF8Binary
+    };
+    function stringToBinary(str, size, encodeType) {
+        var i, len, binary = '';
+        for (i = 0,
+        len = str.length; i < len; i++) {
+            binary = binary + handleFormat[encodeType.toLowerCase()](str.charCodeAt(i))
+        }
+        return binary
+    }
+    function toUTF8Binary(unicode) {
+        var len, binary = '', star = 0, bitStream = unicode.toString(2), bitLen = bitStream.length, i;
+        if (unicode >= 0x000000 && unicode <= 0x00007F) {
+            binary = bitStream;
+            for (i = 0,
+            len = 8; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+        } else if (unicode >= 0x000080 && unicode <= 0x0007FF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 11; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            binary = '110' + binary.substr(0, 5) + '10' + binary.substr(5, 6)
+        } else if (unicode >= 0x000800 && unicode <= 0x00FFFF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 16; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            ;binary = '1110' + binary.substr(0, 4) + '10' + binary.substr(4, 6) + '10' + binary.substr(10, 6)
+        } else if (unicode >= 0x010000 && unicode <= 0x10FFFF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 21; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            binary = '11110' + binary.substr(0, 3) + '10' + binary.substr(3, 6) + '10' + binary.substr(9, 6) + '10' + binary.substr(15, 6)
+        }
+        return binary
+    }
+    function O000o9o00O(binary24, flag) {
+        var i, len, result = '', decode;
+        if (flag == 1) {
+            for (i = 0; i < 4; i++) {
+                decode = parseInt(binary24.substr(i * 6, 6), 2);
+                result = result + OooooOOOOOoo[decode]
+            }
+        } else {
+            for (i = 0,
+            len = Math.floor(flag / 6); i < len + 1; i++) {
+                decode = parseInt(binary24.substr(i * 6, 6), 2);
+                result = result + OooooOOOOOoo[decode]
+            }
+            for (i = 0; i < 3 - len; i++) {
+                result = result + '='
+            }
+        }
+        return result
+    }
+    function o0o0oooOO(str) {
+        var i, len, rem, mer, result = '', strBinaryAry = [], binary = stringToBinary(str, 8, 'utf-8');
+        len = binary.length;
+        mer = Math.floor(len / 24);
+        rem = len % 24;
+        for (i = 0; i < mer; i++) {
+            result = result + O000o9o00O(binary.substr(i * 24, 24), 1)
+        }
+        remCode = binary.substr(len - rem, rem);
+        if (rem > 0) {
+            for (i = 0; i < 24 - rem; i++) {
+                remCode = remCode + 0
+            }
+            result = result + O000o9o00O(remCode, rem)
+        }
+        return result
+    }
+    return encode
+}
+)();
+function OooOoo0(str) {
+    var str = str.toString();
+    var i = str.length;
+    i = i - 1;
+    var result = '';
+    for (var x = i; x >= 0; x--) {
+        result += str.charAt(x)
+    }
+    return result
+}
+function Oo0OoO(str) {
+    return jokecode(str)
+}
+function oOoOoO(dataParams) {
+    var ts = new Date().getTime();
+    // var ts = 1657346648661
+    var key = OooOoo0(ts + dataParams + 'yuncai');
+    var token1 = Oo0OoO(key);
+    var token2 = OooOoo0(token1);
+    var token3 = Oo0OoO(token2);
+    return {
+        ts: ts,
+        token: token3
+    }
+}
+
+function RQ(e, t) {
+    return String(e).padStart(t, "0")
+}
+
+function rid() {
+    var e = new Date
+        , t = (0,
+        RQ)(e.getHours(), 2)
+        , n = (0,
+        RQ)(e.getMinutes(), 2)
+        , r = (0,
+        RQ)(e.getSeconds(), 2);
+    return parseInt(t + n + r).toString(32)
+}
+
+function create_href(t, e, i, n, s) {
+    let a = "/workbench/index-zh_CN.html#/ifr/" + "%2fcpu-fe-tender%2fdist%2fsupplyenlistshowother%2findex.html%3fshow%3d1%26id%3d"
+    let r = "/cpu-lawbid-fe/portalcas.html#/pages/bid_section/biddingDetail"
+    let o = "/workbench/index-zh_CN.html#/ifr/" + "%252fcpu-fe-bid%252fdist%252fsubmitbidnotice%252findex.html%253ftype%253d3%2526id%253d";
+
+    switch (e) {
+        case "0":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fprice%252fbuyOfferDetail%253fid%253d" + t;
+        case "1":
+            return "/workbench/index-zh_CN.html#/ifr/%2fcpu-fe-bid%2fdist%2fbidddocread%2findex.html%3ftype%3d1%26id%3d" + t;
+        case "4":
+        case "6":
+        case "11":
+        case "12":
+            return "lawnotice" == n ? r + "?ispur=false&from=hnfz&endTime=" + s + "&purNoticeId=" + t : a + t;
+        case "5":
+        case "7":
+        case "13":
+        case "14":
+            return o + t;
+        case "8":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidtrade-fe%252fbidtradesupdetail%252findex.zh_CN.html%253ftype%253d1%2526id%253d" + t;
+        case "10":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fpricedecision%252fpricedecisionnotice_publish%253fid%253d" + t;
+        case "15":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidauction-fe%252fbidding_Apply%252findex.zh_CN.html%253ffrom%253dnfsn%2526id%253d" + t;
+        case "17":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidauction-fe%252fbidNotificationPublic%252findex.zh_CN.html%253fid%253d" + t;
+        case "18":
+            return "/cpu-fe-bid/dist/lbnotice/index.html?id=" + t + "&enterpriseId=" + i;
+        case "19":
+            return "/cpu-fe-bid/dist/clarifynotice/index.html?id=" + t + "&enterpriseId=" + i;
+        default:
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fhome%252fnewestprojectmore%253fstatus%253d0"
+    }
+}

a_yyc_jzxtp/中标公告-详情页.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-19
+---------
+@summary: 友云采
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.network.selector import Selector
+from untils.tools import remove_htmldata
+import time
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            timeout = request_params.pop('timeout', 30)
+            yield feapder.Request(url=item.get("parse_url"),item=item,
+                              deal_detail=item.get("deal_detail"),**request_params,timeout=timeout,
+                              callback=eval(item.get("parse")),render=True,
+                              render_time=item.get("render_time",5))
+
+    def detail_get(self,request,response):
+
+        driver = response.browser
+        items = request.item
+        list_item = DataBakItem(**items)
+        xpath_list = ['//div[@class="minW"]','//div[@class="minW index_body__6hbOr"]']
+        html = ""
+        for xpath in xpath_list:
+            html = response.xpath(xpath).extract_first("")  # 标书详细内容
+            if html:
+                break
+        try:
+            driver.switch_to.frame(0)
+            time.sleep(3)
+            notice_html = Selector(driver.page_source).xpath('//body').extract_first("")
+        except:
+            notice_html = ""
+
+        rm_list = ['//div[contains(@class,"index_zhongbiaoheadtop_")]', '//span[@class="pointer fcsb"]',
+                   '//div[@class="index_right__v3vAu"]','//ul[@class="index_recommend__kXvr7"]']
+        html = remove_htmldata(rm_list, html, response)
+
+        list_item.contenthtml = html + notice_html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:yyc_zhbgg").start()

a_yyc_jzxtp/竞争性谈判-列表页.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-19
+---------
+@summary: 友云采
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+import execjs
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "友云采"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('竞争性谈判', 'a_yyc_jzxtp', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Connection": "keep-alive",
+            "Referer": "https://yc.yonyoucloud.com/opportunity/opportunity?type=1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://yc.yonyoucloud.com/trade-opportunity/logout/opportunity/queryNoLoginOpportunityList"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+
+    def download_midware(self, request):
+
+        with open('./yyc_headers.js','r') as fr:
+            ex_js = fr.read()
+        ctx = execjs.compile(ex_js)
+        rid = ctx.call('rid')
+        page = request.page
+        params = {
+            "comprehensive": "1",
+            "type": "7",
+            "pageSize": "20",
+            "pageIndex": f"{page}",
+            "_rid": f"{rid}"
+        }
+
+        request.headers = self.headers
+        request.params = params
+
+    def parse(self, request, response):
+
+        menu = request.item
+        info_list = response.json.get('data').get('list')
+        for info in info_list:
+            hid = info.get('idStr')
+            href = f"https://yc.yonyoucloud.com/opportunity/buyOfferDetail/{hid}"
+            title = info.get('billName').strip()
+            publish_time = timestamp_to_date(int(str(info.get('publishTime'))[:10]))
+
+            area = "全国"  # 省份
+            city = ""  # 城市
+            district = ""  # 区县
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = publish_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area or "全国"  # 省份 默认:全国
+            list_item.city = city  # 城市 默认 为空
+            list_item.district = district  # 区县 默认 为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = []  # 抽取正文xpath
+            list_item.proxies = False
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        # 翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:yyc_zhbgg").start()

a_yyc_xjcg/yyc_headers.js

@@ -0,0 +1,163 @@
+
+var jokecode = (function() {
+    var OooooOOOOOoo = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/']
+      , encode = o0o0oooOO
+      , handleFormat = {
+        'utf-8': toUTF8Binary
+    };
+    function stringToBinary(str, size, encodeType) {
+        var i, len, binary = '';
+        for (i = 0,
+        len = str.length; i < len; i++) {
+            binary = binary + handleFormat[encodeType.toLowerCase()](str.charCodeAt(i))
+        }
+        return binary
+    }
+    function toUTF8Binary(unicode) {
+        var len, binary = '', star = 0, bitStream = unicode.toString(2), bitLen = bitStream.length, i;
+        if (unicode >= 0x000000 && unicode <= 0x00007F) {
+            binary = bitStream;
+            for (i = 0,
+            len = 8; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+        } else if (unicode >= 0x000080 && unicode <= 0x0007FF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 11; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            binary = '110' + binary.substr(0, 5) + '10' + binary.substr(5, 6)
+        } else if (unicode >= 0x000800 && unicode <= 0x00FFFF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 16; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            ;binary = '1110' + binary.substr(0, 4) + '10' + binary.substr(4, 6) + '10' + binary.substr(10, 6)
+        } else if (unicode >= 0x010000 && unicode <= 0x10FFFF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 21; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            binary = '11110' + binary.substr(0, 3) + '10' + binary.substr(3, 6) + '10' + binary.substr(9, 6) + '10' + binary.substr(15, 6)
+        }
+        return binary
+    }
+    function O000o9o00O(binary24, flag) {
+        var i, len, result = '', decode;
+        if (flag == 1) {
+            for (i = 0; i < 4; i++) {
+                decode = parseInt(binary24.substr(i * 6, 6), 2);
+                result = result + OooooOOOOOoo[decode]
+            }
+        } else {
+            for (i = 0,
+            len = Math.floor(flag / 6); i < len + 1; i++) {
+                decode = parseInt(binary24.substr(i * 6, 6), 2);
+                result = result + OooooOOOOOoo[decode]
+            }
+            for (i = 0; i < 3 - len; i++) {
+                result = result + '='
+            }
+        }
+        return result
+    }
+    function o0o0oooOO(str) {
+        var i, len, rem, mer, result = '', strBinaryAry = [], binary = stringToBinary(str, 8, 'utf-8');
+        len = binary.length;
+        mer = Math.floor(len / 24);
+        rem = len % 24;
+        for (i = 0; i < mer; i++) {
+            result = result + O000o9o00O(binary.substr(i * 24, 24), 1)
+        }
+        remCode = binary.substr(len - rem, rem);
+        if (rem > 0) {
+            for (i = 0; i < 24 - rem; i++) {
+                remCode = remCode + 0
+            }
+            result = result + O000o9o00O(remCode, rem)
+        }
+        return result
+    }
+    return encode
+}
+)();
+function OooOoo0(str) {
+    var str = str.toString();
+    var i = str.length;
+    i = i - 1;
+    var result = '';
+    for (var x = i; x >= 0; x--) {
+        result += str.charAt(x)
+    }
+    return result
+}
+function Oo0OoO(str) {
+    return jokecode(str)
+}
+function oOoOoO(dataParams) {
+    var ts = new Date().getTime();
+    // var ts = 1657346648661
+    var key = OooOoo0(ts + dataParams + 'yuncai');
+    var token1 = Oo0OoO(key);
+    var token2 = OooOoo0(token1);
+    var token3 = Oo0OoO(token2);
+    return {
+        ts: ts,
+        token: token3
+    }
+}
+
+function RQ(e, t) {
+    return String(e).padStart(t, "0")
+}
+
+function rid() {
+    var e = new Date
+        , t = (0,
+        RQ)(e.getHours(), 2)
+        , n = (0,
+        RQ)(e.getMinutes(), 2)
+        , r = (0,
+        RQ)(e.getSeconds(), 2);
+    return parseInt(t + n + r).toString(32)
+}
+
+function create_href(t, e, i, n, s) {
+    let a = "/workbench/index-zh_CN.html#/ifr/" + "%2fcpu-fe-tender%2fdist%2fsupplyenlistshowother%2findex.html%3fshow%3d1%26id%3d"
+    let r = "/cpu-lawbid-fe/portalcas.html#/pages/bid_section/biddingDetail"
+    let o = "/workbench/index-zh_CN.html#/ifr/" + "%252fcpu-fe-bid%252fdist%252fsubmitbidnotice%252findex.html%253ftype%253d3%2526id%253d";
+
+    switch (e) {
+        case "0":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fprice%252fbuyOfferDetail%253fid%253d" + t;
+        case "1":
+            return "/workbench/index-zh_CN.html#/ifr/%2fcpu-fe-bid%2fdist%2fbidddocread%2findex.html%3ftype%3d1%26id%3d" + t;
+        case "4":
+        case "6":
+        case "11":
+        case "12":
+            return "lawnotice" == n ? r + "?ispur=false&from=hnfz&endTime=" + s + "&purNoticeId=" + t : a + t;
+        case "5":
+        case "7":
+        case "13":
+        case "14":
+            return o + t;
+        case "8":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidtrade-fe%252fbidtradesupdetail%252findex.zh_CN.html%253ftype%253d1%2526id%253d" + t;
+        case "10":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fpricedecision%252fpricedecisionnotice_publish%253fid%253d" + t;
+        case "15":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidauction-fe%252fbidding_Apply%252findex.zh_CN.html%253ffrom%253dnfsn%2526id%253d" + t;
+        case "17":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidauction-fe%252fbidNotificationPublic%252findex.zh_CN.html%253fid%253d" + t;
+        case "18":
+            return "/cpu-fe-bid/dist/lbnotice/index.html?id=" + t + "&enterpriseId=" + i;
+        case "19":
+            return "/cpu-fe-bid/dist/clarifynotice/index.html?id=" + t + "&enterpriseId=" + i;
+        default:
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fhome%252fnewestprojectmore%253fstatus%253d0"
+    }
+}

a_yyc_xjcg/中标公告-详情页.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-19
+---------
+@summary: 友云采
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.network.selector import Selector
+from untils.tools import remove_htmldata
+import time
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            timeout = request_params.pop('timeout', 30)
+            yield feapder.Request(url=item.get("parse_url"),item=item,
+                              deal_detail=item.get("deal_detail"),**request_params,timeout=timeout,
+                              callback=eval(item.get("parse")),render=True,
+                              render_time=item.get("render_time",5))
+
+    def detail_get(self,request,response):
+
+        driver = response.browser
+        items = request.item
+        list_item = DataBakItem(**items)
+        xpath_list = ['//div[@class="minW"]','//div[@class="minW index_body__6hbOr"]']
+        html = ""
+        for xpath in xpath_list:
+            html = response.xpath(xpath).extract_first("")  # 标书详细内容
+            if html:
+                break
+        try:
+            driver.switch_to.frame(0)
+            time.sleep(3)
+            notice_html = Selector(driver.page_source).xpath('//body').extract_first("")
+        except:
+            notice_html = ""
+
+        rm_list = ['//div[contains(@class,"index_zhongbiaoheadtop_")]', '//span[@class="pointer fcsb"]',
+                   '//div[@class="index_right__v3vAu"]','//ul[@class="index_recommend__kXvr7"]']
+        html = remove_htmldata(rm_list, html, response)
+
+        list_item.contenthtml = html + notice_html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:yyc_zhbgg").start()

a_yyc_xjcg/询价采购-列表页.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-19
+---------
+@summary: 友云采
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+import execjs
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "友云采"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('询价采购', 'a_yyc_xjcg', 5),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Connection": "keep-alive",
+            "Referer": "https://yc.yonyoucloud.com/opportunity/opportunity?type=1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://yc.yonyoucloud.com/trade-opportunity/logout/opportunity/queryNoLoginOpportunityList"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+
+    def download_midware(self, request):
+
+        with open('./yyc_headers.js','r') as fr:
+            ex_js = fr.read()
+        ctx = execjs.compile(ex_js)
+        rid = ctx.call('rid')
+        page = request.page
+        params = {
+            "comprehensive": "1",
+            "type": "0",
+            "pageSize": "20",
+            "pageIndex": f"{page}",
+            "_rid": f"{rid}"
+        }
+
+        request.headers = self.headers
+        request.params = params
+
+    def parse(self, request, response):
+
+        menu = request.item
+        info_list = response.json.get('data').get('list')
+        for info in info_list:
+            hid = info.get('idStr')
+            href = f"https://yc.yonyoucloud.com/opportunity/buyOfferDetail/{hid}"
+            title = info.get('billName').strip()
+            publish_time = timestamp_to_date(int(str(info.get('publishTime'))[:10]))
+
+            area = "全国"  # 省份
+            city = ""  # 城市
+            district = ""  # 区县
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = publish_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area or "全国"  # 省份 默认:全国
+            list_item.city = city  # 城市 默认 为空
+            list_item.district = district  # 区县 默认 为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = []  # 抽取正文xpath
+            list_item.proxies = False
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        # 翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:yyc_zhbgg").start()

a_yyc_zbcg/yyc_headers.js

@@ -0,0 +1,163 @@
+
+var jokecode = (function() {
+    var OooooOOOOOoo = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/']
+      , encode = o0o0oooOO
+      , handleFormat = {
+        'utf-8': toUTF8Binary
+    };
+    function stringToBinary(str, size, encodeType) {
+        var i, len, binary = '';
+        for (i = 0,
+        len = str.length; i < len; i++) {
+            binary = binary + handleFormat[encodeType.toLowerCase()](str.charCodeAt(i))
+        }
+        return binary
+    }
+    function toUTF8Binary(unicode) {
+        var len, binary = '', star = 0, bitStream = unicode.toString(2), bitLen = bitStream.length, i;
+        if (unicode >= 0x000000 && unicode <= 0x00007F) {
+            binary = bitStream;
+            for (i = 0,
+            len = 8; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+        } else if (unicode >= 0x000080 && unicode <= 0x0007FF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 11; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            binary = '110' + binary.substr(0, 5) + '10' + binary.substr(5, 6)
+        } else if (unicode >= 0x000800 && unicode <= 0x00FFFF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 16; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            ;binary = '1110' + binary.substr(0, 4) + '10' + binary.substr(4, 6) + '10' + binary.substr(10, 6)
+        } else if (unicode >= 0x010000 && unicode <= 0x10FFFF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 21; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            binary = '11110' + binary.substr(0, 3) + '10' + binary.substr(3, 6) + '10' + binary.substr(9, 6) + '10' + binary.substr(15, 6)
+        }
+        return binary
+    }
+    function O000o9o00O(binary24, flag) {
+        var i, len, result = '', decode;
+        if (flag == 1) {
+            for (i = 0; i < 4; i++) {
+                decode = parseInt(binary24.substr(i * 6, 6), 2);
+                result = result + OooooOOOOOoo[decode]
+            }
+        } else {
+            for (i = 0,
+            len = Math.floor(flag / 6); i < len + 1; i++) {
+                decode = parseInt(binary24.substr(i * 6, 6), 2);
+                result = result + OooooOOOOOoo[decode]
+            }
+            for (i = 0; i < 3 - len; i++) {
+                result = result + '='
+            }
+        }
+        return result
+    }
+    function o0o0oooOO(str) {
+        var i, len, rem, mer, result = '', strBinaryAry = [], binary = stringToBinary(str, 8, 'utf-8');
+        len = binary.length;
+        mer = Math.floor(len / 24);
+        rem = len % 24;
+        for (i = 0; i < mer; i++) {
+            result = result + O000o9o00O(binary.substr(i * 24, 24), 1)
+        }
+        remCode = binary.substr(len - rem, rem);
+        if (rem > 0) {
+            for (i = 0; i < 24 - rem; i++) {
+                remCode = remCode + 0
+            }
+            result = result + O000o9o00O(remCode, rem)
+        }
+        return result
+    }
+    return encode
+}
+)();
+function OooOoo0(str) {
+    var str = str.toString();
+    var i = str.length;
+    i = i - 1;
+    var result = '';
+    for (var x = i; x >= 0; x--) {
+        result += str.charAt(x)
+    }
+    return result
+}
+function Oo0OoO(str) {
+    return jokecode(str)
+}
+function oOoOoO(dataParams) {
+    var ts = new Date().getTime();
+    // var ts = 1657346648661
+    var key = OooOoo0(ts + dataParams + 'yuncai');
+    var token1 = Oo0OoO(key);
+    var token2 = OooOoo0(token1);
+    var token3 = Oo0OoO(token2);
+    return {
+        ts: ts,
+        token: token3
+    }
+}
+
+function RQ(e, t) {
+    return String(e).padStart(t, "0")
+}
+
+function rid() {
+    var e = new Date
+        , t = (0,
+        RQ)(e.getHours(), 2)
+        , n = (0,
+        RQ)(e.getMinutes(), 2)
+        , r = (0,
+        RQ)(e.getSeconds(), 2);
+    return parseInt(t + n + r).toString(32)
+}
+
+function create_href(t, e, i, n, s) {
+    let a = "/workbench/index-zh_CN.html#/ifr/" + "%2fcpu-fe-tender%2fdist%2fsupplyenlistshowother%2findex.html%3fshow%3d1%26id%3d"
+    let r = "/cpu-lawbid-fe/portalcas.html#/pages/bid_section/biddingDetail"
+    let o = "/workbench/index-zh_CN.html#/ifr/" + "%252fcpu-fe-bid%252fdist%252fsubmitbidnotice%252findex.html%253ftype%253d3%2526id%253d";
+
+    switch (e) {
+        case "0":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fprice%252fbuyOfferDetail%253fid%253d" + t;
+        case "1":
+            return "/workbench/index-zh_CN.html#/ifr/%2fcpu-fe-bid%2fdist%2fbidddocread%2findex.html%3ftype%3d1%26id%3d" + t;
+        case "4":
+        case "6":
+        case "11":
+        case "12":
+            return "lawnotice" == n ? r + "?ispur=false&from=hnfz&endTime=" + s + "&purNoticeId=" + t : a + t;
+        case "5":
+        case "7":
+        case "13":
+        case "14":
+            return o + t;
+        case "8":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidtrade-fe%252fbidtradesupdetail%252findex.zh_CN.html%253ftype%253d1%2526id%253d" + t;
+        case "10":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fpricedecision%252fpricedecisionnotice_publish%253fid%253d" + t;
+        case "15":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidauction-fe%252fbidding_Apply%252findex.zh_CN.html%253ffrom%253dnfsn%2526id%253d" + t;
+        case "17":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidauction-fe%252fbidNotificationPublic%252findex.zh_CN.html%253fid%253d" + t;
+        case "18":
+            return "/cpu-fe-bid/dist/lbnotice/index.html?id=" + t + "&enterpriseId=" + i;
+        case "19":
+            return "/cpu-fe-bid/dist/clarifynotice/index.html?id=" + t + "&enterpriseId=" + i;
+        default:
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fhome%252fnewestprojectmore%253fstatus%253d0"
+    }
+}

a_yyc_zbcg/中标公告-详情页.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-19
+---------
+@summary: 友云采
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.network.selector import Selector
+from untils.tools import remove_htmldata
+import time
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            timeout = request_params.pop('timeout', 30)
+            yield feapder.Request(url=item.get("parse_url"),item=item,
+                              deal_detail=item.get("deal_detail"),**request_params,timeout=timeout,
+                              callback=eval(item.get("parse")),render=True,
+                              render_time=item.get("render_time",5))
+
+    def detail_get(self,request,response):
+
+        driver = response.browser
+        items = request.item
+        list_item = DataBakItem(**items)
+        xpath_list = ['//div[@class="minW"]','//div[@class="minW index_body__6hbOr"]']
+        html = ""
+        for xpath in xpath_list:
+            html = response.xpath(xpath).extract_first("")  # 标书详细内容
+            if html:
+                break
+        try:
+            driver.switch_to.frame(0)
+            time.sleep(3)
+            notice_html = Selector(driver.page_source).xpath('//body').extract_first("")
+        except:
+            notice_html = ""
+
+        rm_list = ['//div[contains(@class,"index_zhongbiaoheadtop_")]', '//span[@class="pointer fcsb"]',
+                   '//div[@class="index_right__v3vAu"]','//ul[@class="index_recommend__kXvr7"]']
+        html = remove_htmldata(rm_list, html, response)
+
+        list_item.contenthtml = html + notice_html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:yyc_zhbgg").start()

a_yyc_zbcg/招标采购-列表页.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-19
+---------
+@summary: 友云采
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+import execjs
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "友云采"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('招标采购', 'a_yyc_zbcg', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Connection": "keep-alive",
+            "Referer": "https://yc.yonyoucloud.com/opportunity/opportunity?type=1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://yc.yonyoucloud.com/trade-opportunity/logout/opportunity/queryNoLoginOpportunityList"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+
+    def download_midware(self, request):
+
+        with open('./yyc_headers.js','r') as fr:
+            ex_js = fr.read()
+        ctx = execjs.compile(ex_js)
+        rid = ctx.call('rid')
+        page = request.page
+        params = {
+            "comprehensive": "1",
+            "type": "1",
+            "pageSize": "20",
+            "pageIndex": f"{page}",
+            "_rid": f"{rid}"
+        }
+
+        request.headers = self.headers
+        request.params = params
+
+    def parse(self, request, response):
+
+        menu = request.item
+        info_list = response.json.get('data').get('list')
+        for info in info_list:
+            hid = info.get('idStr')
+            href = f"https://yc.yonyoucloud.com/opportunity/buyOfferDetail/{hid}"
+            title = info.get('billName').strip()
+            publish_time = timestamp_to_date(int(str(info.get('publishTime'))[:10]))
+
+            area = "全国"  # 省份
+            city = ""  # 城市
+            district = ""  # 区县
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = publish_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area or "全国"  # 省份 默认:全国
+            list_item.city = city  # 城市 默认 为空
+            list_item.district = district  # 区县 默认 为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = []  # 抽取正文xpath
+            list_item.proxies = False
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        # 翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:yyc_zhbgg").start()

a_yyc_zhbgg/yyc_headers.js

@@ -0,0 +1,163 @@
+
+var jokecode = (function() {
+    var OooooOOOOOoo = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/']
+      , encode = o0o0oooOO
+      , handleFormat = {
+        'utf-8': toUTF8Binary
+    };
+    function stringToBinary(str, size, encodeType) {
+        var i, len, binary = '';
+        for (i = 0,
+        len = str.length; i < len; i++) {
+            binary = binary + handleFormat[encodeType.toLowerCase()](str.charCodeAt(i))
+        }
+        return binary
+    }
+    function toUTF8Binary(unicode) {
+        var len, binary = '', star = 0, bitStream = unicode.toString(2), bitLen = bitStream.length, i;
+        if (unicode >= 0x000000 && unicode <= 0x00007F) {
+            binary = bitStream;
+            for (i = 0,
+            len = 8; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+        } else if (unicode >= 0x000080 && unicode <= 0x0007FF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 11; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            binary = '110' + binary.substr(0, 5) + '10' + binary.substr(5, 6)
+        } else if (unicode >= 0x000800 && unicode <= 0x00FFFF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 16; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            ;binary = '1110' + binary.substr(0, 4) + '10' + binary.substr(4, 6) + '10' + binary.substr(10, 6)
+        } else if (unicode >= 0x010000 && unicode <= 0x10FFFF) {
+            binary = bitStream;
+            for (i = 0,
+            len = 21; i < len - bitLen; i++) {
+                binary = 0 + binary;
+            }
+            binary = '11110' + binary.substr(0, 3) + '10' + binary.substr(3, 6) + '10' + binary.substr(9, 6) + '10' + binary.substr(15, 6)
+        }
+        return binary
+    }
+    function O000o9o00O(binary24, flag) {
+        var i, len, result = '', decode;
+        if (flag == 1) {
+            for (i = 0; i < 4; i++) {
+                decode = parseInt(binary24.substr(i * 6, 6), 2);
+                result = result + OooooOOOOOoo[decode]
+            }
+        } else {
+            for (i = 0,
+            len = Math.floor(flag / 6); i < len + 1; i++) {
+                decode = parseInt(binary24.substr(i * 6, 6), 2);
+                result = result + OooooOOOOOoo[decode]
+            }
+            for (i = 0; i < 3 - len; i++) {
+                result = result + '='
+            }
+        }
+        return result
+    }
+    function o0o0oooOO(str) {
+        var i, len, rem, mer, result = '', strBinaryAry = [], binary = stringToBinary(str, 8, 'utf-8');
+        len = binary.length;
+        mer = Math.floor(len / 24);
+        rem = len % 24;
+        for (i = 0; i < mer; i++) {
+            result = result + O000o9o00O(binary.substr(i * 24, 24), 1)
+        }
+        remCode = binary.substr(len - rem, rem);
+        if (rem > 0) {
+            for (i = 0; i < 24 - rem; i++) {
+                remCode = remCode + 0
+            }
+            result = result + O000o9o00O(remCode, rem)
+        }
+        return result
+    }
+    return encode
+}
+)();
+function OooOoo0(str) {
+    var str = str.toString();
+    var i = str.length;
+    i = i - 1;
+    var result = '';
+    for (var x = i; x >= 0; x--) {
+        result += str.charAt(x)
+    }
+    return result
+}
+function Oo0OoO(str) {
+    return jokecode(str)
+}
+function oOoOoO(dataParams) {
+    var ts = new Date().getTime();
+    // var ts = 1657346648661
+    var key = OooOoo0(ts + dataParams + 'yuncai');
+    var token1 = Oo0OoO(key);
+    var token2 = OooOoo0(token1);
+    var token3 = Oo0OoO(token2);
+    return {
+        ts: ts,
+        token: token3
+    }
+}
+
+function RQ(e, t) {
+    return String(e).padStart(t, "0")
+}
+
+function rid() {
+    var e = new Date
+        , t = (0,
+        RQ)(e.getHours(), 2)
+        , n = (0,
+        RQ)(e.getMinutes(), 2)
+        , r = (0,
+        RQ)(e.getSeconds(), 2);
+    return parseInt(t + n + r).toString(32)
+}
+
+function create_href(t, e, i, n, s) {
+    let a = "/workbench/index-zh_CN.html#/ifr/" + "%2fcpu-fe-tender%2fdist%2fsupplyenlistshowother%2findex.html%3fshow%3d1%26id%3d"
+    let r = "/cpu-lawbid-fe/portalcas.html#/pages/bid_section/biddingDetail"
+    let o = "/workbench/index-zh_CN.html#/ifr/" + "%252fcpu-fe-bid%252fdist%252fsubmitbidnotice%252findex.html%253ftype%253d3%2526id%253d";
+
+    switch (e) {
+        case "0":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fprice%252fbuyOfferDetail%253fid%253d" + t;
+        case "1":
+            return "/workbench/index-zh_CN.html#/ifr/%2fcpu-fe-bid%2fdist%2fbidddocread%2findex.html%3ftype%3d1%26id%3d" + t;
+        case "4":
+        case "6":
+        case "11":
+        case "12":
+            return "lawnotice" == n ? r + "?ispur=false&from=hnfz&endTime=" + s + "&purNoticeId=" + t : a + t;
+        case "5":
+        case "7":
+        case "13":
+        case "14":
+            return o + t;
+        case "8":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidtrade-fe%252fbidtradesupdetail%252findex.zh_CN.html%253ftype%253d1%2526id%253d" + t;
+        case "10":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fpricedecision%252fpricedecisionnotice_publish%253fid%253d" + t;
+        case "15":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidauction-fe%252fbidding_Apply%252findex.zh_CN.html%253ffrom%253dnfsn%2526id%253d" + t;
+        case "17":
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-bidauction-fe%252fbidNotificationPublic%252findex.zh_CN.html%253fid%253d" + t;
+        case "18":
+            return "/cpu-fe-bid/dist/lbnotice/index.html?id=" + t + "&enterpriseId=" + i;
+        case "19":
+            return "/cpu-fe-bid/dist/clarifynotice/index.html?id=" + t + "&enterpriseId=" + i;
+        default:
+            return "/workbench/index-zh_CN.html#/ifr/%252fcpu-portal-fe%252fportalcas.html%2523%252fpages%252fhome%252fnewestprojectmore%253fstatus%253d0"
+    }
+}

a_yyc_zhbgg/中标公告-列表页.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-19
+---------
+@summary: 友云采
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+from feapder.utils.tools import timestamp_to_date
+import execjs
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "友云采"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('中标公告', 'a_yyc_zhbgg', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "https://yc.yonyoucloud.com/opportunity/winningNotice",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+        }
+
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://yc.yonyoucloud.com/trade-opportunity/logout/opportunity/queryNoLoginWinBidList"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+
+    def download_midware(self, request):
+
+        with open('./yyc_headers.js','r') as fr:
+            ex_js = fr.read()
+        ctx = execjs.compile(ex_js)
+        rid = ctx.call('rid')
+        page = request.page
+        params = {
+            "comprehensive": "1",
+            "pageSize": "20",
+            "pageIndex": f"{page}",
+            "_rid": f"{rid}"
+        }
+
+        request.headers = self.headers
+        request.params = params
+
+    def parse(self, request, response):
+
+        menu = request.item
+        info_list = response.json.get('data').get('list')
+        for info in info_list:
+            hid = info.get('idStr')
+            channelType = info.get('channelType')
+            if channelType == 'yc':
+                lid = 'winningNotice/noticeDetail'
+            else:
+                lid = 'buyOfferDetail'
+            href = f"https://yc.yonyoucloud.com/opportunity/{lid}/{hid}"
+            title = info.get('billName').strip()
+            publish_time = timestamp_to_date(int(str(info.get('publishTime'))[:10]))
+
+            area = "全国"  # 省份
+            city = ""  # 城市
+            district = ""  # 区县
+
+            list_item = BidingListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = publish_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area or "全国"  # 省份 默认:全国
+            list_item.city = city  # 城市 默认 为空
+            list_item.district = district  # 区县 默认 为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.deal_detail = []  # 抽取正文xpath
+            list_item.proxies = False
+            list_item.parse_url = href  # 详情页请求地址
+
+            yield list_item
+
+        # 翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:yyc_zhbgg").start()

a_yyc_zhbgg/中标公告-详情页.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-19
+---------
+@summary: 友云采
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from feapder.network.selector import Selector
+from untils.tools import remove_htmldata
+import time
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_list = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_list:
+            request_params = item.get("request_params")
+            timeout = request_params.pop('timeout', 30)
+            yield feapder.Request(url=item.get("parse_url"),item=item,
+                              deal_detail=item.get("deal_detail"),**request_params,timeout=timeout,
+                              callback=eval(item.get("parse")),render=True,
+                              render_time=item.get("render_time",5))
+
+    def detail_get(self,request,response):
+
+        driver = response.browser
+        items = request.item
+        list_item = DataBakItem(**items)
+        xpath_list = ['//div[@class="minW"]','//div[@class="minW index_body__6hbOr"]']
+        html = ""
+        for xpath in xpath_list:
+            html = response.xpath(xpath).extract_first("")  # 标书详细内容
+            if html:
+                break
+        try:
+            driver.switch_to.frame(0)
+            time.sleep(3)
+            notice_html = Selector(driver.page_source).xpath('//body').extract_first("")
+        except:
+            notice_html = ""
+
+        rm_list = ['//div[contains(@class,"index_zhongbiaoheadtop_")]', '//span[@class="pointer fcsb"]',
+                   '//div[@class="index_right__v3vAu"]','//ul[@class="index_recommend__kXvr7"]']
+        html = remove_htmldata(rm_list, html, response)
+
+        list_item.contenthtml = html + notice_html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:yyc_zhbgg").start()

a_zgbqdzzbtbjypt_bggg/全部公告-详情页.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-21
+---------
+@summary: 中国兵器电子招标投标交易平台
+---------
+@author: lzz
+"""
+import re
+
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.log import log
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+headers = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "Accept-Language": "zh-CN,zh;q=0.9",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+}
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_lsit:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="zbztb_ggcont pt30"]').extract_first("")
+
+        detail_url = response.xpath('//iframe[@id="pdfIframe"]/@src').extract_first("").strip()
+        file_url = "".join(re.findall("file=(.*)",detail_url))
+
+        attachments = {}
+        if file_url:
+            file_name = list_item['title']
+            file_type = extract_file_type(file_name, file_url)
+
+            if file_type:
+                attachment = AttachmentDownloader().fetch_attachment(
+                    file_name=file_name, file_type=file_type, download_url=file_url)
+                attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        if not html and file_url:
+            html = "详情请访问原网页！"
+
+        list_item.contenthtml = html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgbqdzzbtbjypt_zgysgg").start()

a_zgbqdzzbtbjypt_bggg/变更终止公告-列表页.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-21
+---------
+@summary: 中国兵器电子招标投标交易平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "中国兵器电子招标投标交易平台"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('变更终止公告', 'a_zgbqdzzbtbjypt_bggg', 2),
+        ]
+
+        self.headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "max-age=0",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://bid.norincogroup-ebuy.com",
+            "Referer": "https://bid.norincogroup-ebuy.com/retrieve.do",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://bid.norincogroup-ebuy.com/retrieve.do"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+
+    def download_midware(self, request):
+        page = request.page
+        data = {
+            "fl": "",
+            "hy": "",
+            "dq": "",
+            "es": "1",
+            "keyFlag": "",
+            "packtype": "",
+            "packtypeCode": "",
+            "packtypeValue": "",
+            "packtypeCodeValue": "",
+            "typflag": "3,4,15,16",
+            "fbdays": "0",
+            "esly": "",
+            "validityPeriodFlag": "",
+            "flag1": "",
+            "orderby": "1",
+            "keyConValue": "",
+            "keyCon": "",
+            "fbDateStart": "",
+            "fbDateEnd": "",
+            "radio": "on",
+            "pageNumber": f"{page}",
+            "pageSize": "10",
+            "sortColumns": "undefined"
+        }
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.xpath('//div[@class="zbztbn_searchlist mt15 seachList"]/div[@class="zbztbn_bgbox"]/div')
+        for info in info_list:
+            href = info.xpath('.//p[@class="sldivTitlep"]/a/@href').extract_first()
+            title = info.xpath('.//p[@class="sldivTitlep"]/a/@title').extract_first("").strip()
+            publish_time = info.xpath('.//span[@class="date"]/text()').extract_first("")
+            publish_time = publish_time.replace('发布日期:','').strip()
+
+            area = "全国"  # 省份
+            city = ""  # 城市
+            district = ""  # 区县
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = publish_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area or "全国"  # 省份 默认:全国
+            list_item.city = city  # 城市 默认 为空
+            list_item.district = district  # 区县 默认 为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.request_params = {"headers": self.headers}
+            list_item.deal_detail = []  # 抽取正文xpath
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgbqdzzbtbjypt_zgysgg").start()

a_zgbqdzzbtbjypt_zbhxrgs/中标候选人公示-列表页.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-21
+---------
+@summary: 中国兵器电子招标投标交易平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "中国兵器电子招标投标交易平台"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('中标候选人公示', 'a_zgbqdzzbtbjypt_zbhxrgs', 2),
+        ]
+
+        self.headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "max-age=0",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://bid.norincogroup-ebuy.com",
+            "Referer": "https://bid.norincogroup-ebuy.com/retrieve.do",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://bid.norincogroup-ebuy.com/retrieve.do"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+
+    def download_midware(self, request):
+        page = request.page
+        data = {
+            "fl": "",
+            "hy": "",
+            "dq": "",
+            "es": "1",
+            "keyFlag": "",
+            "packtype": "",
+            "packtypeCode": "",
+            "packtypeValue": "",
+            "packtypeCodeValue": "",
+            "typflag": "5",
+            "fbdays": "0",
+            "esly": "",
+            "validityPeriodFlag": "",
+            "flag1": "",
+            "orderby": "1",
+            "keyConValue": "",
+            "keyCon": "",
+            "fbDateStart": "",
+            "fbDateEnd": "",
+            "radio": "on",
+            "pageNumber": f"{page}",
+            "pageSize": "10",
+            "sortColumns": "undefined"
+        }
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.xpath('//div[@class="zbztbn_searchlist mt15 seachList"]/div[@class="zbztbn_bgbox"]/div')
+        for info in info_list:
+            href = info.xpath('.//p[@class="sldivTitlep"]/a/@href').extract_first()
+            title = info.xpath('.//p[@class="sldivTitlep"]/a/@title').extract_first("").strip()
+            publish_time = info.xpath('.//span[@class="date"]/text()').extract_first("")
+            publish_time = publish_time.replace('发布日期:','').strip()
+
+            area = "全国"  # 省份
+            city = ""  # 城市
+            district = ""  # 区县
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = publish_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area or "全国"  # 省份 默认:全国
+            list_item.city = city  # 城市 默认 为空
+            list_item.district = district  # 区县 默认 为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.request_params = {"headers": self.headers}
+            list_item.deal_detail = []  # 抽取正文xpath
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgbqdzzbtbjypt_zgysgg").start()

a_zgbqdzzbtbjypt_zbhxrgs/全部公告-详情页.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-21
+---------
+@summary: 中国兵器电子招标投标交易平台
+---------
+@author: lzz
+"""
+import re
+
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.log import log
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+headers = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "Accept-Language": "zh-CN,zh;q=0.9",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+}
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_lsit:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="zbztb_ggcont pt30"]').extract_first("")
+
+        detail_url = response.xpath('//iframe[@id="pdfIframe"]/@src').extract_first("").strip()
+        file_url = "".join(re.findall("file=(.*)",detail_url))
+
+        attachments = {}
+        if file_url:
+            file_name = list_item['title']
+            file_type = extract_file_type(file_name, file_url)
+
+            if file_type:
+                attachment = AttachmentDownloader().fetch_attachment(
+                    file_name=file_name, file_type=file_type, download_url=file_url)
+                attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        if not html and file_url:
+            html = "详情请访问原网页！"
+
+        list_item.contenthtml = html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgbqdzzbtbjypt_zgysgg").start()

a_zgbqdzzbtbjypt_zbjggg/中标结果公告-列表页.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-21
+---------
+@summary: 中国兵器电子招标投标交易平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "中国兵器电子招标投标交易平台"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('中标结果公告', 'a_zgbqdzzbtbjypt_zbjggg', 2),
+        ]
+
+        self.headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "max-age=0",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://bid.norincogroup-ebuy.com",
+            "Referer": "https://bid.norincogroup-ebuy.com/retrieve.do",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://bid.norincogroup-ebuy.com/retrieve.do"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
+
+    def download_midware(self, request):
+        page = request.page
+        data = {
+            "fl": "",
+            "hy": "",
+            "dq": "",
+            "es": "1",
+            "keyFlag": "",
+            "packtype": "",
+            "packtypeCode": "",
+            "packtypeValue": "",
+            "packtypeCodeValue": "",
+            "typflag": "6",
+            "fbdays": "0",
+            "esly": "",
+            "validityPeriodFlag": "",
+            "flag1": "",
+            "orderby": "1",
+            "keyConValue": "",
+            "keyCon": "",
+            "fbDateStart": "",
+            "fbDateEnd": "",
+            "radio": "on",
+            "pageNumber": f"{page}",
+            "pageSize": "10",
+            "sortColumns": "undefined"
+        }
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.xpath('//div[@class="zbztbn_searchlist mt15 seachList"]/div[@class="zbztbn_bgbox"]/div')
+        for info in info_list:
+            href = info.xpath('.//p[@class="sldivTitlep"]/a/@href').extract_first()
+            title = info.xpath('.//p[@class="sldivTitlep"]/a/@title').extract_first("").strip()
+            publish_time = info.xpath('.//span[@class="date"]/text()').extract_first("")
+            publish_time = publish_time.replace('发布日期:','').strip()
+
+            area = "全国"  # 省份
+            city = ""  # 城市
+            district = ""  # 区县
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = publish_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area or "全国"  # 省份 默认:全国
+            list_item.city = city  # 城市 默认 为空
+            list_item.district = district  # 区县 默认 为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.request_params = {"headers": self.headers}
+            list_item.deal_detail = []  # 抽取正文xpath
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgbqdzzbtbjypt_zgysgg").start()

a_zgbqdzzbtbjypt_zbjggg/全部公告-详情页.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-21
+---------
+@summary: 中国兵器电子招标投标交易平台
+---------
+@author: lzz
+"""
+import re
+
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.log import log
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+headers = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "Accept-Language": "zh-CN,zh;q=0.9",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+}
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_lsit:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="zbztb_ggcont pt30"]').extract_first("")
+
+        detail_url = response.xpath('//iframe[@id="pdfIframe"]/@src').extract_first("").strip()
+        file_url = "".join(re.findall("file=(.*)",detail_url))
+
+        attachments = {}
+        if file_url:
+            file_name = list_item['title']
+            file_type = extract_file_type(file_name, file_url)
+
+            if file_type:
+                attachment = AttachmentDownloader().fetch_attachment(
+                    file_name=file_name, file_type=file_type, download_url=file_url)
+                attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        if not html and file_url:
+            html = "详情请访问原网页！"
+
+        list_item.contenthtml = html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgbqdzzbtbjypt_zgysgg").start()

a_zgbqdzzbtbjypt_zgysgg/全部公告-详情页.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-21
+---------
+@summary: 中国兵器电子招标投标交易平台
+---------
+@author: lzz
+"""
+import re
+
+import feapder
+from items.spider_item import DataBakItem
+from feapder.utils.log import log
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+headers = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "Accept-Language": "zh-CN,zh;q=0.9",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+}
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=20)
+        for item in data_lsit:
+            # log.debug(item)
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="zbztb_ggcont pt30"]').extract_first("")
+
+        detail_url = response.xpath('//iframe[@id="pdfIframe"]/@src').extract_first("").strip()
+        file_url = "".join(re.findall("file=(.*)",detail_url))
+
+        attachments = {}
+        if file_url:
+            file_name = list_item['title']
+            file_type = extract_file_type(file_name, file_url)
+
+            if file_type:
+                attachment = AttachmentDownloader().fetch_attachment(
+                    file_name=file_name, file_type=file_type, download_url=file_url)
+                attachments[str(len(attachments) + 1)] = attachment
+
+        if attachments:
+            list_item.projectinfo = {"attachments": attachments}
+
+        if not html and file_url:
+            html = "详情请访问原网页！"
+
+        list_item.contenthtml = html
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgbqdzzbtbjypt_zgysgg").start()

a_zgbqdzzbtbjypt_zgysgg/招标资格预审公告-列表页.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-21
+---------
+@summary: 中国兵器电子招标投标交易平台
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import MgpListItem
+from collections import namedtuple
+
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+    def start_callback(self):
+
+        self.site = "中国兵器电子招标投标交易平台"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
+
+        self.menus = [
+            Menu('招标资格预审公告', 'a_zgbqdzzbtbjypt_zgysgg', 2),
+        ]
+
+        self.headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "max-age=0",
+            "Connection": "keep-alive",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Origin": "https://bid.norincogroup-ebuy.com",
+            "Referer": "https://bid.norincogroup-ebuy.com/retrieve.do",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://bid.norincogroup-ebuy.com/retrieve.do"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        data = {
+            "fl": "",
+            "hy": "",
+            "dq": "",
+            "es": "1",
+            "keyFlag": "",
+            "packtype": "",
+            "packtypeCode": "",
+            "packtypeValue": "",
+            "packtypeCodeValue": "",
+            "typflag": "1,2",
+            "fbdays": "0",
+            "esly": "",
+            "validityPeriodFlag": "",
+            "flag1": "",
+            "orderby": "1",
+            "keyConValue": "",
+            "keyCon": "",
+            "fbDateStart": "",
+            "fbDateEnd": "",
+            "radio": "on",
+            "pageNumber": f"{page}",
+            "pageSize": "10",
+            "sortColumns": "undefined"
+        }
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        menu = request.item
+        info_list = response.xpath('//div[@class="zbztbn_searchlist mt15 seachList"]/div[@class="zbztbn_bgbox"]/div')
+        for info in info_list:
+            href = info.xpath('.//p[@class="sldivTitlep"]/a/@href').extract_first()
+            title = info.xpath('.//p[@class="sldivTitlep"]/a/@title').extract_first("").strip()
+            publish_time = info.xpath('.//span[@class="date"]/text()').extract_first("")
+            publish_time = publish_time.replace('发布日期:','').strip()
+
+            area = "全国"  # 省份
+            city = ""  # 城市
+            district = ""  # 区县
+
+            list_item = MgpListItem()  # 存储数据的管道
+            list_item.href = href  # 标书链接
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.publishtime = publish_time  # 标书发布时间
+            list_item.site = self.site
+            list_item.area = area or "全国"  # 省份 默认:全国
+            list_item.city = city  # 城市 默认 为空
+            list_item.district = district  # 区县 默认 为空
+
+            list_item.unique_key = ('href',)
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.request_params = {"headers": self.headers}
+            list_item.deal_detail = []  # 抽取正文xpath
+            list_item.proxies = False
+            list_item.parse_url = href
+
+            yield list_item
+
+        # 翻页
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgbqdzzbtbjypt_zgysgg").start()

a_zgnfhkcgzbw_cggg/其它公告-列表页.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+import json
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+
+    def start_callback(self):
+
+        self.site = "中国南方航空采购招标网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'cid', 'crawl_page'])
+
+        self.menus = [
+            Menu('其它公告', 'a_zgnfhkcgzbw_cggg', '964896148592525312', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json; charset=UTF-8",
+            "Origin": "https://csbidding.csair.cn",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://csbidding.csair.cn/cms/api/dynamicData/queryContentPage"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "pageNo": page,
+            "pageSize": 15,
+            "dto": {
+                "siteId": "747",
+                "categoryId": menu.get('cid'),
+                "agentCompanyName": "",
+                "title": "",
+                "province": "",
+                "city": "",
+                "publishDays": "",
+                "estimateTotalPriceBegin": "",
+                "estimateTotalPriceEnd": ""
+            }
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        response.encoding = response.apparent_encoding
+        menu = request.item
+        info_list = response.json.get('res').get('rows')
+        for info in info_list:
+            href = "https://csbidding.csair.cn/cms/nfhk/webfile" + info.get('url')
+            title = info.get('title').strip()
+            create_time = info.get('publishDate').replace('T',' ').split('.')[0]
+
+            area = "全国"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道  
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.proxies = False
+            list_item.deal_detail = ['//div[@class="box-bottom"]']
+            list_item.parse_url = href
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_cggg/招标采购-详情页.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=30)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="box-bottom"]').extract_first()
+        ex_html = response.xpath('//div[@class="box-bottom"]//div[contains(text(),"相关公告")]/..').extract_first()
+        if ex_html:
+            html = html.replace(ex_html,'')
+
+        list_item.contenthtml = html
+
+        attachments = {}
+
+        file_list = response.xpath('//div[@class="box-bottom"]//a')
+        if file_list:
+            for info in file_list:
+                file_name = info.xpath('./span/text()').extract_first("").strip()
+                file_url = info.xpath('./@href').extract_first("").strip()
+                file_type = extract_file_type(file_name)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+                list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_cgjg/招标采购-详情页.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=30)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="box-bottom"]').extract_first()
+        ex_html = response.xpath('//div[@class="box-bottom"]//div[contains(text(),"相关公告")]/..').extract_first()
+        if ex_html:
+            html = html.replace(ex_html,'')
+
+        list_item.contenthtml = html
+
+        attachments = {}
+
+        file_list = response.xpath('//div[@class="box-bottom"]//a')
+        if file_list:
+            for info in file_list:
+                file_name = info.xpath('./span/text()').extract_first("").strip()
+                file_url = info.xpath('./@href').extract_first("").strip()
+                file_type = extract_file_type(file_name)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+                list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_cgjg/非招标采购-采购结果-列表页.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+import json
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+
+    def start_callback(self):
+
+        self.site = "中国南方航空采购招标网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'cid', 'crawl_page'])
+
+        self.menus = [
+            Menu('非招标采购-采购结果', 'a_zgnfhkcgzbw_cgjg', '964894525724033024', 2),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json; charset=UTF-8",
+            "Origin": "https://csbidding.csair.cn",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://csbidding.csair.cn/cms/api/dynamicData/queryContentPage"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "pageNo": page,
+            "pageSize": 15,
+            "dto": {
+                "siteId": "747",
+                "categoryId": menu.get('cid'),
+                "agentCompanyName": "",
+                "title": "",
+                "province": "",
+                "city": "",
+                "publishDays": "",
+                "estimateTotalPriceBegin": "",
+                "estimateTotalPriceEnd": ""
+            }
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        response.encoding = response.apparent_encoding
+        menu = request.item
+        info_list = response.json.get('res').get('rows')
+        for info in info_list:
+            href = "https://csbidding.csair.cn/cms/nfhk/webfile" + info.get('url')
+            title = info.get('title').strip()
+            create_time = info.get('publishDate').replace('T',' ').split('.')[0]
+
+            area = "全国"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道  
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.proxies = False
+            list_item.deal_detail = ['//div[@class="box-bottom"]']
+            list_item.parse_url = href
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_fzbcg_cggg/招标采购-详情页.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=30)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="box-bottom"]').extract_first()
+        ex_html = response.xpath('//div[@class="box-bottom"]//div[contains(text(),"相关公告")]/..').extract_first()
+        if ex_html:
+            html = html.replace(ex_html,'')
+
+        list_item.contenthtml = html
+
+        attachments = {}
+
+        file_list = response.xpath('//div[@class="box-bottom"]//a')
+        if file_list:
+            for info in file_list:
+                file_name = info.xpath('./span/text()').extract_first("").strip()
+                file_url = info.xpath('./@href').extract_first("").strip()
+                file_type = extract_file_type(file_name)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+                list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_fzbcg_cggg/非招标采购-采购公告-列表页.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+import json
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+
+    def start_callback(self):
+
+        self.site = "中国南方航空采购招标网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'cid', 'crawl_page'])
+
+        self.menus = [
+            Menu('非招标采购-采购公告', 'a_zgnfhkcgzbw_fzbcg_cggg', '964893593951010816', 3),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json; charset=UTF-8",
+            "Origin": "https://csbidding.csair.cn",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://csbidding.csair.cn/cms/api/dynamicData/queryContentPage"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "pageNo": page,
+            "pageSize": 15,
+            "dto": {
+                "siteId": "747",
+                "categoryId": menu.get('cid'),
+                "agentCompanyName": "",
+                "title": "",
+                "province": "",
+                "city": "",
+                "publishDays": "",
+                "estimateTotalPriceBegin": "",
+                "estimateTotalPriceEnd": ""
+            }
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        response.encoding = response.apparent_encoding
+        menu = request.item
+        info_list = response.json.get('res').get('rows')
+        for info in info_list:
+            href = "https://csbidding.csair.cn/cms/nfhk/webfile" + info.get('url')
+            title = info.get('title').strip()
+            create_time = info.get('publishDate').replace('T',' ').split('.')[0]
+
+            area = "全国"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道  
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.proxies = False
+            list_item.deal_detail = ['//div[@class="box-bottom"]']
+            list_item.parse_url = href
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_pbgs/招标采购-详情页.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=30)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="box-bottom"]').extract_first()
+        ex_html = response.xpath('//div[@class="box-bottom"]//div[contains(text(),"相关公告")]/..').extract_first()
+        if ex_html:
+            html = html.replace(ex_html,'')
+
+        list_item.contenthtml = html
+
+        attachments = {}
+
+        file_list = response.xpath('//div[@class="box-bottom"]//a')
+        if file_list:
+            for info in file_list:
+                file_name = info.xpath('./span/text()').extract_first("").strip()
+                file_url = info.xpath('./@href').extract_first("").strip()
+                file_type = extract_file_type(file_name)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+                list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_pbgs/评标公示-列表页.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+import json
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+
+    def start_callback(self):
+
+        self.site = "中国南方航空采购招标网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'cid', 'crawl_page'])
+
+        self.menus = [
+            Menu('评标公示', 'a_zgnfhkcgzbw_pbgs', '964895309316489216', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json; charset=UTF-8",
+            "Origin": "https://csbidding.csair.cn",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://csbidding.csair.cn/cms/api/dynamicData/queryContentPage"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "pageNo": page,
+            "pageSize": 15,
+            "dto": {
+                "siteId": "747",
+                "categoryId": menu.get('cid'),
+                "agentCompanyName": "",
+                "title": "",
+                "province": "",
+                "city": "",
+                "publishDays": "",
+                "estimateTotalPriceBegin": "",
+                "estimateTotalPriceEnd": ""
+            }
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        response.encoding = response.apparent_encoding
+        menu = request.item
+        info_list = response.json.get('res').get('rows')
+        for info in info_list:
+            href = "https://csbidding.csair.cn/cms/nfhk/webfile" + info.get('url')
+            title = info.get('title').strip()
+            create_time = info.get('publishDate').replace('T',' ').split('.')[0]
+
+            area = "全国"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道  
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.proxies = False
+            list_item.deal_detail = ['//div[@class="box-bottom"]']
+            list_item.parse_url = href
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_qtgg/招标采购-详情页.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=30)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="box-bottom"]').extract_first()
+        ex_html = response.xpath('//div[@class="box-bottom"]//div[contains(text(),"相关公告")]/..').extract_first()
+        if ex_html:
+            html = html.replace(ex_html,'')
+
+        list_item.contenthtml = html
+
+        attachments = {}
+
+        file_list = response.xpath('//div[@class="box-bottom"]//a')
+        if file_list:
+            for info in file_list:
+                file_name = info.xpath('./span/text()').extract_first("").strip()
+                file_url = info.xpath('./@href').extract_first("").strip()
+                file_type = extract_file_type(file_name)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+                list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_qtgg/非招标采购-其它公告-列表页.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+import json
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+
+    def start_callback(self):
+
+        self.site = "中国南方航空采购招标网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'cid', 'crawl_page'])
+
+        self.menus = [
+            Menu('非招标采购-其它公告', 'a_zgnfhkcgzbw_qtgg', '964894728657043456', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json; charset=UTF-8",
+            "Origin": "https://csbidding.csair.cn",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://csbidding.csair.cn/cms/api/dynamicData/queryContentPage"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "pageNo": page,
+            "pageSize": 15,
+            "dto": {
+                "siteId": "747",
+                "categoryId": menu.get('cid'),
+                "agentCompanyName": "",
+                "title": "",
+                "province": "",
+                "city": "",
+                "publishDays": "",
+                "estimateTotalPriceBegin": "",
+                "estimateTotalPriceEnd": ""
+            }
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        response.encoding = response.apparent_encoding
+        menu = request.item
+        info_list = response.json.get('res').get('rows')
+        for info in info_list:
+            href = "https://csbidding.csair.cn/cms/nfhk/webfile" + info.get('url')
+            title = info.get('title').strip()
+            create_time = info.get('publishDate').replace('T',' ').split('.')[0]
+
+            area = "全国"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道  
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.proxies = False
+            list_item.deal_detail = ['//div[@class="box-bottom"]']
+            list_item.parse_url = href
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_zbgg/招标公告-列表页.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+import json
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+
+    def start_callback(self):
+
+        self.site = "中国南方航空采购招标网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'cid', 'crawl_page'])
+
+        self.menus = [
+            Menu('招标公告', 'a_zgnfhkcgzbw_zbgg', '964895061131132928', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json; charset=UTF-8",
+            "Origin": "https://csbidding.csair.cn",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://csbidding.csair.cn/cms/api/dynamicData/queryContentPage"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "pageNo": page,
+            "pageSize": 15,
+            "dto": {
+                "siteId": "747",
+                "categoryId": menu.get('cid'),
+                "agentCompanyName": "",
+                "title": "",
+                "province": "",
+                "city": "",
+                "publishDays": "",
+                "estimateTotalPriceBegin": "",
+                "estimateTotalPriceEnd": ""
+            }
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        response.encoding = response.apparent_encoding
+        menu = request.item
+        info_list = response.json.get('res').get('rows')
+        for info in info_list:
+            href = "https://csbidding.csair.cn/cms/nfhk/webfile" + info.get('url')
+            title = info.get('title').strip()
+            create_time = info.get('publishDate').replace('T',' ').split('.')[0]
+
+            area = "全国"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道  
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.proxies = False
+            list_item.deal_detail = ['//div[@class="box-bottom"]']
+            list_item.parse_url = href
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_zbgg/招标采购-详情页.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=30)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="box-bottom"]').extract_first()
+        ex_html = response.xpath('//div[@class="box-bottom"]//div[contains(text(),"相关公告")]/..').extract_first()
+        if ex_html:
+            html = html.replace(ex_html,'')
+
+        list_item.contenthtml = html
+
+        attachments = {}
+
+        file_list = response.xpath('//div[@class="box-bottom"]//a')
+        if file_list:
+            for info in file_list:
+                file_name = info.xpath('./span/text()').extract_first("").strip()
+                file_url = info.xpath('./@href').extract_first("").strip()
+                file_type = extract_file_type(file_name)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+                list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_zhbgg/中标公告-列表页.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import BidingListItem
+from collections import namedtuple
+import json
+
+
+class ZtbpcFeapder(feapder.BiddingListSpider):
+
+
+    def start_callback(self):
+
+        self.site = "中国南方航空采购招标网"
+
+        Menu = namedtuple('Menu', ['channel', 'code', 'cid', 'crawl_page'])
+
+        self.menus = [
+            Menu('中标公告', 'a_zgnfhkcgzbw_zhbgg', '964895915208867840', 1),
+        ]
+
+        self.headers = {
+            "Accept": "application/json, text/javascript, */*; q=0.01",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json; charset=UTF-8",
+            "Origin": "https://csbidding.csair.cn",
+            "Pragma": "no-cache",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+            "X-Requested-With": "XMLHttpRequest",
+        }
+
+    def start_requests(self):
+        for menu in self.menus:
+            start_url = "https://csbidding.csair.cn/cms/api/dynamicData/queryContentPage"
+            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
+
+    def download_midware(self, request):
+        page = request.page
+        menu = request.item
+        data = {
+            "pageNo": page,
+            "pageSize": 15,
+            "dto": {
+                "siteId": "747",
+                "categoryId": menu.get('cid'),
+                "agentCompanyName": "",
+                "title": "",
+                "province": "",
+                "city": "",
+                "publishDays": "",
+                "estimateTotalPriceBegin": "",
+                "estimateTotalPriceEnd": ""
+            }
+        }
+        data = json.dumps(data, separators=(',', ':'))
+        request.data = data
+        request.headers = self.headers
+
+    def parse(self, request, response):
+        response.encoding = response.apparent_encoding
+        menu = request.item
+        info_list = response.json.get('res').get('rows')
+        for info in info_list:
+            href = "https://csbidding.csair.cn/cms/nfhk/webfile" + info.get('url')
+            title = info.get('title').strip()
+            create_time = info.get('publishDate').replace('T',' ').split('.')[0]
+
+            area = "全国"
+            city = ""
+
+            list_item = BidingListItem()  # 存储数据的管道  
+            list_item.href = href  # 标书链接
+            list_item.unique_key = ('href',)
+            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
+            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
+            list_item.title = title  # 标题
+            list_item.site = self.site
+            list_item.publishtime = create_time
+            list_item.area = area  # 城市默认:全国
+            list_item.city = city  # 城市 默认为空
+            list_item.parse = "self.detail_get"  # 详情页回调方法
+            list_item.proxies = False
+            list_item.deal_detail = ['//div[@class="box-bottom"]']
+            list_item.parse_url = href
+
+            yield list_item
+
+        request = self.infinite_pages(request, response)
+        yield request
+
+
+if __name__ == "__main__":
+    ZtbpcFeapder(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgnfhkcgzbw_zhbgg/招标采购-详情页.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""
+Created on 2025-05-26
+---------
+@summary: 中国南方航空采购招标网
+---------
+@author: lzz
+"""
+import feapder
+from items.spider_item import DataBakItem
+from untils.attachment import AttachmentDownloader
+from untils.tools import extract_file_type
+
+
+
+
+class Details(feapder.BiddingDetailSpider):
+
+    def start_requests(self):
+        data_lsit = self.get_tasks_by_rabbitmq(limit=30)
+        for item in data_lsit:
+            request_params = item.get("request_params")
+            timeout = request_params.get('timeout', 10)
+            request_params.pop('timeout', None)
+
+            yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
+                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
+                                  **request_params, timeout=timeout)
+
+    def detail_get(self, request, response):
+
+        items = request.item
+        list_item = DataBakItem(**items)
+
+        html = response.xpath('//div[@class="box-bottom"]').extract_first()
+        ex_html = response.xpath('//div[@class="box-bottom"]//div[contains(text(),"相关公告")]/..').extract_first()
+        if ex_html:
+            html = html.replace(ex_html,'')
+
+        list_item.contenthtml = html
+
+        attachments = {}
+
+        file_list = response.xpath('//div[@class="box-bottom"]//a')
+        if file_list:
+            for info in file_list:
+                file_name = info.xpath('./span/text()').extract_first("").strip()
+                file_url = info.xpath('./@href').extract_first("").strip()
+                file_type = extract_file_type(file_name)
+                if file_type:
+                    attachment = AttachmentDownloader().fetch_attachment(
+                        file_name=file_name, file_type=file_type, download_url=file_url)
+                    attachments[str(len(attachments) + 1)] = attachment
+
+            if attachments:
+                list_item.projectinfo = {"attachments": attachments}
+
+        yield list_item
+
+
+if __name__ == "__main__":
+    Details(redis_key="lzz:zgnfhkcgzbw_fzbcg").start()

a_zgsyzbtbw_zgysgg/log/2025-06-12/资格预审公告-列表页.log

@@ -0,0 +1,376 @@
+Thread-8|2025-06-12 18:46:28,263|parser_control.py|run|line:786|DEBUG| 等待任务...
+Spider|2025-06-12 18:46:28,696|spider.py|register_task_api_token|line:104|DEBUG| register api token:eyJpZCI6MX0.aEqwBA.9bEoGu3hMuBpfLmh9yzYBzyDS_Q
+Thread-8|2025-06-12 18:46:32,762|request.py|get_response|line:364|DEBUG| 
+                    -------------- Spider.parse request for ----------------
+                    url  = https://www.cnpcbidding.com/cms/article/page
+                    method = POST
+                    args = {'proxies': {'http': 'socks5://111.72.199.75:8861', 'https': 'socks5://111.72.199.75:8861'}, 'headers': {'Accept': 'application/json, text/plain, */*', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'Content-Type': 'application/json;charset=UTF-8', 'MACHINE_CODE': '1749725188828', 'Origin': 'https://www.cnpcbidding.com', 'Pragma': 'no-cache', 'Referer': 'https://www.cnpcbidding.com/', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'}, 'json': 'KVKABqwfVuSHX+ASpEhQSZfh/GLE6S1i0KFOlkuQMprT9nsWTRqL+pgfPsXtLoF9fUcsKU+U/w/Jv3vU1WfCuI7IVZ5T9M2N6JqeLRY+0JUqrKo9I14eOVIy/XyUGaDWb5weZePjcrrME2nq1U+KTU/0yphpLg+dWxdcPW7eMnY=', 'timeout': 60, 'stream': True, 'verify': False}
+                    
+Thread-7|2025-06-12 18:46:43,711|jy_item_buffer.py|__dedup_items|line:242|INFO| 待入库数据 4 条， 重复 0 条，实际待入库数据 4 条
+Thread-7|2025-06-12 18:46:43,716|jy_item_buffer.py|__add_item_to_db|line:365|DEBUG| 
+                -------------- item 批量入库 --------------
+                表名: pyspider_listdata
+                datas: [
+                {
+                                "pyuuid": "873dfec4477a11f0be5bacde48001122",
+                                "comeintime": 1749725202,
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "area": "全国",
+                                "city": "",
+                                "district": "",
+                                "href": "https://www.cnpcbidding.com/#/procurementNotice?300940",
+                                "request_params": {
+                                                "datapm": {
+                                                                "columnId": "1",
+                                                                "articleId": 300940
+                                                },
+                                                "method": "POST"
+                                },
+                                "title": "PQ for MANUAL BALL VALVES",
+                                "publishtime": "2025-06-05",
+                                "parse_url": "https://www.cnpcbidding.com/cms/article/details",
+                                "parser_name": "",
+                                "parse": "self.detail_get",
+                                "proxies": false,
+                                "deal_detail": [],
+                                "ex_js": "",
+                                "ex_python": null,
+                                "files": false,
+                                "is_mixed": false,
+                                "queue_name": "lzz:zgsyzbtbw_details:z_items",
+                                "update_at": 1749725203,
+                                "state": 1
+                },
+                {
+                                "pyuuid": "874d2c1e477a11f0be5bacde48001122",
+                                "comeintime": 1749725203,
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "area": "全国",
+                                "city": "",
+                                "district": "",
+                                "href": "https://www.cnpcbidding.com/#/procurementNotice?300702",
+                                "request_params": {
+                                                "datapm": {
+                                                                "columnId": "1",
+                                                                "articleId": 300702
+                                                },
+                                                "method": "POST"
+                                },
+                                "title": "中国石油天然气股份有限公司江西销售分公司2025-2027年度非油商品供应商资格准入（第二批）",
+                                "publishtime": "2025-06-04",
+                                "parse_url": "https://www.cnpcbidding.com/cms/article/details",
+                                "parser_name": "",
+                                "parse": "self.detail_get",
+                                "proxies": false,
+                                "deal_detail": [],
+                                "ex_js": "",
+                                "ex_python": null,
+                                "files": false,
+                                "is_mixed": false,
+                                "queue_name": "lzz:zgsyzbtbw_details:z_items",
+                                "update_at": 1749725203,
+                                "state": 1
+                },
+                {
+                                "pyuuid": "875a66f4477a11f0be5bacde48001122",
+                                "comeintime": 1749725203,
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "area": "全国",
+                                "city": "",
+                                "district": "",
+                                "href": "https://www.cnpcbidding.com/#/procurementNotice?296159",
+                                "request_params": {
+                                                "datapm": {
+                                                                "columnId": "1",
+                                                                "articleId": 296159
+                                                },
+                                                "method": "POST"
+                                },
+                                "title": "渤海石油装备（天津）新世纪机械制造有限公司实心抽油杆原材料供应商集中资格招标项目",
+                                "publishtime": "2025-04-24",
+                                "parse_url": "https://www.cnpcbidding.com/cms/article/details",
+                                "parser_name": "",
+                                "parse": "self.detail_get",
+                                "proxies": false,
+                                "deal_detail": [],
+                                "ex_js": "",
+                                "ex_python": null,
+                                "files": false,
+                                "is_mixed": false,
+                                "queue_name": "lzz:zgsyzbtbw_details:z_items",
+                                "update_at": 1749725203,
+                                "state": 1
+                },
+                {
+                                "pyuuid": "8767724a477a11f0be5bacde48001122",
+                                "comeintime": 1749725203,
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "area": "全国",
+                                "city": "",
+                                "district": "",
+                                "href": "https://www.cnpcbidding.com/#/procurementNotice?295978",
+                                "request_params": {
+                                                "datapm": {
+                                                                "columnId": "1",
+                                                                "articleId": 295978
+                                                },
+                                                "method": "POST"
+                                },
+                                "title": "重庆销售加油站综合汽服合作商准入(变更)",
+                                "publishtime": "2025-04-23",
+                                "parse_url": "https://www.cnpcbidding.com/cms/article/details",
+                                "parser_name": "",
+                                "parse": "self.detail_get",
+                                "proxies": false,
+                                "deal_detail": [],
+                                "ex_js": "",
+                                "ex_python": null,
+                                "files": false,
+                                "is_mixed": false,
+                                "queue_name": "lzz:zgsyzbtbw_details:z_items",
+                                "update_at": 1749725203,
+                                "state": 1
+                }
+]
+                    
+Thread-8|2025-06-12 18:46:44,184|parser_control.py|run|line:786|DEBUG| 等待任务...
+Thread-6|2025-06-12 18:46:44,302|heartbeat_buffer.py|__add_item_to_db|line:180|DEBUG| 
+                -------------- item 批量入库 --------------
+                表名: pyspider_heartbeat
+                datas: [
+                {
+                                "batch_no": "db36e0cae3316443c99df8726bc2d84d",
+                                "node_ip": null,
+                                "crawlab_taskid": null,
+                                "filepath": "/Users/dongzhaorui/Desktop/jy-git/platform-spiders/a_zgsyzbtbw_zgysgg/资格预审公告-列表页.py",
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "business_type": "BiddingList",
+                                "runtime": "2025-06-12",
+                                "url": "https://www.cnpcbidding.com/cms/article/page",
+                                "status_code": 200,
+                                "nowpage": 1,
+                                "count": 10,
+                                "failed_retry_times": 0,
+                                "rel_count": 10,
+                                "failed_task_count": 0,
+                                "success_task_count": 1,
+                                "create_at": 1749725204,
+                                "expire_at": "2025-06-12 10:46:44.183737"
+                }
+]
+                    
+Thread-6|2025-06-12 18:46:49,191|rabbitmq_pipeline.py|save_items|line:39|INFO| 共导出 1 条数据到 pyspider_heartbeat
+Thread-7|2025-06-12 18:46:49,279|rabbitmq_pipeline.py|save_items|line:39|INFO| 共导出 4 条数据到 pyspider_listdata
+Thread-7|2025-06-12 18:46:51,637|jy_item_buffer.py|__dedup_items|line:242|INFO| 待入库数据 6 条， 重复 0 条，实际待入库数据 6 条
+Thread-7|2025-06-12 18:46:51,653|jy_item_buffer.py|__add_item_to_db|line:365|DEBUG| 
+                -------------- item 批量入库 --------------
+                表名: pyspider_listdata
+                datas: [
+                {
+                                "pyuuid": "8774a992477a11f0be5bacde48001122",
+                                "comeintime": 1749725203,
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "area": "全国",
+                                "city": "",
+                                "district": "",
+                                "href": "https://www.cnpcbidding.com/#/procurementNotice?295966",
+                                "request_params": {
+                                                "datapm": {
+                                                                "columnId": "1",
+                                                                "articleId": 295966
+                                                },
+                                                "method": "POST"
+                                },
+                                "title": "重庆销售2025-2026年度咖啡、茶饮合作准入项目(变更)",
+                                "publishtime": "2025-04-23",
+                                "parse_url": "https://www.cnpcbidding.com/cms/article/details",
+                                "parser_name": "",
+                                "parse": "self.detail_get",
+                                "proxies": false,
+                                "deal_detail": [],
+                                "ex_js": "",
+                                "ex_python": null,
+                                "files": false,
+                                "is_mixed": false,
+                                "queue_name": "lzz:zgsyzbtbw_details:z_items",
+                                "update_at": 1749725210,
+                                "state": 1
+                },
+                {
+                                "pyuuid": "878191b6477a11f0be5bacde48001122",
+                                "comeintime": 1749725203,
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "area": "全国",
+                                "city": "",
+                                "district": "",
+                                "href": "https://www.cnpcbidding.com/#/procurementNotice?295643",
+                                "request_params": {
+                                                "datapm": {
+                                                                "columnId": "1",
+                                                                "articleId": 295643
+                                                },
+                                                "method": "POST"
+                                },
+                                "title": "渤海装备螺旋缝和直缝埋弧焊及高频直缝电阻焊制管用热轧卷板和中板供应商集中资格招标(变更)(变更)(变更)(变更)",
+                                "publishtime": "2025-04-21",
+                                "parse_url": "https://www.cnpcbidding.com/cms/article/details",
+                                "parser_name": "",
+                                "parse": "self.detail_get",
+                                "proxies": false,
+                                "deal_detail": [],
+                                "ex_js": "",
+                                "ex_python": null,
+                                "files": false,
+                                "is_mixed": false,
+                                "queue_name": "lzz:zgsyzbtbw_details:z_items",
+                                "update_at": 1749725210,
+                                "state": 1
+                },
+                {
+                                "pyuuid": "87ac2d86477a11f0be5bacde48001122",
+                                "comeintime": 1749725203,
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "area": "全国",
+                                "city": "",
+                                "district": "",
+                                "href": "https://www.cnpcbidding.com/#/procurementNotice?295301",
+                                "request_params": {
+                                                "datapm": {
+                                                                "columnId": "1",
+                                                                "articleId": 295301
+                                                },
+                                                "method": "POST"
+                                },
+                                "title": "重庆销售2025-2026年度咖啡、茶饮合作准入项目",
+                                "publishtime": "2025-04-17",
+                                "parse_url": "https://www.cnpcbidding.com/cms/article/details",
+                                "parser_name": "",
+                                "parse": "self.detail_get",
+                                "proxies": false,
+                                "deal_detail": [],
+                                "ex_js": "",
+                                "ex_python": null,
+                                "files": false,
+                                "is_mixed": false,
+                                "queue_name": "lzz:zgsyzbtbw_details:z_items",
+                                "update_at": 1749725210,
+                                "state": 1
+                },
+                {
+                                "pyuuid": "87bf032a477a11f0be5bacde48001122",
+                                "comeintime": 1749725203,
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "area": "全国",
+                                "city": "",
+                                "district": "",
+                                "href": "https://www.cnpcbidding.com/#/procurementNotice?295302",
+                                "request_params": {
+                                                "datapm": {
+                                                                "columnId": "1",
+                                                                "articleId": 295302
+                                                },
+                                                "method": "POST"
+                                },
+                                "title": "重庆销售加油站综合汽服合作商准入",
+                                "publishtime": "2025-04-17",
+                                "parse_url": "https://www.cnpcbidding.com/cms/article/details",
+                                "parser_name": "",
+                                "parse": "self.detail_get",
+                                "proxies": false,
+                                "deal_detail": [],
+                                "ex_js": "",
+                                "ex_python": null,
+                                "files": false,
+                                "is_mixed": false,
+                                "queue_name": "lzz:zgsyzbtbw_details:z_items",
+                                "update_at": 1749725210,
+                                "state": 1
+                },
+                {
+                                "pyuuid": "87cbd7e4477a11f0be5bacde48001122",
+                                "comeintime": 1749725203,
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "area": "全国",
+                                "city": "",
+                                "district": "",
+                                "href": "https://www.cnpcbidding.com/#/procurementNotice?294957",
+                                "request_params": {
+                                                "datapm": {
+                                                                "columnId": "1",
+                                                                "articleId": 294957
+                                                },
+                                                "method": "POST"
+                                },
+                                "title": "中国石油贵州销售公司二级律师库公开招标选商",
+                                "publishtime": "2025-04-15",
+                                "parse_url": "https://www.cnpcbidding.com/cms/article/details",
+                                "parser_name": "",
+                                "parse": "self.detail_get",
+                                "proxies": false,
+                                "deal_detail": [],
+                                "ex_js": "",
+                                "ex_python": null,
+                                "files": false,
+                                "is_mixed": false,
+                                "queue_name": "lzz:zgsyzbtbw_details:z_items",
+                                "update_at": 1749725210,
+                                "state": 1
+                },
+                {
+                                "pyuuid": "87f47e38477a11f0be5bacde48001122",
+                                "comeintime": 1749725204,
+                                "site": "中国石油招标投标网",
+                                "channel": "资格预审公告",
+                                "spidercode": "a_zgsyzbtbw_zgysgg",
+                                "area": "全国",
+                                "city": "",
+                                "district": "",
+                                "href": "https://www.cnpcbidding.com/#/procurementNotice?294664",
+                                "request_params": {
+                                                "datapm": {
+                                                                "columnId": "1",
+                                                                "articleId": 294664
+                                                },
+                                                "method": "POST"
+                                },
+                                "title": "渤海装备螺旋缝和直缝埋弧焊及高频直缝电阻焊制管用热轧卷板和中板供应商集中资格招标(变更)(变更)(变更)",
+                                "publishtime": "2025-04-14",
+                                "parse_url": "https://www.cnpcbidding.com/cms/article/details",
+                                "parser_name": "",
+                                "parse": "self.detail_get",
+                                "proxies": false,
+                                "deal_detail": [],
+                                "ex_js": "",
+                                "ex_python": null,
+                                "files": false,
+                                "is_mixed": false,
+                                "queue_name": "lzz:zgsyzbtbw_details:z_items",
+                                "update_at": 1749725210,
+                                "state": 1
+                }
+]
+                    
+Thread-7|2025-06-12 18:46:51,661|rabbitmq_pipeline.py|save_items|line:39|INFO| 共导出 6 条数据到 pyspider_listdata
+Spider|2025-06-12 18:46:55,839|spider.py|run|line:142|INFO| 无任务，爬虫结束
+Thread-6|2025-06-12 18:46:56,256|rabbitMq.py|close|line:480|DEBUG| 关闭 RabbitMQ -1
+Thread-7|2025-06-12 18:46:56,686|rabbitMq.py|close|line:480|DEBUG| 关闭 RabbitMQ -1