platform-spiders/a_bjgc_qxgs/取消公示-列表页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-04-17
---------
@summary: 八戒公采
---------
@author: lzz
"""
import feapder
from items.spider_item import MgpListItem
from collections import namedtuple
from feapder.utils.tools import get_today_of_day,timestamp_to_date
import json



class Feapder(feapder.BiddingListSpider):

    def start_callback(self):
        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])

        self.site = "八戒公采"

        self.menus = [
            Menu('取消公示', 'a_bjgc_qxgs', 3),
        ]

        self.headers = {
            "authority": "bridgezhyc.zbj.com",
            "accept": "application/json, text/plain, */*",
            "accept-language": "zh-CN,zh;q=0.9",
            "accesstoken": "undefined",
            "cache-control": "no-cache",
            "content-type": "application/json",
            "logintoken": "undefined",
            "origin": "https://cg.zbj.com",
            "pragma": "no-cache",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
            "x-auth-token": "undefined",
            "x-requested-with": "XMLHttpRequest"
        }

    def start_requests(self):
        for menu in self.menus:
            start_url = "https://bridgezhyc.zbj.com/api/notice/queryNoticeList"
            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)

    def download_midware(self, request):
        page = request.page
        data = {
            "data": {
                "businessId": "",
                "biddingType": "0",
                "purchasingInformation": "",
                "transactionSupplierName": "",
                "regionVal": [],
                "requestId": "1531362372728",
                "requirementName": "",
                "type": "1",
                "page": page,
                "pageSize": 10,
                "province": "",
                "city": "",
                "region": "",
                "startTime": f"{get_today_of_day(-3)}",
                "endTime": f"{get_today_of_day()}"
            }
        }
        data = json.dumps(data, separators=(',', ':'))
        request.data = data
        request.headers = self.headers


    def parse(self, request, response):

        menu = request.item
        info_list = response.json.get('data').get('data')
        for info in info_list:
            hid = info.get('id')
            htp = info.get('type')
            appId = info.get('appId')
            href = f"https://cg.zbj.com/publicityDetails?id={hid}&type={htp}"
            if appId == "HLJGCY":
                title = info.get('purchasingInformation','').strip() + info.get('name').strip() + "更正公告"
            else:
                title = info.get('name').strip()
            create_time = timestamp_to_date(int(str(info.get('publishDate'))[:10]))

            area = "全国"
            city = ""

            list_item = MgpListItem()  # 存储数据的管道
            list_item.href = href  # 标书链接
            list_item.unique_key = ('href',create_time)
            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
            list_item.title = title  # 标题
            list_item.site = self.site
            list_item.publishtime = create_time
            list_item.area = area or "全国"  # 城市默认:全国
            list_item.city = city  # 城市 默认为空

            list_item.parse = "self.detail_get"  # 详情页回调方法
            list_item.deal_detail = []  # 抽取正文xpath
            list_item.proxies = False

            ddata = {
                "data": {
                    "id": f"{hid}"
                }
            }
            ddata = json.dumps(ddata, separators=(',', ':'))
            list_item.request_params = {"data":ddata}
            list_item.parse_url = "https://bridgezhyc.zbj.com/api/buyer/queryPurchasingDemandHistoryById"


            yield list_item

        # 无限翻页
        request = self.infinite_pages(request, response)
        yield request


if __name__ == "__main__":
    Feapder(redis_key="lzz:bjgc_qxgs").start()