platform-spiders/a_bjgc_cggg_xbcg/采购公告-询比采购-列表页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-04-17
---------
@summary: 八戒公采
---------
@author: lzz
"""
import feapder
from items.spider_item import MgpListItem
from collections import namedtuple
from feapder.utils.tools import timestamp_to_date
import json


class Feapder(feapder.BiddingListSpider):

    def start_callback(self):
        Menu = namedtuple('Menu', ['channel', 'code', 'typeid', 'crawl_page'])
        self.site = "八戒公采"

        self.menus = [
            Menu('采购公告-询比采购', 'a_bjgc_cggg_xbcg', 1, 2),
        ]

        self.headers = {
            "authority": "bridgezhyc.zbj.com",
            "accept": "application/json, text/plain, */*",
            "accept-language": "zh-CN,zh;q=0.9",
            "accesstoken": "undefined",
            "cache-control": "no-cache",
            "content-type": "application/json",
            "logintoken": "undefined",
            "origin": "https://cg.zbj.com",
            "pragma": "no-cache",
            "referer": "https://cg.zbj.com/buyerHall",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
            "x-auth-token": "undefined",
            "x-requested-with": "XMLHttpRequest"
        }

    def start_requests(self):
        for menu in self.menus:
            start_url = "https://bridgezhyc.zbj.com/api/platform/queryPurchase"
            yield feapder.Request(url=start_url, item=menu._asdict(), page=1)

    def parse(self, request, response):
        menu = request.item
        info_list = response.json.get('data').get('data')
        for info in info_list:
            title = info.get('purchaseDemandName').strip()
            hid = info.get('originOrderNo')
            if len(hid) > 16:
                href = f"https://cg.zbj.com/newdetailsDemand?id={hid}"
            else:
                href = f"https://chinazhyc.zbj.com/detailsDemand?id={hid}"
            create_time = timestamp_to_date(int(str(info.get('publishDate'))[:-3]))

            area = "全国"  # 省份
            city = ""  # 城市

            list_item = MgpListItem()  # 存储数据的管道
            list_item.href = href  # 标书链接
            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
            list_item.title = title  # 标题
            list_item.publishtime = create_time  # 标书发布时间
            list_item.site = self.site
            list_item.area = area  # 城市默认:全国
            list_item.city = city  # 城市 默认为空

            list_item.unique_key = ('href', create_time)
            list_item.parse = "self.detail_get"
            list_item.request_params = {"headers": self.headers}
            list_item.proxies = False
            list_item.deal_detail = ['//div[@class="project"]']
            list_item.parse_url = href

            yield list_item

        # 无限翻页
        request = self.infinite_pages(request, response)
        yield request

    def download_midware(self, request):
        page = request.page
        menu = request.item
        data = {
            "data": {
                "purchasetypeLevel1Id": None,
                "purchasetypeLevel2Id": None,
                "purchasetypeLevel3Id": None,
                "purchasetypeLevel1IdName": "",
                "purchasetypeLevel2IdName": "",
                "purchasetypeLevel3IdName": "",
                "town": "",
                "expirytTimeSort": 0,
                "priceSort": 0,
                "releaseTimeSort": 0,
                "highPrice": None,
                "lowPrice": None,
                "page": page,
                "biddingType": menu.get('typeid'),
                "purchaseState": 0,
                "baseAppId": "99",
                "appId": "99",
                "pageSize": 40,
                "cityCode": "",
                "province": "",
                "city": "",
                "otherTown": "",
                "defaultRegion": "",
                "townType": 1,
                "title": ""
            }
        }
        data = json.dumps(data)
        request.data = data
        request.headers = self.headers


if __name__ == "__main__":
    Feapder(redis_key="lzz:bjgc_cggg").start()