platform-spiders/gd_gdsztbjgw_zbjh/招标计划-列表页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-04-05
---------
@summary: 广东省招投标监管网
---------
@author: lzz
"""
import feapder
from items.spider_item import BidingListItem
from collections import namedtuple
from ztbjgw_tool import *
import json



class ZtbpcFeapder(feapder.BiddingListSpider):


    def start_callback(self):

        self.site = "广东省招投标监管网"
        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
        self.menus = [
            Menu('招标计划', 'gd_gdsztbjgw_zbjh', 1),
        ]

        self.headers = {
            'Accept': 'application/json, text/plain, */*',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,sq;q=0.7',
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Content-Type': 'application/json',
            'Origin': 'https://zbtb.gd.gov.cn',
            'Pragma': 'no-cache',
            'Referer': 'https://zbtb.gd.gov.cn/',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
        }


    def start_requests(self):
        for menu in self.menus:
            start_url = "https://zbtb.gd.gov.cn/ztbjg-portal/center/notice/search"
            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)

    def validate(self, request, response):
        data = response.json.get('data')
        if not data:
            raise ValueError(f"data is '{data}'")
        return True

    def download_midware(self, request):
        page = request.page
        data = {
            "type": "trading-type",
            "publishStartTime": "",
            "publishEndTime": "",
            "keyword": "",
            "thirdType": 14,
            "siteCode": "44",
            "secondType": "A",
            "projectType": "",
            "dateType": "",
            "searchType": "pf",
            "total": 25311,
            "pageNo": page,
            "pageSize": 50,
            "openConvert": False
        }
        x_dgi = get_x_dgi(data)
        x_dgi['X-Dgi-Req-Timestamp'] = str(x_dgi['X-Dgi-Req-Timestamp'])
        self.headers.update(x_dgi)
        data = json.dumps(data, separators=(',', ':'))
        request.data = data
        request.headers = self.headers

    def parse(self, request, response):
        response.encoding = response.apparent_encoding
        menu = request.item
        info_list = response.json.get('data').get('pageData')
        for info in info_list:
            noticeSecondType = info.get('noticeSecondType')
            edition = info.get('edition')
            noticeId = info.get('noticeId')
            projectCode = info.get('projectCode')
            pubServicePlat = info.get('pubServicePlat')
            noticeSecondTypeDesc = info.get('noticeSecondTypeDesc')
            publishDate = info.get('publishDate')
            tradingProcess = info.get('tradingProcess')

            href = f"https://zbtb.gd.gov.cn/#/jygg/{edition}/{projectCode}/{noticeSecondType}/{tradingProcess}?rowGuid={noticeId}&siteName={pubServicePlat}&titleDetails={noticeSecondTypeDesc}"
            title = info.get('noticeTitle').strip()
            create_time = deal_time(publishDate)
            siteName = info.get('siteName', '').strip()

            area = "广东"
            city = siteName

            list_item = BidingListItem()  # 存储数据的管道
            list_item.href = href  # 标书链接
            list_item.unique_key = ('href',)
            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
            list_item.title = title  # 标题
            list_item.site = self.site
            list_item.publishtime = create_time
            list_item.area = area  # 城市默认:全国
            list_item.city = city  # 城市 默认为空

            list_item.parse = "self.detail_get"  # 详情页回调方法

            dparams = {
                "projectCode": projectCode,
                "tradingType": noticeSecondType,
                "tradingProcess": tradingProcess,
                "siteCode": "44"
            }
            list_item.request_params = {"params": dparams}
            list_item.deal_detail = []
            list_item.proxies = False
            list_item.parse_url = "https://zbtb.gd.gov.cn/ztbjg-portal/center/trading-notice/detail"

            yield list_item

        request = self.infinite_pages(request, response)
        yield request


if __name__ == "__main__":
    ZtbpcFeapder(redis_key="lzz:gdsztbjgw_zbjh").start()