platform-spiders/a_dfxwszhcgpt_zbgg/Dfxwszhcgpt.py

# -*- coding: utf-8 -*-
"""
Created on 2025-01-06
---------
@summary: 东方希望数字化采购平台
---------
@author: lzz
"""
from collections import namedtuple

import feapder
from items.spider_item import BidingListItem


class Spider(feapder.BiddingListSpider):

    def start_callback(self):
        Menu = namedtuple('Menu', ['channel', 'code', 'sourceType', 'purchaseType', 'crawl_page'])
        self.site = "东方希望数字化采购平台"

        self.menus = [
            Menu('招标公告', 'a_dfxwszhcgpt_zbgg', 'BR', '1', 3),
        ]
        self.headers = {
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Origin": "https://srm.easthope.cn",
            "Referer": "https://srm.easthope.cn/",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
        }

    def start_requests(self):
        url = "https://srm-gw.easthope.cn/ssrc/v1/53/source-notices/br-list/advance/public"
        for menu in self.menus:
            yield feapder.Request(url, item=menu._asdict(), page=1)

    def download_midware(self, request):
        page = request.page
        sourceType = request.item.get('sourceType')
        purchaseType = request.item.get('purchaseType')
        params = {
            "sourceType": f"{sourceType}",
            "page": f"{page - 1}",
            "size": "10",
            "bidTitle": "",
            "projectType": "",
            "startDate": "",
            "endDate": "",
            "industry": "",
            "purchaseType": f"{purchaseType}"
        }
        request.params = params
        request.headers = self.headers

    def parse(self, request, response):
        menu = request.item
        info_list = response.json.get('content')
        for info in info_list:
            sourceType = info.get('sourceType')
            noticeId = info.get('noticeId')
            sourceHeaderId = info.get('sourceHeaderId')
            advanceNoticeId = info.get('advanceNoticeId')
            sourceFrom = info.get('sourceFrom')
            href = f"https://srm.easthope.cn/oauth/public/default/source_notice_advance_detail.html?type={sourceType}&from=ZBXX_HOME&noticeId={noticeId}&sourceHeaderId={sourceHeaderId}&advanceNoticeId={advanceNoticeId}&sourceFrom={sourceFrom}"
            title = info.get('bidTitle').strip()
            create_time = info.get('approvedDate')

            area = "全国"  # 省份
            city = ""  # 城市

            list_item = BidingListItem()  # 存储数据的管道
            list_item.href = href  # 标书链接
            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
            list_item.title = title  # 标题
            list_item.publishtime = create_time  # 标书发布时间
            list_item.site = self.site
            list_item.area = area  # 城市默认:全国
            list_item.city = city  # 城市 默认为空

            list_item.unique_key = ('href', 'title')
            list_item.parse = "self.detail_get"
            list_item.deal_detail = ['//div[@class="notice-detail-body"]']
            list_item.proxies = False
            list_item.parse_url = href
            yield list_item

        request = self.infinite_pages(request, response)
        yield request


if __name__ == "__main__":
    Spider(redis_key="lzz:DfxwszhcgptNew").start()