platform-spiders/hn_hnswszjfwcs_wxxmgs/无效项目公示-列表页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-04-24
---------
@summary: 湖南省网上中介服务超市
---------
@author: lzz
"""
import feapder
from items.spider_item import MgpListItem
from collections import namedtuple
import json



class Feapder(feapder.BiddingListSpider):

    def start_callback(self):
        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])

        self.site = "湖南省网上中介服务超市"

        self.menus = [
            Menu('无效项目公示', 'hn_hnswszjfwcs_wxxmgs', 1),
        ]

        self.headers = {
            "Accept": "application/json, text/plain, */*",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/json",
            "Origin": "http://zwfw-new.hunan.gov.cn",
            "Pragma": "no-cache",
            "Referer": "http://zwfw-new.hunan.gov.cn/hn-zjcs-qt/bidResultNotice/invalidProjectList",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
        }


    def start_requests(self):
        for menu in self.menus:
            start_url = "http://zwfw-new.hunan.gov.cn/hn-zjcs-pub/api/invalidProject/rest"
            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)

    def download_midware(self, request):
        data = {
            "projectName": "",
            "divisionCode": "430000",
            "purOrgName": "",
            "invalidSubmitTimeBegin": "",
            "invalidSubmitTimeEnd": "",
            "pageNumber": 0
        }
        data = json.dumps(data, separators=(',', ':'))
        request.data = data
        request.headers = self.headers


    def parse(self, request, response):

        menu = request.item
        info_list = response.json.get('data').get('page').get('content')
        for info in info_list:
            projectCode = info.get('projectCode')
            href = f"https://zwfw-new.hunan.gov.cn/hn-zjcs-qt/bidResultNotice/invalidProjectView/{projectCode}"
            title = info.get('projectName').strip()
            create_time = info.get('publishDate')

            area = "湖南"    # 省份
            city = ""    # 城市

            list_item = MgpListItem()         # 存储数据的管道
            list_item.href = href             # 标书链接
            list_item.unique_key = ('href',)
            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
            list_item.title = title           # 标题
            list_item.site = self.site
            list_item.publishtime = create_time
            list_item.area = area         # 城市默认:全国
            list_item.city = city         # 城市 默认为空

            list_item.parse = "self.detail_get"  # 详情页回调方法
            list_item.deal_detail = ['//div[@class="left-body"]']  # 抽取正文xpath
            list_item.proxies = False
            list_item.parse_url = f"https://zwfw-new.hunan.gov.cn/hn-zjcs-pub/api/invalidProject/view/{projectCode}"

            yield list_item

        # 无限翻页
        request = self.infinite_pages(request, response)
        yield request


if __name__ == "__main__":
    Feapder(redis_key="lzz:hnswszjfwcs_wxxmgs").start()