platform-spiders/gs_gssggzyjyw_tdhkyq_sjpt/土地和矿业权-省级平台-列表页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-04-19
---------
@summary: 甘肃省公共资源交易网
---------
@author: lzz
"""
import feapder
from items.spider_item import MgpListItem
from collections import namedtuple
import re


class Gssggzyjyw(feapder.BiddingListSpider):

    def start_callback(self):
        Menu = namedtuple('Menu', ['channel', 'code', 'href_param', 'area', 'type', 'crawl_page'])
        self.site = "甘肃省公共资源交易网"

        self.menus = [
            Menu('土地和矿业权-省级平台', 'gs_gssggzyjyw_tdhkyq_sjpt', 'newprovince/annogoodsmine/getLandAnnoList', '620000', 'B',1),
        ]

        self.headers = {
            "Accept": "text/html, */*; q=0.01",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
            "Origin": "http://47.110.59.239:9207",
            "Pragma": "no-cache",
            "Referer": "http://47.110.59.239:9207/f/newprovince/annogoods/list",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
            "X-Requested-With": "XMLHttpRequest"
        }

    def start_requests(self):

        for menu in self.menus:
            start_url = f'http://47.110.59.239:9207/f/{menu.href_param}'
            yield feapder.Request(url=start_url, item=menu._asdict(),page=1)

    def download_midware(self, request):
        page = request.page
        menu = request.item
        data = {
            "pageNo": f"{page}",
            "pageSize": "10",
            "area": f"{menu.get('area')}",
            "projecttype": f"{menu.get('type')}",
            "prjpropertynewI": "I",
            "prjpropertynewA": "A",
            "prjpropertynewD": "D",
            "prjpropertynewC": "C",
            "prjpropertynewB": "B",
            "prjpropertynewE": "E",
            "prjpropertynewZ": "Z",
            "projectname": ""
        }
        request.data = data
        request.headers = self.headers

    def parse(self, request, response):
        menu = request.item
        info_list = response.xpath('//div[@class="sTradingInformationSelectedBtoList"]/dl|//dl[@class="sDisclosurLeftConDetailList"]')
        for info in info_list:
            href = info.xpath('./dd//a/@href').extract_first()
            title = info.xpath('./dd//a/@title').extract_first("").strip()
            if not title:
                title = "".join(info.xpath('./dd//a/text()').extract()).strip()
            create_time = info.xpath('./dd//i/text()').extract_first().strip()

            hid = "".join(re.findall('tenderproject/(.*?)/',href))
            area = "甘肃"  # 省份
            city = ""  # 城市

            list_item = MgpListItem()  # 存储数据的管道
            list_item.href = href  # 标书链接
            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
            list_item.title = title  # 标题
            list_item.publishtime = create_time  # 标书发布时间
            list_item.site = self.site
            list_item.area = area  # 城市默认:全国
            list_item.city = city  # 城市 默认为空

            dp = href + create_time + title
            list_item.unique_key = (dp,)
            list_item.parse = "self.detail_get"
            list_item.proxies = False
            data = {
                "bidpackages": "",
                "tenderprojectid": hid,
                "index": "0"
            }
            list_item.request_params = {"data": data}
            list_item.parse_url = "http://47.110.59.239:9207/f/newprovince/tenderproject/flowListAndAuctionpage"

            yield list_item

        # 无限翻页
        request = self.infinite_pages(request, response)
        yield request



if __name__ == "__main__":
    Gssggzyjyw(redis_key="lzz:gssggzyjyw_tdhkyq_sjpt").start()