platform-spiders/gs_gssggzyjyw_zfcg_qy/政府采购-庆阳-列表页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-04-19
---------
@summary: 甘肃省公共资源交易网
---------
@author: lzz
"""
import feapder
from items.spider_item import MgpListItem
from collections import namedtuple
import base64


def openDetail(tradingCode,projectType,tableName,platformCode,pubServicePlatCode):
    return (tradingCode,projectType,tableName,platformCode,pubServicePlatCode)


class Gssggzyjyw(feapder.BiddingListSpider):

    def start_callback(self):
        Menu = namedtuple('Menu', ['channel', 'code', 'href_param', 'area', 'type', 'crawl_page'])
        self.site = "甘肃省公共资源交易网"

        self.menus = [
            Menu('政府采购-庆阳', 'gs_gssggzyjyw_zfcg_qy','newprovince/annogoods/getAnnoList','621000','D',1),
        ]
        self.headers = {
            "Accept": "text/html, */*; q=0.01",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
            "Origin": "http://47.110.59.239:9207",
            "Pragma": "no-cache",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
            "X-Requested-With": "XMLHttpRequest"
        }


    def start_requests(self):

        for menu in self.menus:

            start_url = f'http://47.110.59.239:9207/f/{menu.href_param}'
            yield feapder.Request(url=start_url, item=menu._asdict(),page=1)

    def download_midware(self, request):
        page = request.page
        menu = request.item
        data = {
            "pageNo": f"{page}",
            "pageSize": "10",
            "area": f"{menu.get('area')}",
            "projecttype": f"{menu.get('type')}",
            "prjpropertynewI": "I",
            "prjpropertynewA": "A",
            "prjpropertynewD": "D",
            "prjpropertynewC": "C",
            "prjpropertynewB": "B",
            "prjpropertynewE": "E",
            "prjpropertynewZ": "Z",
            "projectname": ""
        }
        request.data = data
        request.headers = self.headers

    def parse(self, request, response):
        menu = request.item
        info_list = response.xpath('//div[@class="sTradingInformationSelectedBtoList"]/dl|//dl[@class="sDisclosurLeftConDetailList"]')
        for info in info_list:
            parse_param = info.xpath('./dd//a/@onclick').extract_first("").strip()
            tradingCode,projectType,tableName,platformCode,pubServicePlatCode = eval(parse_param)
            href_param = base64.b64encode(tradingCode.encode('utf-8')).decode("utf-8")
            href = f'http://47.110.59.239:9207/f/cityTenderProject/{href_param}/cityTenderprojectIndex?projectType={projectType}&table={tableName}&platformCode={platformCode}&pubServicePlatCode={pubServicePlatCode}'
            title = info.xpath('./dd//a/@title').extract_first("").strip()
            if not title == None:
                title = "".join(info.xpath('./dd//a/text()').extract()).strip()
            create_time = info.xpath('./dd//i/text()').extract_first().strip()

            area = "甘肃"  # 省份
            city = menu.get('channel').split('-')[-1] + "市"  # 城市

            list_item = MgpListItem()  # 存储数据的管道
            list_item.href = href  # 标书链接
            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
            list_item.title = title  # 标题
            list_item.publishtime = create_time  # 标书发布时间
            list_item.site = self.site
            list_item.area = area  # 城市默认:全国
            list_item.city = city  # 城市 默认为空

            dp = href + create_time + title

            list_item.unique_key = (dp,)
            list_item.parse = "self.detail_get"

            url = "http://47.110.59.239:9207/f/cityTenderProject/flowpage"
            data = {
                "tradingCode": tradingCode,
                "platformCode": platformCode,
                "pubServicePlatCode": pubServicePlatCode,
                "index": "0",
                "projectType": projectType,
                "tableName": tableName
            }
            list_item.proxies = False
            list_item.request_params = {"data":data}
            list_item.parse_url = url

            yield list_item

        # 无限翻页
        request = self.infinite_pages(request, response)
        yield request


if __name__ == "__main__":
    Gssggzyjyw(redis_key="lzz:gssggzyjyw_zfcg").start()