# -*- coding: utf-8 -*-
"""
Created on 2025-06-04
---------
@summary: 济宁市公共资源交易网
---------
@author: lzz
"""
import feapder
from items.spider_item import MgpListItem
from collections import namedtuple
from html import unescape



class Feapder(feapder.BiddingListSpider):


    def start_callback(self):
        Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])

        self.site = "济宁市公共资源交易网"

        self.menus = [
            Menu('地矿资源', 'sd_jnsggzyjyw_dkzy_py', '600', 1),
            Menu('地矿资源', 'sd_jnsggzyjyw_dkzy_py', '601', 1),
        ]
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Connection": "keep-alive",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
        }

    def start_requests(self):
        for menu in self.menus:
            start_url = "https://jnggzy.jnzbtb.cn:4430/JiNing/Posts"
            yield feapder.Request(url=start_url, item=menu._asdict(), page=1,  proxies=False)

    def download_midware(self, request):
        page = request.page
        menu = request.item
        tid = menu.get('tid')
        params = {
            "cat": tid,
            "pn": f"{page}"
        }
        request.params = params
        request.headers = self.headers


    def parse(self, request, response):

        menu = request.item
        info_list = response.xpath('//ul[@class="list-group"]/li')
        for info in info_list:
            href = info.xpath('./a/@href').extract_first("").strip()
            title = unescape("".join(info.xpath('./a/text()').extract()).strip())
            create_time = info.xpath('./span/text()').extract_first("").strip()

            area = "山东"    # 省份
            city = "济宁市"     # 城市

            list_item = MgpListItem()  # 存储数据的管道
            list_item.href = href  # 标书链接
            list_item.unique_key = ('title', 'href')
            list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
            list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
            list_item.title = title  # 标题
            list_item.site = self.site
            list_item.publishtime = create_time
            list_item.area = area  # 城市默认:全国
            list_item.city = city  # 城市 默认为空

            list_item.parse = "self.detail_get"  # 详情页回调方法

            list_item.deal_detail = ['//div[@class="card-body"]']
            list_item.proxies = False
            list_item.parse_url = href

            yield list_item

        # 无限翻页
        request = self.infinite_pages(request, response)
        yield request


if __name__ == "__main__":
    Feapder(redis_key="lzz:jnsggzyjyw_lzzqgg").start()