platform-spiders/sd_jnsggzyjyw_cqjy_py/交易网-详情页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-06-04
---------
@summary: 济宁市公共资源交易网
---------
@author: lzz
"""
import feapder
from items.spider_item import DataBakItem
from feapder.utils.tools import log
from untils.attachment import AttachmentDownloader
from untils.tools import extract_file_type,remove_htmldata



class Details(feapder.BiddingDetailSpider):

    def start_requests(self):
        data_list = self.get_tasks_by_rabbitmq(limit=20)
        for item in data_list:
            # log.debug(item)
            request_params = item.get("request_params")
            yield feapder.Request(url=item.get("parse_url"), item=item,
                                  deal_detail=item.get("deal_detail"),proxies=False,
                                  callback=eval(item.get("parse")),  **request_params)

    def detail_get(self, request, response):

        items = request.item
        list_item = DataBakItem(**items)

        html = response.xpath('//div[@class="card-body"]').extract_first()  # 标书详细内容

        rm_list =['//div[@class="bm-side-link"]','下载招标文件','//ul[@id="list-detail"]','//ul[@class="clearfix"]',
                  '//div[@class="text-center margin-top-40 margin-bottom-40"]','//h4[@style="margin:10px;"]']
        html = remove_htmldata(rm_list, html, response)

        list_item.contenthtml = html

        file_list = response.xpath('//div[@class="card-body"]//a[@href]|//div[@id="ctn-detail"]//a[@href]')
        if file_list:
            attachments = {}
            for info in file_list:
                file_url = info.xpath('./@href').extract_first()
                file_name = "".join(info.xpath('.//text()').extract())
                if file_name:
                    file_name = file_name.strip()
                    file_type = extract_file_type(file_name,file_url,['zbd','zbdx'])
                    if file_type:
                        attachment = AttachmentDownloader().fetch_attachment(
                            file_name=file_name, file_type=file_type, download_url=file_url,)
                        if attachment.get('size'):
                            attachments[str(len(attachments) + 1)] = attachment
            if attachments:
                list_item.projectinfo = {"attachments": attachments}

        yield list_item


if __name__ == "__main__":
    Details(redis_key="lzz:jnsggzyjyw_lzzqgg").start()