platform-spiders/gs_gssggzyjyw_zfcg_jc/政府采购-详情页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-04-19
---------
@summary: 甘肃省公共资源交易网
---------
@author: lzz
"""
import time
import feapder
from feapder.utils.tools import log
from items.spider_item import DataBakItem
from untils.attachment import AttachmentDownloader
from untils.tools import extract_file_type,text_search

headers = {
    "Accept": "*/*",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
    "Origin": "http://47.110.59.239:9207",
    "Pragma": "no-cache",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
    "X-Requested-With": "XMLHttpRequest"
}


class FirefoxDetails(feapder.BiddingDetailSpider):

    def start_requests(self):
        data_list = self.get_tasks_by_rabbitmq(limit=30)
        for item in data_list:
            # log.debug(item)
            request_params = item.get("request_params")
            yield feapder.Request(url=item.get("parse_url"), item=item, files_info=item.get("files"),
                                  deal_detail=item.get("deal_detail"), **request_params, headers=headers,
                                  callback=eval(item.get("parse")))

    def detail_get(self, request, response):
        items = request.item
        list_item = DataBakItem(**items)

        html = response.xpath('//div[@class="jxGonggaoInformationDetail"]').extract_first("")

        attachments = {}
        file__list = response.xpath('//div[@class="jxGonggaoInformationDetail"]//iframe|//img')
        if file__list:
            for index,info in enumerate(file__list):
                index += 1
                file_url = info.xpath('./@src').extract_first("")
                file_type = extract_file_type(file_url=file_url)
                if file_url and file_type:
                    attachment = AttachmentDownloader().fetch_attachment(
                        file_name=f'{index}', file_type=file_type, download_url=file_url)
                    attachments[str(len(attachments) + 1)] = attachment

        if attachments:
            list_item.projectinfo = {"attachments": attachments}
            if text_search(html).total < 10:
                html = "详情请访问原网页！"

        list_item.contenthtml = html

        yield list_item


if __name__ == "__main__":
    FirefoxDetails(redis_key="lzz:gssggzyjyw_zfcg").start()