platform-spiders/a_whswszjcs_cggg/交易信息-详情页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-06-02
---------
@summary: 芜湖市网上中介超市
---------
@author: lzz
"""
import feapder
from items.spider_item import DataBakItem
from untils.attachment import AttachmentDownloader
from untils.tools import extract_file_type
import requests




def get_file(did):
    headers = {
        "Accept": "application/json, text/plain, */*",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Content-Type": "application/x-www-form-urlencoded",
        "Origin": "https://wh.ahzwfw.gov.cn",
        "Pragma": "no-cache",
        "Referer": "https://wh.ahzwfw.gov.cn/wszjcs-web/views/projectnotice/projectnotice.html",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
    }

    url = "https://wh.ahzwfw.gov.cn/wszjcs-web/tradingInformation/getTradingInformationDetails.do"
    data = {
        "id": did,
    }
    response = requests.post(url, headers=headers, data=data, timeout=30, verify=False)
    return response.json().get('data').get('attachmentDtos')



class Details(feapder.BiddingDetailSpider):

    def start_requests(self):
        data_lsit = self.get_tasks_by_rabbitmq(limit=10)
        for item in data_lsit:
            request_params = item.get("request_params")
            timeout = request_params.get('timeout', 10)
            request_params.pop('timeout', None)
            did = item.get('did')
            yield feapder.Request(url=item.get("parse_url"), item=item, proxies=False,render=True,render_time=5,
                                  deal_detail=item.get("deal_detail"), callback=eval(item.get("parse")),
                                  **request_params, timeout=timeout,did=did)

    def detail_get(self, request, response):

        items = request.item
        list_item = DataBakItem(**items)

        html = response.xpath('//div[@class="detail"]').extract_first()

        attachments = {}
        file_list = get_file(request.did)
        if file_list:
            for info in file_list:
                file_name = info.get('fileName').strip()
                fid = info.get('filePath')
                file_url = f"https://wh.ahzwfw.gov.cn/wszjcs-web/file/downloadFile.do?fileId={fid}&fileName={file_name}"
                file_type = extract_file_type(file_name=file_name, file_url=fid)
                if file_type:
                    attachment = AttachmentDownloader().fetch_attachment(
                        file_name=file_name, file_type=file_type, download_url=file_url,)
                    attachments[str(len(attachments) + 1)] = attachment

        if attachments:
            list_item.projectinfo = {"attachments": attachments}

        list_item.contenthtml = html

        yield list_item


if __name__ == "__main__":
    Details(redis_key="lzz:wnjtdzzbcgpt_jggs").start()