platform-spiders/a_gdsggzyjypt_pwq_jggg/排污权-详情页.py

# -*- coding: utf-8 -*-
"""
Created on 2024-01-04
---------
@summary: 广东省公共资源交易平台
---------
@author: lzz
"""
import re

import feapder
from feapder.network.selector import Selector
from items.spider_item import DataBakItem
from untils.attachment import AttachmentDownloader
from untils.tools import extract_file_type

from gd_utils import *


class Details(feapder.BiddingDetailSpider):

    def start_requests(self):
        data_list = self.get_tasks_by_rabbitmq(limit=20)
        for item in data_list:
            request_params = item.get("request_params")
            yield feapder.Request(url=item.get("parse_url"), item=item,proxies=False,
                                  deal_detail=item.get("deal_detail"), **request_params,
                                  callback='parse')

    def download_midware(self, request):
        en_str = get_enstr(request.params)
        request.proxies = get_proxy(socks5h=True)

        request.headers = {
            "Accept": "application/json, text/plain, */*",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Pragma": "no-cache",
            "Referer": "https://ygp.gdzwfw.gov.cn/ggzy-portal/",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
            "X-Dgi-Req-App": en_str.get('X-Dgi-Req-App'),
            "X-Dgi-Req-Nonce": en_str.get('X-Dgi-Req-Nonce'),
            "X-Dgi-Req-Signature": en_str.get('X-Dgi-Req-Signature'),
            "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
        }


    def parse(self, request, response):
        attachments = {}
        items = request.item
        list_item = DataBakItem(**items)

        detail_info = response.json.get('data').get('tradingNoticeColumnModelList')
        ggxx_info = detail_info[0].get('multiKeyValueTableList')[0]

        tphtml = ""
        if ggxx_info:
            for gd in ggxx_info:
                temps = f'''
                    <tr>
                        <th colspan="1"><span>{gd.get('key')}</span></th>
                        <td colspan="3"><span>{gd.get('value', '无')}</span>
                        </td>
                    </tr>
                    '''
                tphtml += temps

        ggxx_html = f'''
            <section>
                <h2 id="公告信息" class="subtitle">公告信息</h2>
                <div class="mt-2">
                    <div>
                        <div>
                            <table>
                                <tbody>
                                {tphtml}
                                </tbody>
                            </table>
                        </div>
                    </div>
                </div>
            </section>
            '''

        ggnr_html = detail_info[1].get('richtext') or ""
        try:
            f_list = detail_info[-1].get('noticeFileBOList')
        except:
            f_list = None
        if f_list:
            ff_html = ""
            index = 1
            for f in f_list:
                f_id = f.get('rowGuid')
                version = "".join(re.findall('new/jygg/(.*?)/',list_item.href))
                f_url = f"https://ygp.gdzwfw.gov.cn/ggzy-portal/base/sys-file/download/{version}/{f_id}"
                f_name = f.get('fileName').strip()
                temp = f'''
                <li>
                    <span>附件名称 {index}</span>
                    <div>
                        <div>
                            <a href="{f_url}">{f_name}</a>
                        </div>
                    </div>
                </li>
                '''
                index += 1
                ff_html += temp
                f_type = extract_file_type(f_name, f_url)
                if f_type:
                    attachment = AttachmentDownloader().fetch_attachment(
                        file_name=f_name, file_type=f_type, download_url=f_url,
                        proxies=request.get_proxies())
                    attachments[str(len(attachments) + 1)] = attachment

            file_html = f'''
            <div class="fileList">
                <h2 id="相关附件" class="subtitle">相关附件</h2>
                <ul>
                    {ff_html}
                </ul>
            </div>
            '''
        else:
            file_html = ""

        list_item.contenthtml = ggxx_html + ggnr_html + file_html

        iframe_url = Selector(ggnr_html).xpath('//iframe/@src').extract_first()

        fm_type = extract_file_type('公告内容', iframe_url)
        if fm_type:
            attachmentf = AttachmentDownloader().fetch_attachment(
                file_name='公告内容', file_type=fm_type, download_url=iframe_url,
                proxies=request.get_proxies())
            attachments[str(len(attachments) + 1)] = attachmentf

        file_list = Selector(ggnr_html).xpath('//a[@href]')
        if file_list:
            for info in file_list:
                file_name = "".join(info.xpath('.//text()').extract()).strip()
                file_url = info.xpath('./@href').extract_first()
                file_type = extract_file_type(file_name,file_url)
                if file_type:
                    attachment = AttachmentDownloader().fetch_attachment(
                        file_name=file_name, file_type=file_type, download_url=file_url,
                        proxies=request.get_proxies())
                    attachments[str(len(attachments) + 1)] = attachment

        if attachments:
            list_item.projectinfo = {"attachments": attachments}

        yield list_item


if __name__ == '__main__':
    Details(redis_key="lzz:gdsggzyjypt_pwq_jygg").start()