platform-spiders/sn_sxcgyzbw_zhbjggs2/陕西采购与招标网-详情页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-01-03
---------
@summary: 陕西采购与招标网
---------
@author: lzz
"""
import random
import time

import execjs
import feapder
import requests
from items.spider_item import DataBakItem
from untils.attachment import AttachmentDownloader
from untils.tools import get_proxy

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Pragma": "no-cache",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"
}


class Details(feapder.BiddingDetailSpider):
    proxy = get_proxy()

    def start_requests(self):
        data_list = self.get_tasks_by_rabbitmq(limit=50)
        for item in data_list:
            # log.debug(item)
            request_params = item.get("request_params")
            yield feapder.Request(url=item.get("parse_url"),
                                  deal_detail=item.get("deal_detail"),
                                  callback=eval(item.get("parse")),
                                  item=item,
                                  **request_params)

    def detail_get(self, request, response):
        items = request.item
        list_item = DataBakItem(**items)

        html = response.xpath('//div[@class="mian_list"]').extract_first()  # 标书详细内容
        list_item.contenthtml = html

        attachments = {}
        fid = response.xpath('//div[@class="mian_list_03"]/@index').extract_first()
        if fid:
            for i in range(5):
                list_item.contenthtml = "详情请访问原网页！"
                fheaders = {
                    "Accept": "*/*",
                    "Accept-Language": "zh-CN,zh;q=0.9",
                    "Cache-Control": "no-cache",
                    "Connection": "keep-alive",
                    "Content-Length": "0",
                    "Origin": "http://bulletin.sntba.com",
                    "Pragma": "no-cache",
                    "Referer": "http://bulletin.sntba.com/",
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"
                }
                furl = "http://39.107.102.206:8087/permission/getSecretKey"
                resp = requests.post(furl, headers=fheaders, timeout=120, proxies=self.proxy, verify=False)
                pdata = resp.text.strip()

                with open('./sxcgyzbw_file.js', 'r') as f:
                    ex_js = f.read()
                ctx = execjs.compile(ex_js)
                file_url = ctx.call('get_key', pdata, fid)
                file_name = list_item.title
                file_type = "pdf"
                attachment = AttachmentDownloader().fetch_attachment(
                    file_name=file_name,
                    file_type=file_type,
                    download_url=file_url,
                    proxies=self.proxy,
                    is_check=True)
                if attachment.get('size'):
                    attachments[str(len(attachments) + 1)] = attachment
                    break

                time.sleep(random.randint(3, 6))
                self.proxy = get_proxy()
                if i == 4:
                    raise FileExistsError

        if len(attachments) > 0:
            list_item.projectinfo = {"attachments": attachments}

        yield list_item


if __name__ == "__main__":
    Details(redis_key="lzz:sxcgyzbw_zgysgg2").start()