platform-spiders/a_zgltcgyzbw_bggg/招标信息-详情页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-04-29
---------
@summary: 中国联通采购与招标网
---------
@author: lzz
"""
import feapder
from items.spider_item import DataBakItem
from untils.WebCookiePool import WebCookiePool


headers = {
    "Accept": "*/*",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Authorization": "null",
    "Connection": "keep-alive",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "roleId;": ""
}


class Spider(feapder.BiddingDetailSpider):

    def start_callback(self):
        self.cookie_pool = WebCookiePool(redis_key="zgydcgyzbw_ck",
                                         page_url="http://www.chinaunicombidding.cn/bidInformation",
                                         cookie_key="jqmEwVYRfTEJT")
        self.cookie_pool.user_agent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36")

    def start_requests(self):
        data_lsit = self.get_tasks_by_rabbitmq(limit=50)
        for item in data_lsit:
            request_params = item.get("request_params")
            timeout = request_params.pop('timeout', 10)

            yield feapder.Request(url=item.get("parse_url"),
                                  timeout=timeout,
                                  proxies=False,
                                  callback=eval(item.get("parse")),
                                  item=item,
                                  deal_detail=item.get("deal_detail"),
                                  **request_params)

    def download_midware(self, request):
        request.headers = headers
        request.cookies = self.cookie_pool.get_cookie()

    def validate(self, request, response):
        if response.status_code != 200:
            raise ConnectionRefusedError
        return True

    def detail_get(self, request, response):
        items = request.item
        data_item = DataBakItem(**items)
        html = response.json.get('data').get('annoText')
        data_item.contenthtml = html
        yield data_item

    def exception_request(self, request, response):
        self.cookie_pool.del_cookie(self.cookie_pool.get_cookie())
        yield request


if __name__ == "__main__":
    Spider(redis_key="lzz:zgydcgyzbw_cgxqgs").start()