platform-spiders/a_obyg_dylygs/单一来源公示-列表页.py

# -*- coding: utf-8 -*-
"""
Created on 2025-04-18
---------
@summary: 欧贝易购
---------
@author: lzz
"""
import feapder
from items.spider_item import MgpListItem
from collections import namedtuple
import json
from create_ck import cr_ck
from untils.tools import get_proxy



class Feapder(feapder.BiddingListSpider):

    def start_callback(self):
        Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])

        self.site = "欧贝易购"

        self.menus = [
            Menu('单一来源公示', 'a_obyg_dylygs', 1),
        ]

        self.headers = {
            "Accept": "*/*",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Authorization": "null",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/json",
            "Eco-Code": "",
            "Origin": "https://www.obei.com.cn",
            "Pragma": "no-cache",
            "Referer": "https://www.obei.com.cn/obei-web-ec-ego/ego/home/noticeList.html?noticeType=1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
            "X-Requested-With": "XMLHttpRequest",
        }

        self.ct = 0
        self.proxy = get_proxy()
        self.cookies = {}

    def start_requests(self):
        for menu in self.menus:
            start_url = "https://www.obei.com.cn/obei-gateway/pc-bff/n/elasticsearch/getNotice"
            yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)

    def download_midware(self, request):
        if not self.cookies:
            self.cookies.update(cr_ck(self.proxy))

        page = request.page
        data = {
            "pageFlag": "addSelect",
            "memo": "obei",
            "noticeType": "3",
            "rfqMethod": "RAQ",
            "publicBiddingFlag": "",
            "ouName": "",
            "sidx": "issueDate",
            "sord": "desc",
            "registrationEndDateStartDate": "",
            "registrationEndDateEndDate": "",
            "quotationEndDateStartDate": "",
            "quotationEndDateEndDate": "",
            "province": "",
            "extendUuCodes": "E21104700,E25711600,E27423800,U12626,UI0283",
            "page": page,
            "rows": 10
        }
        data = json.dumps(data, separators=(',', ':'))
        request.data = data
        request.cookies = self.cookies
        request.headers = self.headers
        request.proxies = self.proxy


    def parse(self, request, response):

        if self.ct > 5:
            return
        if response.status_code != 200:
            self.ct += 1
            self.cookies = {}
            self.proxy = get_proxy()
            yield request
        else:
            self.ct = 0
            menu = request.item
            info_list = response.json.get('list')
            for info in info_list:
                hid = info.get('id')
                rfqMethod = info.get('rfqMethod')
                uuCode = info.get('uuCode')
                href = f"https://www.obei.com.cn/obei-web-ec-ego/ego/rfq/deploy/egoBusinessOpportunity.html#/id={hid}/rfqMethod={rfqMethod}/orgCode={uuCode}/statusFlag=2"
                title = info.get('title').strip().replace('\\t',' ')
                create_time = info.get('issueDate') + ":00"

                area = "河北"    # 省份
                city = ""    # 城市

                list_item = MgpListItem()         # 存储数据的管道
                list_item.href = href             # 标书链接
                list_item.unique_key = ('href',)
                list_item.channel = menu.get("channel")  # 最上方定义的抓取栏目 （编辑器定的）
                list_item.spidercode = menu.get("code")  # 最上方定义的爬虫code（编辑器定的）
                list_item.title = title           # 标题
                list_item.site = self.site
                list_item.publishtime = create_time
                list_item.area = area         # 城市默认:全国
                list_item.city = city         # 城市 默认为空

                list_item.parse = "self.detail_get"  # 详情页回调方法
                list_item.deal_detail = ['//div[@class="left-body"]']  # 抽取正文xpath
                list_item.proxies = False
                list_item.parse_url = href
                list_item.proxies = True

                yield list_item

            # 无限翻页
            request = self.infinite_pages(request, response)
            yield request


if __name__ == "__main__":
    Feapder(redis_key="lzz:obyg_dylygs").start()