# -*- coding: utf-8 -*- """ Created on 2025-01-06 --------- @summary: 东方希望数字化采购平台 --------- @author: lzz """ from collections import namedtuple import feapder from items.spider_item import BidingListItem class Spider(feapder.BiddingListSpider): def start_callback(self): Menu = namedtuple('Menu', ['channel', 'code', 'sourceType', 'purchaseType', 'crawl_page']) self.site = "东方希望数字化采购平台" self.menus = [ Menu('招标公告', 'a_dfxwszhcgpt_zbgg', 'BR', '1', 3), ] self.headers = { "Accept": "application/json, text/javascript, */*; q=0.01", "Accept-Language": "zh-CN,zh;q=0.9", "Cache-Control": "no-cache", "Origin": "https://srm.easthope.cn", "Referer": "https://srm.easthope.cn/", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36", } def start_requests(self): url = "https://srm-gw.easthope.cn/ssrc/v1/53/source-notices/br-list/advance/public" for menu in self.menus: yield feapder.Request(url, item=menu._asdict(), page=1) def download_midware(self, request): page = request.page sourceType = request.item.get('sourceType') purchaseType = request.item.get('purchaseType') params = { "sourceType": f"{sourceType}", "page": f"{page - 1}", "size": "10", "bidTitle": "", "projectType": "", "startDate": "", "endDate": "", "industry": "", "purchaseType": f"{purchaseType}" } request.params = params request.headers = self.headers def parse(self, request, response): menu = request.item info_list = response.json.get('content') for info in info_list: sourceType = info.get('sourceType') noticeId = info.get('noticeId') sourceHeaderId = info.get('sourceHeaderId') advanceNoticeId = info.get('advanceNoticeId') sourceFrom = info.get('sourceFrom') href = f"https://srm.easthope.cn/oauth/public/default/source_notice_advance_detail.html?type={sourceType}&from=ZBXX_HOME¬iceId={noticeId}&sourceHeaderId={sourceHeaderId}&advanceNoticeId={advanceNoticeId}&sourceFrom={sourceFrom}" title = info.get('bidTitle').strip() create_time = info.get('approvedDate') area = "全国" # 省份 city = "" # 城市 list_item = BidingListItem() # 存储数据的管道 list_item.href = href # 标书链接 list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 (编辑器定的) list_item.spidercode = menu.get("code") # 最上方定义的爬虫code(编辑器定的) list_item.title = title # 标题 list_item.publishtime = create_time # 标书发布时间 list_item.site = self.site list_item.area = area # 城市默认:全国 list_item.city = city # 城市 默认为空 list_item.unique_key = ('href', 'title') list_item.parse = "self.detail_get" list_item.deal_detail = ['//div[@class="notice-detail-body"]'] list_item.proxies = False list_item.parse_url = href yield list_item request = self.infinite_pages(request, response) yield request if __name__ == "__main__": Spider(redis_key="lzz:DfxwszhcgptNew").start()