| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899 |
- # -*- coding: utf-8 -*-
- """
- Created on 2025-05-19
- ---------
- @summary: 友云采
- ---------
- @author: lzz
- """
- import feapder
- from items.spider_item import BidingListItem
- from collections import namedtuple
- from feapder.utils.tools import timestamp_to_date
- import execjs
- class ZtbpcFeapder(feapder.BiddingListSpider):
- def start_callback(self):
- self.site = "友云采"
- Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
- self.menus = [
- Menu('招标采购', 'a_yyc_zbcg', 1),
- ]
- self.headers = {
- "Accept": "application/json, text/plain, */*",
- "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
- "Connection": "keep-alive",
- "Referer": "https://yc.yonyoucloud.com/opportunity/opportunity?type=1",
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
- }
- def start_requests(self):
- for menu in self.menus:
- start_url = "https://yc.yonyoucloud.com/trade-opportunity/logout/opportunity/queryNoLoginOpportunityList"
- yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
- def download_midware(self, request):
- with open('./yyc_headers.js','r') as fr:
- ex_js = fr.read()
- ctx = execjs.compile(ex_js)
- rid = ctx.call('rid')
- page = request.page
- params = {
- "comprehensive": "1",
- "type": "1",
- "pageSize": "20",
- "pageIndex": f"{page}",
- "_rid": f"{rid}"
- }
- request.headers = self.headers
- request.params = params
- def parse(self, request, response):
- menu = request.item
- info_list = response.json.get('data').get('list')
- for info in info_list:
- hid = info.get('idStr')
- href = f"https://yc.yonyoucloud.com/opportunity/buyOfferDetail/{hid}"
- title = info.get('billName').strip()
- publish_time = timestamp_to_date(int(str(info.get('publishTime'))[:10]))
- area = "全国" # 省份
- city = "" # 城市
- district = "" # 区县
- list_item = BidingListItem() # 存储数据的管道
- list_item.href = href # 标书链接
- list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 （编辑器定的）
- list_item.spidercode = menu.get("code") # 最上方定义的爬虫code（编辑器定的）
- list_item.title = title # 标题
- list_item.publishtime = publish_time # 标书发布时间
- list_item.site = self.site
- list_item.area = area or "全国" # 省份 默认:全国
- list_item.city = city # 城市 默认 为空
- list_item.district = district # 区县 默认 为空
- list_item.unique_key = ('href',)
- list_item.parse = "self.detail_get" # 详情页回调方法
- list_item.deal_detail = [] # 抽取正文xpath
- list_item.proxies = False
- list_item.parse_url = href # 详情页请求地址
- yield list_item
- # 翻页
- request = self.infinite_pages(request, response)
- yield request
- if __name__ == "__main__":
- ZtbpcFeapder(redis_key="lzz:yyc_zhbgg").start()
|
