中标公告-列表页.py 3.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105
  1. # -*- coding: utf-8 -*-
  2. """
  3. Created on 2025-05-19
  4. ---------
  5. @summary: 友云采
  6. ---------
  7. @author: lzz
  8. """
  9. import feapder
  10. from items.spider_item import BidingListItem
  11. from collections import namedtuple
  12. from feapder.utils.tools import timestamp_to_date
  13. import execjs
  14. class ZtbpcFeapder(feapder.BiddingListSpider):
  15. def start_callback(self):
  16. self.site = "友云采"
  17. Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
  18. self.menus = [
  19. Menu('中标公告', 'a_yyc_zhbgg', 1),
  20. ]
  21. self.headers = {
  22. "Accept": "application/json, text/plain, */*",
  23. "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
  24. "Cache-Control": "no-cache",
  25. "Connection": "keep-alive",
  26. "Pragma": "no-cache",
  27. "Referer": "https://yc.yonyoucloud.com/opportunity/winningNotice",
  28. "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
  29. }
  30. def start_requests(self):
  31. for menu in self.menus:
  32. start_url = "https://yc.yonyoucloud.com/trade-opportunity/logout/opportunity/queryNoLoginWinBidList"
  33. yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
  34. def download_midware(self, request):
  35. with open('./yyc_headers.js','r') as fr:
  36. ex_js = fr.read()
  37. ctx = execjs.compile(ex_js)
  38. rid = ctx.call('rid')
  39. page = request.page
  40. params = {
  41. "comprehensive": "1",
  42. "pageSize": "20",
  43. "pageIndex": f"{page}",
  44. "_rid": f"{rid}"
  45. }
  46. request.headers = self.headers
  47. request.params = params
  48. def parse(self, request, response):
  49. menu = request.item
  50. info_list = response.json.get('data').get('list')
  51. for info in info_list:
  52. hid = info.get('idStr')
  53. channelType = info.get('channelType')
  54. if channelType == 'yc':
  55. lid = 'winningNotice/noticeDetail'
  56. else:
  57. lid = 'buyOfferDetail'
  58. href = f"https://yc.yonyoucloud.com/opportunity/{lid}/{hid}"
  59. title = info.get('billName').strip()
  60. publish_time = timestamp_to_date(int(str(info.get('publishTime'))[:10]))
  61. area = "全国" # 省份
  62. city = "" # 城市
  63. district = "" # 区县
  64. list_item = BidingListItem() # 存储数据的管道
  65. list_item.href = href # 标书链接
  66. list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 (编辑器定的)
  67. list_item.spidercode = menu.get("code") # 最上方定义的爬虫code(编辑器定的)
  68. list_item.title = title # 标题
  69. list_item.publishtime = publish_time # 标书发布时间
  70. list_item.site = self.site
  71. list_item.area = area or "全国" # 省份 默认:全国
  72. list_item.city = city # 城市 默认 为空
  73. list_item.district = district # 区县 默认 为空
  74. list_item.unique_key = ('href',)
  75. list_item.parse = "self.detail_get" # 详情页回调方法
  76. list_item.deal_detail = [] # 抽取正文xpath
  77. list_item.proxies = False
  78. list_item.parse_url = href # 详情页请求地址
  79. yield list_item
  80. # 翻页
  81. request = self.infinite_pages(request, response)
  82. yield request
  83. if __name__ == "__main__":
  84. ZtbpcFeapder(redis_key="lzz:yyc_zhbgg").start()