金融资产-列表页.py 3.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990
  1. # -*- coding: utf-8 -*-
  2. """
  3. Created on 2025-05-28
  4. ---------
  5. @summary: 沈阳联合产权交易所
  6. ---------
  7. @author: lzz
  8. """
  9. import feapder
  10. from items.spider_item import MgpListItem
  11. from collections import namedtuple
  12. class Sylhcqjys(feapder.BiddingListSpider):
  13. def start_callback(self):
  14. Menu = namedtuple('Menu', ['channel', 'code','typeone', 'typetwo','typethree', 'crawl_page'])
  15. self.site = "沈阳联合产权交易所"
  16. self.menus = [
  17. Menu('金融资产', 'a_sylhcqjys_jrzc','jrzcGetList', '', '', 1),
  18. ]
  19. self.headers = {
  20. "Accept": "application/json, text/javascript, */*; q=0.01",
  21. "Accept-Language": "zh-CN,zh;q=0.9",
  22. "Cache-Control": "no-cache",
  23. "Connection": "keep-alive",
  24. "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
  25. "Origin": "https://test.sprtc.com",
  26. "Pragma": "no-cache",
  27. "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
  28. "X-Requested-With": "XMLHttpRequest",
  29. }
  30. def start_requests(self):
  31. for menu in self.menus:
  32. start_url = 'https://test.sprtc.com/idtAppServiceV6/exclude/control/getGrid'
  33. yield feapder.Request(url=start_url,item=menu._asdict(),page=1, proxies=False)
  34. def download_midware(self, request):
  35. page = request.page
  36. menu = request.item
  37. data = {
  38. "script": f"{menu.get('typeone')}",
  39. "args": "[]",
  40. "keys": f"[\"{menu.get('typetwo')}\"]",
  41. "replaceMap": "{\"developer\":\"sjsxm\"}",
  42. "pageSize": "20",
  43. "nowPage": f"{page}"
  44. }
  45. request.data = data
  46. request.headers=self.headers
  47. def parse(self, request, response):
  48. menu = request.item
  49. info_list = response.json.get('data').get('DataList')
  50. for info in info_list:
  51. href_id = info.get('UUID').strip()
  52. href = f"https://www.sprtc.com/index/qrtwo.htm?id={href_id}"
  53. title = info.get('BDMC').strip()
  54. create_time = info.get('GPJSSJ').strip()
  55. area = "辽宁" # 省份
  56. city = "沈阳市" # 城市
  57. list_item = MgpListItem() # 存储数据的管道
  58. list_item.href = href # 标书链接
  59. list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 (编辑器定的)
  60. list_item.spidercode = menu.get("code") # 最上方定义的爬虫code(编辑器定的)
  61. list_item.title = title # 标题
  62. list_item.publishtime = create_time # 标书发布时间
  63. list_item.site = self.site
  64. list_item.area = area # 城市默认:全国
  65. list_item.city = city # 城市 默认为空
  66. list_item.unique_key = ('href',)
  67. list_item.parse = "self.detail_get"
  68. list_item.deal_detail = ['//div[@id="formApp"]/div[@id="formScrollWrap"]']
  69. list_item.proxies = False
  70. list_item.parse_url = href
  71. list_item.render_time = 5
  72. yield list_item
  73. if __name__ == "__main__":
  74. Sylhcqjys(redis_key="lzz:Sylhcqjys").start()