补充耕地指标-列表页.py 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137
  1. # -*- coding: utf-8 -*-
  2. """
  3. Created on 2025-05-13
  4. ---------
  5. @summary: 湛江市公共资源交易平台
  6. ---------
  7. @author: lzz
  8. """
  9. import feapder
  10. from items.spider_item import BidingListItem
  11. from collections import namedtuple
  12. from gdzj_utils import *
  13. import json
  14. class ZtbpcFeapder(feapder.BiddingListSpider):
  15. def start_callback(self):
  16. self.site = "湛江市公共资源交易平台"
  17. Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
  18. self.menus = [
  19. Menu('补充耕地指标', 'gx_zjsggzyjypt_bcgdzb', 1),
  20. ]
  21. def start_requests(self):
  22. for menu in self.menus:
  23. start_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/search/v2/items"
  24. yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
  25. def download_midware(self, request):
  26. page = request.page
  27. data = {
  28. "type": "trading-type",
  29. "openConvert": False,
  30. "keyword": "",
  31. "siteCode": "440800",
  32. "secondType": "P",
  33. "tradingProcess": "",
  34. "thirdType": "[]",
  35. "projectType": "",
  36. "publishStartTime": "",
  37. "publishEndTime": "",
  38. "pageNo": page,
  39. "pageSize": 10
  40. }
  41. en_str = get_enstr(data)
  42. data = json.dumps(data, separators=(',', ':'))
  43. request.data = data
  44. request.headers = {
  45. "Accept": "application/json, text/plain, */*",
  46. "Accept-Language": "zh-CN,zh;q=0.9",
  47. "Cache-Control": "no-cache",
  48. "Connection": "keep-alive",
  49. "Content-Type": "application/json",
  50. "Origin": "https://ygp.gdzwfw.gov.cn",
  51. "Pragma": "no-cache",
  52. "Referer": "https://ygp.gdzwfw.gov.cn/",
  53. "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
  54. "X-Dgi-Req-App": en_str.get('X-Dgi-Req-App'),
  55. "X-Dgi-Req-Nonce": en_str.get('X-Dgi-Req-Nonce'),
  56. "X-Dgi-Req-Signature": en_str.get('X-Dgi-Req-Signature'),
  57. "X-Dgi-Req-Timestamp": en_str.get('X-Dgi-Req-Timestamp'),
  58. }
  59. def parse(self, request, response):
  60. menu = request.item
  61. info_list = response.json.get('data').get('pageData')
  62. for info in info_list:
  63. noticeSecondType = info.get('noticeSecondType')
  64. edition = info.get('edition')
  65. noticeId = info.get('noticeId')
  66. projectCode = info.get('projectCode')
  67. tradingProcess = info.get('tradingProcess')
  68. siteCode = info.get('regionCode')
  69. publishDate = info.get('publishDate')
  70. projectType = info.get('projectType')
  71. params = {
  72. "siteCode": f"{siteCode}",
  73. "tradingType": f"{noticeSecondType}",
  74. "bizCode": f"{tradingProcess}",
  75. "projectCode": f"{projectCode}",
  76. "classify": f"{projectType}"
  77. }
  78. nodeId = get_nodeId(params, proxies=request.proxies()).get(noticeId)
  79. info['nodeId'] = nodeId
  80. href = create_href(info) + f"&nodeId={nodeId}"
  81. title = info.get('noticeTitle').strip()
  82. create_time = deal_time(publishDate)
  83. regionName = info.get('regionName', '').strip()
  84. area = "广东"
  85. city = regionName
  86. list_item = BidingListItem() # 存储数据的管道
  87. list_item.href = href # 标书链接
  88. list_item.unique_key = ('href',)
  89. list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 (编辑器定的)
  90. list_item.spidercode = menu.get("code") # 最上方定义的爬虫code(编辑器定的)
  91. list_item.title = title # 标题
  92. list_item.site = self.site
  93. list_item.publishtime = create_time
  94. list_item.area = area # 城市默认:全国
  95. list_item.city = city # 城市 默认为空
  96. list_item.infoformat = 3
  97. list_item.parse = "self.detail_get" # 详情页回调方法
  98. dparams = {
  99. "nodeId": f"{nodeId}",
  100. "version": f"{edition}",
  101. "tradingType": f"{noticeSecondType}",
  102. "noticeId": f"{noticeId}",
  103. "bizCode": f"{tradingProcess}",
  104. "projectCode": f"{projectCode}",
  105. "siteCode": f"{siteCode}"
  106. }
  107. list_item.request_params = {"params": dparams}
  108. list_item.deal_detail = [] # 抽取正文xpath
  109. list_item.proxies = False
  110. list_item.parse_url = "https://ygp.gdzwfw.gov.cn/ggzy-portal/center/apis/trading-notice/new/detail"
  111. yield list_item
  112. request = self.infinite_pages(request, response)
  113. yield request
  114. if __name__ == "__main__":
  115. ZtbpcFeapder(redis_key="lzz:zjsggzyjypt_gcjs").start()