Hljzfcgfwgccs.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130
  1. # -*- coding: utf-8 -*-
  2. """
  3. Created on 2024-08-05
  4. ---------
  5. @summary: 黑龙江政府采购服务工程超市
  6. ---------
  7. @author: lzz
  8. """
  9. import json
  10. from collections import namedtuple
  11. import feapder
  12. from feapder.utils.tools import get_month, timestamp_to_date
  13. from items.spider_item import BidingListItem
  14. class Spider(feapder.BiddingListSpider):
  15. def start_callback(self):
  16. Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'cid', 'crawl_page'])
  17. self.site = "黑龙江政府采购服务工程超市"
  18. self.menus = [
  19. Menu('服务工程超市公告-废标终止公告', 'hl_hljzfcgfwgccs_fwgccsgg_fbzzgg', '4', '终止公告', 2),
  20. ]
  21. self.headers = {
  22. "Accept": "application/json, text/plain, */*",
  23. "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
  24. "Cache-Control": "no-cache",
  25. "Connection": "keep-alive",
  26. "Content-Type": "application/json",
  27. "Origin": "https://hljcg.hlj.gov.cn",
  28. "Pragma": "no-cache",
  29. "Referer": "https://hljcg.hlj.gov.cn/fwgccs/publicity?type=3",
  30. "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
  31. "X-Requested-With": "XMLHttpRequest",
  32. "accessToken": "undefined",
  33. "loginToken": "undefined",
  34. "x-auth-token": "undefined"
  35. }
  36. def start_requests(self):
  37. url = "https://hljcg.hlj.gov.cn/fwgccsapi/api/notice/queryNoticeList"
  38. for menu in self.menus:
  39. yield feapder.Request(url, item=menu._asdict(), page=1)
  40. def parse(self, request, response):
  41. menu = request.item
  42. info_list = response.json.get('data').get('data')
  43. for info in info_list:
  44. href_id = info.get('id')
  45. href_type = info.get('type')
  46. t1 = info.get('unitName') or info.get('purchasingInformation') or info.get('purchaser') or ""
  47. href = f"https://hljcg.hlj.gov.cn/fwgccs/publicityDetails?id={href_id}&type={href_type}"
  48. title = info.get('name') or info.get('purchaseDemandName')
  49. title = t1 + title.strip() + menu.get('cid')
  50. create_time = timestamp_to_date(int(str(info.get('time'))[:10]))
  51. area = "黑龙江" # 省份
  52. city = "" # 城市
  53. list_item = BidingListItem() # 存储数据的管道
  54. list_item.href = href # 标书链接
  55. list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 (编辑器定的)
  56. list_item.spidercode = menu.get("code") # 最上方定义的爬虫code(编辑器定的)
  57. list_item.title = title # 标题
  58. list_item.publishtime = create_time # 标书发布时间
  59. list_item.site = self.site
  60. list_item.area = area # 城市默认:全国
  61. list_item.city = city # 城市 默认为空
  62. list_item.unique_key = ("href", "title")
  63. list_item.parse = "self.detail_get"
  64. list_item.proxies = True
  65. list_item.request_params = {"rm_list":['//div[contains(@class,"relevant")]',]}
  66. list_item.deal_detail = ['//div[@class="page-content"]','//div[@class="right-content"]']
  67. list_item.parse_url = href
  68. list_item.render_time = 5
  69. list_item.files = { # 附件采集规则
  70. "list_xpath": '//div[@class="right-content"]//a[@href]',
  71. "url_xpath": './@href',
  72. "name_xpath": './text()',
  73. "files_type": ('zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'hzzbs',
  74. 'jpg', 'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg'), # 需要下载的附件类型
  75. # "file_type":'pdf', # 默认的附件类型,用于url中未带附件类型的
  76. "url_key": 'http', # 用于区别连接是否为正常附件连接的url关键词 必须携带,如无可填http
  77. "host": '', # 需要拼接url的host
  78. }
  79. yield list_item
  80. # 无限翻页设置
  81. request = self.infinite_pages(request, response)
  82. yield request
  83. def download_midware(self, request):
  84. page = request.page
  85. menu = request.item
  86. data = {
  87. "data": {
  88. "businessId": "",
  89. "biddingType": "0",
  90. "purchasingInformation": "",
  91. "transactionSupplierName": "",
  92. "regionVal": [
  93. "黑龙江省",
  94. "",
  95. ""
  96. ],
  97. "requestId": "1531362372728",
  98. "requirementName": "",
  99. "type": menu.get('tid'),
  100. "page": page,
  101. "pageSize": 20,
  102. "province": "黑龙江省",
  103. "city": "",
  104. "region": "",
  105. "startTime": f"{get_month(-3)}",
  106. "endTime": f"{get_month()}"
  107. }
  108. }
  109. data = json.dumps(data, separators=(',', ':'))
  110. request.data = data
  111. request.headers = self.headers
  112. if __name__ == "__main__":
  113. Spider(redis_key="detail:chrome").start()