梁山县-建设工程-列表页.py 3.9 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798
  1. # -*- coding: utf-8 -*-
  2. """
  3. Created on 2025-06-05
  4. ---------
  5. @summary: 济宁市公共资源交易网梁山县-建设工程
  6. ---------
  7. @author: lzz
  8. """
  9. import feapder
  10. from items.spider_item import MgpListItem
  11. from collections import namedtuple
  12. from html import unescape
  13. class Feapder(feapder.BiddingListSpider):
  14. def start_callback(self):
  15. Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'crawl_page'])
  16. self.site = "济宁市公共资源交易网梁山县"
  17. self.menus = [
  18. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '536', 1),
  19. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '503000', 1),
  20. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '503002', 1),
  21. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '504002', 1),
  22. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '503003', 1),
  23. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '516001', 1),
  24. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '511001', 1),
  25. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '513001', 1),
  26. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '517001', 1),
  27. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '531000', 1),
  28. Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '512000', 1),
  29. # Menu('建设工程', 'sd_jnsggzyjyw_lsx_zbxx_py', '512002', 1), # 暂无数据
  30. ]
  31. self.headers = {
  32. "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
  33. "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
  34. "Connection": "keep-alive",
  35. "Upgrade-Insecure-Requests": "1",
  36. "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
  37. }
  38. def start_requests(self):
  39. for menu in self.menus:
  40. start_url = "https://jnggzy.jnzbtb.cn:4430/LiangShan/Posts"
  41. yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
  42. def download_midware(self, request):
  43. page = request.page
  44. menu = request.item
  45. tid = menu.get('tid')
  46. params = {
  47. "cat": tid,
  48. "pn": f"{page}"
  49. }
  50. request.params = params
  51. request.headers = self.headers
  52. def parse(self, request, response):
  53. menu = request.item
  54. info_list = response.xpath('//ul[@class="list-group"]/li')
  55. for info in info_list:
  56. href = info.xpath('./a/@href').extract_first("").strip()
  57. title = unescape("".join(info.xpath('./a/text()').extract()).strip())
  58. create_time = info.xpath('./span/text()').extract_first("").strip()
  59. area = "山东" # 省份
  60. city = "济宁市" # 城市
  61. list_item = MgpListItem() # 存储数据的管道
  62. list_item.href = href # 标书链接
  63. list_item.unique_key = ('title', 'href')
  64. list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 (编辑器定的)
  65. list_item.spidercode = menu.get("code") # 最上方定义的爬虫code(编辑器定的)
  66. list_item.title = title # 标题
  67. list_item.site = self.site
  68. list_item.publishtime = create_time
  69. list_item.area = area # 城市默认:全国
  70. list_item.city = city # 城市 默认为空
  71. list_item.parse = "self.detail_get" # 详情页回调方法
  72. list_item.deal_detail = ['//div[@class="card-body"]']
  73. list_item.proxies = False
  74. list_item.parse_url = href
  75. yield list_item
  76. # 无限翻页
  77. request = self.infinite_pages(request, response)
  78. yield request
  79. if __name__ == "__main__":
  80. Feapder(redis_key="lzz:jnsggzyjyw_lzzqgg").start()