成交公告-列表页.py 3.1 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394
  1. # -*- coding: utf-8 -*-
  2. """
  3. Created on 2025-06-02
  4. ---------
  5. @summary: 江苏土地市场网
  6. ---------
  7. @author: lzz
  8. """
  9. import feapder
  10. from items.spider_item import BidingListItem
  11. from collections import namedtuple
  12. from feapder.utils.tools import timestamp_to_date
  13. import json
  14. class ZtbpcFeapder(feapder.BiddingListSpider):
  15. def start_callback(self):
  16. self.site = "江苏土地市场网"
  17. Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
  18. self.menus = [
  19. Menu('成交公告', 'a_jstdscw_cjgg', 1),
  20. ]
  21. self.headers = {
  22. "Accept": "*/*",
  23. "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
  24. "Cache-Control": "no-cache",
  25. "Connection": "keep-alive",
  26. "Content-Type": "application/json",
  27. "Origin": "http://www.landjs.com",
  28. "Pragma": "no-cache",
  29. "Referer": "http://www.landjs.com/tAfficheParcel/bargainParcel",
  30. "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
  31. "X-Requested-With": "XMLHttpRequest"
  32. }
  33. def start_requests(self):
  34. for menu in self.menus:
  35. start_url = "http://www.landjs.com/tAfficheParcel/searchBargainParcel"
  36. yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
  37. def download_midware(self, request):
  38. page = request.page
  39. data = {
  40. "index": page,
  41. "size": 10,
  42. "mrFlag": 2
  43. }
  44. data = json.dumps(data, separators=(',', ':'))
  45. request.data = data
  46. request.headers = self.headers
  47. def parse(self, request, response):
  48. menu = request.item
  49. info_list = response.json.get('bargainParcelList')
  50. for info in info_list:
  51. href = "http://www.landjs.com/tAfficheParcel/detail/bargain/" + info.get('cjgsGuid')
  52. title = info.get('xmMc')
  53. create_time = timestamp_to_date(int(str(info.get('createDate'))[:10]))
  54. area = "江苏"
  55. city = ""
  56. list_item = BidingListItem() # 存储数据的管道
  57. list_item.href = href # 标书链接
  58. list_item.unique_key = ('href',)
  59. list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 (编辑器定的)
  60. list_item.spidercode = menu.get("code") # 最上方定义的爬虫code(编辑器定的)
  61. list_item.title = title # 标题
  62. list_item.site = self.site
  63. list_item.publishtime = create_time
  64. list_item.area = area # 城市默认:全国
  65. list_item.city = city # 城市 默认为空
  66. list_item.parse = "self.detail_get" # 详情页回调方法
  67. list_item.deal_detail = ['//div[@class="mainContent"]']
  68. list_item.parse_url = href # 详情页请求地址
  69. yield list_item
  70. request = self.infinite_pages(request, response)
  71. yield request
  72. if __name__ == "__main__":
  73. ZtbpcFeapder(redis_key="lzz:jstdscw_jydt").start()