| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192 |
- # -*- coding: utf-8 -*-
- """
- Created on2025-05-20
- ---------
- @summary: 湖南省公共资源交易服务平台新网址
- ---------
- @author: lzz
- """
- import feapder
- from items.spider_item import MgpListItem
- from collections import namedtuple
- class ZtbpcFeapder(feapder.BiddingListSpider):
- def start_callback(self):
- self.site = "湖南省公共资源交易服务平台新网址"
- Menu = namedtuple('Menu', ['channel', 'code', 'crawl_page'])
- self.menus = [
- Menu('政府采购', 'hn_hnsggzyjyfwptxwz_zfcg', 20),
- ]
- self.headers = {
- "Accept": "application/json, text/plain, */*",
- "Accept-Language": "zh-CN,zh;q=0.9",
- "Cache-Control": "no-cache",
- "Connection": "keep-alive",
- "Pragma": "no-cache",
- "Referer": "https://hnsbenji.hnsggzy.com/",
- "Sec-Fetch-Site": "same-origin",
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
- }
- def start_requests(self):
- for menu in self.menus:
- start_url = "https://hnsbenji.hnsggzy.com/tradeApi/governmentPurchase/projectInformation/selectAll"
- yield feapder.Request(url=start_url, item=menu._asdict(), page=1, proxies=False)
- def download_midware(self, request):
- page = request.page
- params = {
- "current": f"{page}",
- "size": "10",
- }
- request.params = params
- request.headers = self.headers
- def parse(self, request, response):
- menu = request.item
- info_list = response.json.get('data').get('records')
- for info in info_list:
- hid = info.get('fileId')
- regionCode = info.get('regionCode')
- href = f"https://hnsbenji.hnsggzy.com/#/resources/projectDetail/governmentPurchase?id={hid}®ionCode={regionCode}"
- title = info.get('noticeName').strip()
- publish_time = info.get('noticeSendTime').split('T')[0]
- area = "湖南" # 省份
- city = "" # 城市
- district = "" # 区县
- list_item = MgpListItem() # 存储数据的管道
- list_item.href = href # 标书链接
- list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 （编辑器定的）
- list_item.spidercode = menu.get("code") # 最上方定义的爬虫code（编辑器定的）
- list_item.title = title # 标题
- list_item.publishtime = publish_time # 标书发布时间
- list_item.site = self.site
- list_item.area = area or "全国" # 省份 默认:全国
- list_item.city = city # 城市 默认 为空
- list_item.district = district # 区县 默认 为空
- list_item.unique_key = ('href',)
- list_item.parse = "self.detail_get" # 详情页回调方法
- list_item.deal_detail = []
- list_item.proxies = False
- list_item.parse_url = f"https://hnsbenji.hnsggzy.com/tradeApi/governmentPurchase/projectInformation/selectProjectInformation/{hid}" # 详情页请求地址
- yield list_item
- # 翻页 criter
- request = self.infinite_pages(request, response)
- yield request
- if __name__ == "__main__":
- ZtbpcFeapder(redis_key="lzz:hnsggzyjyfwptxwz_zfcg").start()
|
