123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101 |
- # -*- coding: utf-8 -*-
- """
- Created on 2025-04-21
- ---------
- @summary: 甘肃省公共资源交易网
- ---------
- @author: lzz
- """
- import feapder
- from items.spider_item import MgpListItem
- from collections import namedtuple
- import re, base64
- class Gssggzyjyw(feapder.BiddingListSpider):
- def start_callback(self):
- Menu = namedtuple('Menu', ['channel', 'code', 'href_param', 'area', 'type', 'crawl_page'])
- self.site = "甘肃省公共资源交易网"
- self.menus = [
- Menu('土地和矿业权-定西', 'gs_gssggzyjyw_tdhkyq_dx', 'newprovince/annogoods/getAnnoList', '621100', 'B', 1),
- ]
- self.headers = {
- "Accept": "text/html, */*; q=0.01",
- "Accept-Language": "zh-CN,zh;q=0.9",
- "Cache-Control": "no-cache",
- "Connection": "keep-alive",
- "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
- "Origin": "http://47.110.59.239:9207",
- "Pragma": "no-cache",
- "Referer": "http://47.110.59.239:9207/f/newprovince/annogoods/list",
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
- "X-Requested-With": "XMLHttpRequest"
- }
- def start_requests(self):
- for menu in self.menus:
- start_url = f'http://47.110.59.239:9207/f/{menu.href_param}'
- yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
- def download_midware(self, request):
- page = request.page
- menu = request.item
- data = {
- "pageNo": f"{page}",
- "pageSize": "10",
- "area": f"{menu.get('area')}",
- "projecttype": f"{menu.get('type')}",
- "prjpropertynewI": "I",
- "prjpropertynewA": "A",
- "prjpropertynewD": "D",
- "prjpropertynewC": "C",
- "prjpropertynewB": "B",
- "prjpropertynewE": "E",
- "prjpropertynewZ": "Z",
- "projectname": ""
- }
- request.data = data
- request.headers = self.headers
- def parse(self, request, response):
- menu = request.item
- info_list = response.xpath('//div[@class="sTradingInformationSelectedBtoList"]/dl|//dl[@class="sDisclosurLeftConDetailList"]')
- for info in info_list:
- href = info.xpath('./dd//a/@href').extract_first()
- title = info.xpath('./dd//a/@title').extract_first("").strip()
- if not title:
- title = "".join(info.xpath('./dd//a/text()').extract()).strip()
- create_time = info.xpath('./dd//i/text()').extract_first().strip()
- area = "甘肃" # 省份
- city = menu.get('channel').split('-')[-1] + "市" # 城市
- list_item = MgpListItem() # 存储数据的管道
- list_item.href = href # 标书链接
- list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 (编辑器定的)
- list_item.spidercode = menu.get("code") # 最上方定义的爬虫code(编辑器定的)
- list_item.title = title # 标题
- list_item.publishtime = create_time # 标书发布时间
- list_item.site = self.site
- list_item.area = area # 城市默认:全国
- list_item.city = city # 城市 默认为空
- dp = href + create_time + title
- list_item.unique_key = (dp,)
- list_item.parse = "self.detail_get"
- list_item.proxies = False
- list_item.parse_url = href
- yield list_item
- # 无限翻页
- request = self.infinite_pages(request, response)
- yield request
- if __name__ == "__main__":
- Gssggzyjyw(redis_key="lzz:gssggzyjyw_tdhkyq").start()
|