| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101 |
- # -*- coding: utf-8 -*-
- """
- Created on 2025-01-03
- ---------
- @summary: 陕西采购与招标网
- ---------
- @author: lzz
- """
- import random
- import re
- import time
- from collections import namedtuple
- import feapder
- from items.spider_item import BidingListItem
- class Feapder(feapder.BiddingListSpider):
- def start_callback(self):
- Menu = namedtuple('Menu', ['channel', 'code', 'tid', 'cid', 'crawl_page'])
- self.site = "陕西采购与招标网"
- self.menus = [
- Menu('中标结果公示', 'sn_sxcgyzbw_zhbjggs2', 'result', '90', 5),
- ]
- self.headers = {
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
- "Accept-Language": "zh-CN,zh;q=0.9",
- "Cache-Control": "no-cache",
- "Connection": "keep-alive",
- "Pragma": "no-cache",
- "Upgrade-Insecure-Requests": "1",
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"
- }
- def start_requests(self):
- for menu in self.menus:
- start_url = f"http://bulletin.sntba.com/xxfbcmses/search/{menu.tid}.html"
- yield feapder.Request(url=start_url, item=menu._asdict(), page=1)
- def download_midware(self, request):
- page = request.page
- menu = request.item
- params = {
- "searchDate": "1998-03-25",
- "dates": "300",
- "word": "",
- "categoryId": menu.get('cid'),
- "industryName": "",
- "area": "",
- "status": "",
- "publishMedia": "",
- "sourceInfo": "",
- "showStatus": "",
- "page": f"{page}"
- }
- request.params = params
- request.headers = self.headers
- def parse(self, request, response):
- menu = request.item
- info_list = response.xpath('//table[@class="table_text"]/tr')
- for info in info_list[1:]:
- href_org = info.xpath('./td[1]/a/@href').extract_first().strip()
- href = "".join(re.findall("javascript:urlOpen\('(.*?)'",href_org))
- title = info.xpath('./td[1]/a/@title').extract_first().strip()
- create_time = info.xpath('./td[last()-1]/text()').extract_first().strip()
- if menu.get('code') == "sn_sxcgyzbw_zhbjggs2":
- create_time = info.xpath('./td[last()]/text()').extract_first().strip()
- area = "陕西" # 省份
- city = "" # 城市
- district = "" # 区/县
- list_item = BidingListItem() # 存储数据的管道
- list_item.href = href # 标书链接
- list_item.unique_key = ('href',)
- list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 （编辑器定的）
- list_item.spidercode = menu.get("code") # 最上方定义的爬虫code（编辑器定的）
- list_item.title = title # 标题
- list_item.publishtime = create_time # 标书发布时间
- list_item.site = self.site
- list_item.area = area # 城市默认:全国
- list_item.city = city # 城市 默认为空
- list_item.district = district # 区/县
- list_item.parse = "self.detail_get"
- list_item.deal_detail = ['//div[@class="mian_list"]']
- list_item.parse_url = href
- yield list_item
- time.sleep(random.randint(3, 5))
- request = self.infinite_pages(request, response)
- yield request
- if __name__ == "__main__":
- Feapder(redis_key="lzz:sxcgyzbw_zgysgg2").start()
|
