| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120 |
- # -*- coding: utf-8 -*-
- """
- Created on 2025-01-17
- ---------
- @summary: 深圳阳光采购平台
- ---------
- @author: lzz
- """
- from collections import namedtuple
- import feapder
- from feapder.utils.tools import timestamp_to_date
- from items.spider_item import BidingListItem
- class Spider(feapder.BiddingListSpider):
- def start_callback(self):
- Menu = namedtuple('Menu', ['channel', 'code', 'com', 'ggLeiXing', 'crawl_page'])
- self.site = "深圳阳光采购平台"
- self.menus = [
- Menu('交易信息-招标', 'gd_szygcgpt_jyxx_zb', 'Purchase', 1, 1),
- Menu('交易信息-招标', 'gd_szygcgpt_jyxx_zb', 'Change', 2, 1),
- Menu('交易信息-招标', 'gd_szygcgpt_jyxx_zb', 'Candidate', 3, 1),
- Menu('交易信息-招标', 'gd_szygcgpt_jyxx_zb', 'CaliTender', 6, 1),
- Menu('交易信息-招标', 'gd_szygcgpt_jyxx_zb', 'Result', 4, 1),
- Menu('交易信息-招标', 'gd_szygcgpt_jyxx_zb', 'Invitation', 5, 1),
- Menu('交易信息-招标', 'gd_szygcgpt_jyxx_zb', 'Project', 7, 1),
- Menu('交易信息-招标', 'gd_szygcgpt_jyxx_zb', 'Contract', 8, 1),
- ]
- self.headers = {
- "Accept": "application/json, text/plain, */*",
- "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
- "Cache-Control": "no-cache",
- "Content-Type": "application/json;charset=UTF-8",
- "Origin": "https://www.szygcgpt.com",
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
- }
- def start_requests(self):
- url = "https://www.szygcgpt.com/app/home/pageGGList.do"
- for menu in self.menus:
- yield feapder.Request(url, item=menu._asdict(), page=1, proxies=False)
- def download_midware(self, request):
- page = request.page
- menu = request.item
- data = {
- "page": page,
- "rows": 100,
- "xmLeiXing": "",
- "caiGouType": 0,
- "ggLeiXing": menu.get('ggLeiXing'),
- "isShiShuGuoQi": "",
- "isZhanLueYingJiWuZi": "",
- "keyWords": ""
- }
- request.json = data
- request.headers = self.headers
- def parse(self, request, response):
- menu = request.item
- info_list = response.json.get('data').get('list')
- for info in info_list:
- guid = info.get('guid')
- ggGuid = info.get('ggGuid')
- bdGuid = info.get('bdGuid')
- ggLeiXing = menu.get('ggLeiXing')
- dataSource = info.get('dataSource')
- href = f"https://www.szygcgpt.com/ygcg/detailTop?com={menu.get('com')}&guid={guid}&ggGuid={ggGuid}&bdGuid={bdGuid}&ggLeiXing={ggLeiXing}&dataSource={dataSource}&type=purchase"
- title = info.get('ggName').strip()
- create_time = timestamp_to_date(int(str(info.get('faBuTime'))[:10]),time_format="%Y-%m-%d")
- area = "广东"
- city = "深圳市"
- list_item = BidingListItem() # 存储数据的管道
- list_item.href = href # 标书链接
- list_item.unique_key = ('href',)
- list_item.channel = menu.get("channel") # 最上方定义的抓取栏目 （编辑器定的）
- list_item.spidercode = menu.get("code") # 最上方定义的爬虫code（编辑器定的）
- list_item.title = title # 标题
- list_item.site = self.site
- list_item.publishtime = create_time
- list_item.area = area # 城市默认:全国
- list_item.city = city # 城市 默认为空
- list_item.parse = "self.detail_get" # 详情页回调方法
- list_item.request_params = {
- "rm_list": [
- '//td[contains(text(),"原公告地址:")]/..',
- '//span[@class="tijiao"]'
- ]
- }
- list_item.deal_detail = ['//div[@class="contentDetail"]'] # 抽取正文xpath
- list_item.parse_url = href
- list_item.files = { # 附件采集规则
- "list_xpath": '//div[@class="contentDetail"]//a[@href]',
- "url_xpath": './@href',
- "name_xpath": './text()',
- "files_type": (
- 'zip', 'docx', 'ftp', 'pdf', 'doc', 'rar', 'gzzb', 'hzzbs',
- 'jpg', 'png', 'zbid', 'xls', 'xlsx', 'swp', 'dwg'
- ), # 需要下载的附件类型
- # "file_type":'pdf', # 默认的附件类型，用于url中未带附件类型的
- "url_key": 'http', # 用于区别连接是否为正常附件连接的url关键词 必须携带，如无可填http
- "host": '', # 需要拼接url的host
- }
- yield list_item
- # 无限翻页
- request = self.infinite_pages(request, response)
- yield request
- if __name__ == "__main__":
- Spider(redis_key="detail:chrome").start()
|
