|
|
@@ -21,13 +21,6 @@ Menu = namedtuple('Menu', ['channel', 'code', 'type', 'businessKeyWord'])
|
|
|
|
|
|
class ListPageSpider(feapder.AirSpider):
|
|
|
|
|
|
- __custom_setting__ = dict(
|
|
|
- ITEM_FILTER_ENABLE=True,
|
|
|
- ITEM_FILTER_SETTING=dict(
|
|
|
- filter_type=5
|
|
|
- )
|
|
|
- )
|
|
|
-
|
|
|
@property
|
|
|
def proxy(self):
|
|
|
return swordfish_proxy()
|
|
|
@@ -53,16 +46,16 @@ class ListPageSpider(feapder.AirSpider):
|
|
|
|
|
|
def start_requests(self):
|
|
|
self._proxies = self.proxy
|
|
|
- start_time = '2天'
|
|
|
- page_size = 1000
|
|
|
- # today = datetime.date.today()
|
|
|
- # end_day = today - datetime.timedelta(days=-3)
|
|
|
task_menus = [
|
|
|
Menu('未按数据规范-招标公告', 'a_zgzbtbggfwpt_wasjgf_zbgg', '招标公告', 'tenderBulletin'),
|
|
|
Menu('未按数据规范-开标记录', 'a_zgzbtbggfwpt_wasjgf_kbjl', '开标记录', 'openBidRecord'),
|
|
|
Menu('未按数据规范-评标公示', 'a_zgzbtbggfwpt_wasjgf_pbgs', '评标公示', 'winCandidateBulletin'),
|
|
|
Menu('未按数据规范-中标公告', 'a_zgzbtbggfwpt_wasjgf_zhbgg', '中标公告', 'winBidBulletin'),
|
|
|
]
|
|
|
+ start_time = '2天'
|
|
|
+ page_size = 1000
|
|
|
+ # today = datetime.date.today()
|
|
|
+ # end_day = today - datetime.timedelta(days=-3)
|
|
|
for menu in task_menus:
|
|
|
business_type = menu.type
|
|
|
for page in range(1, 10):
|
