|
|
@@ -20,31 +20,31 @@ class BreadthCrawler:
|
|
|
self.enable_query_org = False
|
|
|
'''同步与定时推送数据服务'''
|
|
|
SyncData(
|
|
|
- init_validator=kwargs.get('init_validator', False),
|
|
|
- init_collector=kwargs.get('init_collector', True),
|
|
|
- loop_interval=kwargs.get('loop_sync_interval', 1200)
|
|
|
+ init_validator=kwargs.pop('init_validator', False),
|
|
|
+ init_collector=kwargs.pop('init_collector', True),
|
|
|
+ loop_interval=kwargs.pop('loop_sync_interval', 1200)
|
|
|
)
|
|
|
'''查询搜索词'''
|
|
|
if query_kw_engine is not None:
|
|
|
self._query_kw = QueryKeyWord(
|
|
|
engine=query_kw_engine,
|
|
|
- query_workers=kwargs.get('query_kw_workers', 1),
|
|
|
- loop_query_interval=kwargs.get('loop_query_kw_interval', 60),
|
|
|
- max_query_page=kwargs.get('max_query_page', 3)
|
|
|
+ query_workers=kwargs.pop('query_kw_workers', 1),
|
|
|
+ loop_query_interval=kwargs.pop('loop_query_kw_interval', 60),
|
|
|
+ max_query_page=kwargs.pop('max_query_page', 3)
|
|
|
)
|
|
|
self.enable_query_kw = True
|
|
|
'''查询组织单位'''
|
|
|
if query_org_engine is not None:
|
|
|
self._query_org = QueryOrganization(
|
|
|
engine=query_org_engine,
|
|
|
- query_workers=kwargs.get('query_org_workers', 1),
|
|
|
- loop_query_interval=kwargs.get('loop_query_org_interval', 60),
|
|
|
+ query_workers=kwargs.pop('query_org_workers', 1),
|
|
|
+ loop_query_interval=kwargs.pop('loop_query_org_interval', 60),
|
|
|
)
|
|
|
self.enable_query_org = True
|
|
|
'''数据挖掘'''
|
|
|
self._excavator = DataExcavate(
|
|
|
- workers=kwargs.get('excavate_workers', 1),
|
|
|
- loop_interval=kwargs.get('loop_excavate_interval', 20)
|
|
|
+ workers=kwargs.pop('excavate_workers', 1),
|
|
|
+ loop_interval=kwargs.pop('loop_excavate_interval', 20)
|
|
|
)
|
|
|
|
|
|
def start(self):
|
