|
|
@@ -18,12 +18,13 @@ class BreadthCrawler:
|
|
|
):
|
|
|
self.enable_query_kw = False
|
|
|
self.enable_query_org = False
|
|
|
- '''同步服务'''
|
|
|
+ '''同步与定时推送数据服务'''
|
|
|
SyncData(
|
|
|
- init_validator=kwargs.get('init_validator', True),
|
|
|
+ init_validator=kwargs.get('init_validator', False),
|
|
|
+ init_collector=kwargs.get('init_collector', True),
|
|
|
loop_interval=kwargs.get('loop_sync_interval', 1200)
|
|
|
)
|
|
|
- '''查询服务'''
|
|
|
+ '''查询搜索词'''
|
|
|
if query_kw_engine is not None:
|
|
|
self._query_kw = QueryKeyWord(
|
|
|
engine=query_kw_engine,
|
|
|
@@ -32,6 +33,7 @@ class BreadthCrawler:
|
|
|
max_query_page=kwargs.get('max_query_page', 3)
|
|
|
)
|
|
|
self.enable_query_kw = True
|
|
|
+ '''查询组织单位'''
|
|
|
if query_org_engine is not None:
|
|
|
self._query_org = QueryOrganization(
|
|
|
engine=query_org_engine,
|
|
|
@@ -39,7 +41,7 @@ class BreadthCrawler:
|
|
|
loop_query_interval=kwargs.get('loop_query_org_interval', 60),
|
|
|
)
|
|
|
self.enable_query_org = True
|
|
|
- '''数据挖掘服务'''
|
|
|
+ '''数据挖掘'''
|
|
|
self._excavator = DataExcavate(
|
|
|
workers=kwargs.get('excavate_workers', 1),
|
|
|
loop_interval=kwargs.get('loop_excavate_interval', 20)
|
