|
|
@@ -1,20 +1,21 @@
|
|
|
from crawler import BreadthCrawler
|
|
|
-from crawler.engines import BingSearchEngine
|
|
|
+from crawler.engines import BingSearchEngine, QccSearchEngine
|
|
|
|
|
|
|
|
|
def main():
|
|
|
- engines_lst = [BingSearchEngine()]
|
|
|
BreadthCrawler(
|
|
|
init_validator=True,
|
|
|
url_weight=20,
|
|
|
org_weight=5,
|
|
|
keyword_weight=15,
|
|
|
- engines=engines_lst,
|
|
|
- max_query_page=30,
|
|
|
loop_sync_interval=1200,
|
|
|
- loop_query_interval=30,
|
|
|
+ query_kw_engine=BingSearchEngine(),
|
|
|
+ loop_query_kw_interval=10,
|
|
|
+ max_query_page=30,
|
|
|
+ query_org_engine=QccSearchEngine(),
|
|
|
+ loop_query_org_interval=300,
|
|
|
loop_excavate_interval=10,
|
|
|
- excavate_workers=1,
|
|
|
+ excavate_workers=1
|
|
|
).start()
|
|
|
|
|
|
|
