|
@@ -1,17 +1,14 @@
|
|
|
from crawler import BreadthCrawler
|
|
from crawler import BreadthCrawler
|
|
|
-from crawler.engines import BingSearchEngine, QccSearchEngine
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
def main():
|
|
|
BreadthCrawler(
|
|
BreadthCrawler(
|
|
|
allow_sync_data=True,
|
|
allow_sync_data=True,
|
|
|
allow_query=True,
|
|
allow_query=True,
|
|
|
- keyword_query_engine=BingSearchEngine(),
|
|
|
|
|
- org_query_engine=QccSearchEngine(),
|
|
|
|
|
|
|
+ allow_excavate=True,
|
|
|
url_weight=20,
|
|
url_weight=20,
|
|
|
org_weight=5,
|
|
org_weight=5,
|
|
|
keyword_weight=15,
|
|
keyword_weight=15,
|
|
|
- excavate_depth=3,
|
|
|
|
|
excavate_workers=1,
|
|
excavate_workers=1,
|
|
|
).start()
|
|
).start()
|
|
|
|
|
|
