@@ -1,5 +1,9 @@
+import sys
+
from crawler import BreadthCrawler
+EXCAVATE_WORKERS = sys.argv[0] or 1
def main():
BreadthCrawler(
@@ -7,7 +11,7 @@ def main():
enable_search=False,
enable_query=False,
enable_excavate=True,
- excavate_workers=1,
+ excavate_workers=EXCAVATE_WORKERS,
).start()