|
@@ -1,5 +1,9 @@
|
|
|
|
|
+import sys
|
|
|
|
|
+
|
|
|
from crawler import BreadthCrawler
|
|
from crawler import BreadthCrawler
|
|
|
|
|
|
|
|
|
|
+EXCAVATE_WORKERS = sys.argv[0] or 1
|
|
|
|
|
+
|
|
|
|
|
|
|
|
def main():
|
|
def main():
|
|
|
BreadthCrawler(
|
|
BreadthCrawler(
|
|
@@ -7,7 +11,7 @@ def main():
|
|
|
enable_search=False,
|
|
enable_search=False,
|
|
|
enable_query=False,
|
|
enable_query=False,
|
|
|
enable_excavate=True,
|
|
enable_excavate=True,
|
|
|
- excavate_workers=1,
|
|
|
|
|
|
|
+ excavate_workers=EXCAVATE_WORKERS,
|
|
|
).start()
|
|
).start()
|
|
|
|
|
|
|
|
|
|
|
