topic_spider/find_source/build_excavate.py

import sys

from crawler import BreadthCrawler

EXCAVATE_WORKERS = int(sys.argv[1]) if len(sys.argv) > 1 else 1


def main():
    BreadthCrawler(
        enable_sync_data=False,
        enable_search=False,
        enable_query=False,
        enable_excavate=True,
        excavate_workers=EXCAVATE_WORKERS,
    ).start()


if __name__ == '__main__':
    main()