from crawler import BreadthCrawler from crawler.search_engine import BingSearchEngine def main(): engines = [ BingSearchEngine() ] BreadthCrawler( allow_load_filter=False, workers=1, ).set_search_engines(engines).start() if __name__ == '__main__': main()