from crawler import BreadthCrawler from crawler.search_engine import BingSearchEngine def main(): engines_lst = [BingSearchEngine()] BreadthCrawler( allow_load_filter=True, engines=engines_lst, max_search_page=1, visit_workers=1, ).start() if __name__ == '__main__': main()