topic_spider/find_source/crawler/__init__.py

import threading

from crawler.spiders import SyncData, SearchEngine, VisitDomain


class BreadthCrawler:

    def __init__(self, engines=None, visit_workers=1, **kwargs):
        SyncData(**kwargs)
        self._engines = SearchEngine(**kwargs)
        self._engines.set_engines(engines)
        self._excavator = VisitDomain(visit_workers=visit_workers, **kwargs)

    def start(self):
        threading.Thread(
            target=self._engines.start,
            name='MainSearchEngine'
        ).start()
        threading.Thread(
            target=self._excavator.start,
            name='MainSearchVisit'
        ).start()