topic_spider/find_source/crawler/__init__.py

import threading

from crawler.spiders import SyncData, SearchEngine, DataExcavate


class BreadthCrawler:

    def __init__(self, engines=None, **kwargs):
        SyncData(**kwargs)
        self._engines = SearchEngine(engines, **kwargs)
        self._excavator = DataExcavate(**kwargs)

    def start(self):
        threading.Thread(
            target=self._engines.start,
            name='MainSearchEngine'
        ).start()
        threading.Thread(
            target=self._excavator.start,
            name='MainSearchVisit'
        ).start()