topic_spider/company/crawler/__init__.py

from concurrent.futures import ThreadPoolExecutor, ALL_COMPLETED, wait

from crawler.spiders import (
    SXSpider,
    GDSpider,
    BJSpider,
    TJSpider,
    HuSpider,
    SHSpider,
    HBSpider,
    SHNDSpider
)
from utils.log import logger


def bj_spider(max_workers):
    return BJSpider().run(True, max_workers)


def tj_spider(max_workers):
    return TJSpider().run(True, max_workers)


def hu_spider(max_workers):
    return HuSpider().run(True, max_workers)


def gd_spider(max_workers):
    return GDSpider().run(True, max_workers)


def sx_spider(max_workers):
    return SXSpider().run(True, max_workers)


def sh_spider(max_workers):
    return SHSpider().run(True, max_workers)


def hb_spider(max_workers):
    return HBSpider().run(True, max_workers)


def sh_nd_spider(max_workers):
    return SHNDSpider().run(True, max_workers)


def activate_spider(max_workers: int = 1):
    futures = []
    with ThreadPoolExecutor(max_workers=max_workers) as Executor:
        # futures.append(Executor.submit(bj_spider, 2))
        # futures.append(Executor.submit(tj_spider, 2))
        # futures.append(Executor.submit(hb_spider, 2))
        futures.append(Executor.submit(hu_spider, 2))
        # futures.append(Executor.submit(gd_spider, 2))
        # futures.append(Executor.submit(sx_spider, 2))
        # futures.append(Executor.submit(sh_spider, 2))
        # futures.append(Executor.submit(sh_nd_spider, 2))
        wait(futures, return_when=ALL_COMPLETED)
    logger.info('[采集]采集完成')