match_spider/zbytb/main.py

from crawler.spiders.DetailPageSpider import CrawlDetailPageSpider
from crawler.spiders.ListPageSpider import CrawlListPageSpider


def list_page_spider():
    headers = {
        'Host': 'www.zbytb.com',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    }
    CrawlListPageSpider(
        db='py_spider',
        crawl_tab='zbytb_list',
        error_tab='crawl_error',
        enable_proxy=True,
        max_page=20,
        headers=headers,
        allow_show_exception=False
    ).start(workers=4)


def detail_page_spider():
    CrawlDetailPageSpider(
        db='py_spider',
        crawl_tab='zbytb_list',
        save_tab='data_bak',
        error_tab='crawl_error',
    ).start()


if __name__ == '__main__':
    detail_page_spider()