ybw-redis去重

ybw/list_spider.py

@@ -22,12 +22,12 @@ class ListSpider:
     def __init__(self, db: str, crawl_tab: str, crawl_max_page=None):
         self.crawl_menus = [
             # CrawlMenu('企业采购', 'a_ybwcgyzbw_qycg', '7%2C'),
-            # CrawlMenu('政府采购', 'a_ybwcgyzbw_zfcg', '6%2C'),
-            # CrawlMenu('招标预告', 'a_ybwcgyzbw_zbyg', '5%2C'),
+            CrawlMenu('政府采购', 'a_ybwcgyzbw_zfcg', '6%2C'),
+            CrawlMenu('招标预告', 'a_ybwcgyzbw_zbyg', '5%2C'),
             CrawlMenu('中标公示', 'a_ybwcgyzbw_zbgs', '4%2C'),
-            # CrawlMenu('服务招标', 'a_ybwcgyzbw_fwzb', '3%2C'),
-            # CrawlMenu('货物招标', 'a_ybwcgyzbw_hwzb', '2%2C'),
-            # CrawlMenu('工程招标', 'a_ybwcgyzbw_gczb', '1%2C'),
+            CrawlMenu('服务招标', 'a_ybwcgyzbw_fwzb', '3%2C'),
+            CrawlMenu('货物招标', 'a_ybwcgyzbw_hwzb', '2%2C'),
+            CrawlMenu('工程招标', 'a_ybwcgyzbw_gczb', '1%2C'),
         ]
         self.crawl_max_page = crawl_max_page or 1
         self.crawl_tab = mongo_table(db, crawl_tab)