Explorar o código

增加非开标记录失败重采类

dzr hai 2 meses
pai
achega
8cacfddc46

+ 22 - 0
lzz_theme/zgzbtbggfwpt_wagf/zgzbtbggfwpt_wagf_details_retry.py

@@ -28,5 +28,27 @@ class RetrySpider(Spider):
         yield from results
 
 
+class Retry2Spider(Spider):
+
+    def get_tasks(self, sizes, show_debug=False):
+        results = []
+        query = {
+            'parser_name': 'ztpc_zgzbtbggfwpt_wagf',
+            'failed': True,
+            'is_crawl': False,
+            'spidercode': {'$ne': 'a_zgzbtbggfwpt_wasjgf_kbjl'},
+            'retry': {'$lt': 5}
+        }
+        with self.theme_list.find(query, limit=sizes) as cursor:
+            for item in cursor:
+                if show_debug:
+                    logger.debug(item)
+
+                results.append(item)
+
+        yield from results
+
+
 if __name__ == '__main__':
     RetrySpider(sizes=10000, threads=50, interval=0.75).start()
+    Retry2Spider(sizes=10000, threads=50, interval=0.75).start()