|
@@ -0,0 +1,32 @@
|
|
|
+# -*- coding: utf-8 -*-
|
|
|
+"""
|
|
|
+Created on 2025-05-06
|
|
|
+---------
|
|
|
+@summary: 中国招标投标公共服务平台[未按规范] - 详情失败重采
|
|
|
+"""
|
|
|
+from spider_detail import Spider
|
|
|
+from utils.log import logger
|
|
|
+
|
|
|
+
|
|
|
+class RetrySpider(Spider):
|
|
|
+
|
|
|
+ def get_tasks(self, sizes, show_debug=False):
|
|
|
+ results = []
|
|
|
+ query = {
|
|
|
+ 'spidercode': 'ztpc_zgzbtbggfwpt_wagf',
|
|
|
+ 'failed': True,
|
|
|
+ 'is_crawl': False,
|
|
|
+ 'retry': {'$lt': 9}
|
|
|
+ }
|
|
|
+ with self.theme_list.find(query, limit=sizes) as cursor:
|
|
|
+ for item in cursor:
|
|
|
+ if show_debug:
|
|
|
+ logger.debug(item)
|
|
|
+
|
|
|
+ results.append(item)
|
|
|
+
|
|
|
+ yield from results
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == '__main__':
|
|
|
+ RetrySpider(sizes=10000, threads=50, interval=0.65).start()
|