|
@@ -28,5 +28,27 @@ class RetrySpider(Spider):
|
|
|
yield from results
|
|
|
|
|
|
|
|
|
+class Retry2Spider(Spider):
|
|
|
+
|
|
|
+ def get_tasks(self, sizes, show_debug=False):
|
|
|
+ results = []
|
|
|
+ query = {
|
|
|
+ 'parser_name': 'ztpc_zgzbtbggfwpt_wagf',
|
|
|
+ 'failed': True,
|
|
|
+ 'is_crawl': False,
|
|
|
+ 'spidercode': {'$ne': 'a_zgzbtbggfwpt_wasjgf_kbjl'},
|
|
|
+ 'retry': {'$lt': 5}
|
|
|
+ }
|
|
|
+ with self.theme_list.find(query, limit=sizes) as cursor:
|
|
|
+ for item in cursor:
|
|
|
+ if show_debug:
|
|
|
+ logger.debug(item)
|
|
|
+
|
|
|
+ results.append(item)
|
|
|
+
|
|
|
+ yield from results
|
|
|
+
|
|
|
+
|
|
|
if __name__ == '__main__':
|
|
|
RetrySpider(sizes=10000, threads=50, interval=0.75).start()
|
|
|
+ Retry2Spider(sizes=10000, threads=50, interval=0.75).start()
|