新增单条任务推送

find_source/crawler/q.py

@@ -27,13 +27,14 @@ class RedisQueue:
         keys = sorted(keys, key=lambda x: x.split('-')[-1], reverse=True)
         return keys
 
-    def push_task(self, key, tasks, level=1):
+    def push_task(self, key, tasks, level=1, allow_output_log=False):
         """
         双端队列，左边推进任务
 
         :param key: 键名称
         :param tasks: 任务列表
         :param level: 优先级(int类型)，数值越大优先级越高，默认1
+        :param allow_output_log: 允许输出日志
         :return: 任务队列任务数量
         """
         # 重新定义优先队列的key
@@ -41,7 +42,8 @@ class RedisQueue:
 
         # 序列化任务参数
         tasks = [json.dumps(t, default=lambda obj: obj.__dict__['data']) for t in tasks]
-        logger.info(f'RedisQueue info > the number of push tasks: {len(tasks)}')
+        if allow_output_log:
+            logger.info(f'RedisQueue info > the number of push tasks: {len(tasks)}')
 
         if not tasks:
             return self.get_len(key)
@@ -80,16 +82,18 @@ class RedisQueue:
                     pass
             return results
 
-    def push_task_by_key(self, key, tasks):
+    def push_task_by_key(self, key, tasks, allow_output_log=False):
         """
         通过键名称从左边推进任务
 
         :param key: 键名称
         :param tasks: 推送的任务
+        :param allow_output_log: 允许输出日志
         :return:
         """
         tasks = [json.dumps(t, default=lambda obj: obj.__dict__['data']) for t in tasks]
-        logger.info(f'RedisQueue info > the number of push tasks: {len(tasks)}')
+        if allow_output_log:
+            logger.info(f'RedisQueue info > the number of push tasks: {len(tasks)}')
         if not tasks:
             return self.redis.llen(key)