update:新增重复键数据删除逻辑

A数据处理/sync_data/backfill_task.py

@@ -14,11 +14,8 @@ from pymongo.errors import DuplicateKeyError
 
 from log import logger
 
-# MONGO_HOST = "172.17.4.87"
-# MONGO_PORT = 27080
-
-MONGO_HOST = "127.0.0.1"
-MONGO_PORT = 27001
+MONGO_HOST = "172.17.4.87"
+MONGO_PORT = 27080
 client = MongoClient(MONGO_HOST, MONGO_PORT)
 
 
@@ -46,8 +43,8 @@ def backfill_data():
             if count % 100 == 0:
                 logger.info(f"[BackFill]放入采集队列{count}条任务")
 
-            coll_name = doc["origin"]
-            document = doc["origin_data"]
+            coll_name = doc["origin"]  # 数据来源的表名
+            document = doc["origin_data"]  # 原始数据
             document["failed_times"] = doc["failed_times"]
             # 清除错误记录字段
             for field in ["save", "pri", "failed", "error", "code"]:
@@ -61,6 +58,7 @@ def backfill_data():
                 listdata_err.delete_one({"_id": doc["_id"]})
             except DuplicateKeyError as e:
                 logger.warning(f"[BackFill]重复键错误:{e.details.get('errmsg')}")
+                listdata_err.delete_one({"_id": doc["_id"]})
 
         logger.info(f"[BackFill]共计放入采集队列{count}条任务")
     finally: