更新 bidding标签任务

bidding_tags/config.go

@@ -20,7 +20,8 @@ type MgoConf struct {
 type EnvConf struct {
 	PortraitIndex string
 	PortraitMgo   string
-	Days          int
+	Start         int
+	End           int
 	Spec          string
 	Rulea         string
 	Ruleb         string

bidding_tags/config.toml

@@ -23,7 +23,8 @@
 
 [env]
 
-    days = -1 ## 默认取一天前的数据，
+    start = -2 ## 开始取2天前的数据，
+    end = -1 ## 截止取1天前的数据，
     spec = "0 00 01 * * *"  ## 定时任务，每天凌晨1点执行
     ## 情报_环境采购
     rulea = "A02100400\nA02100401\nA02100402\nA02100403\nA02100404\nA02100405\nA02100406\nA02100407\nA02100408\nA02100409\nA02100410\nA02100411\nA02100412\nA02100413\nA02100414\nA02100415\nA02100416\nA02100417\nA02100418\nA02100419\nA02100420\nA02100499\nA02100603\nA02101800\nA02101900\nA02102700\nA02120503\nA02130400\nA02130500\nA02140100\nA02140200\nA02140300\nA02221100\nA02360000\nA02360100\nA02360200\nA02360300\nA02360400\nA02360500\nA02360600\nA02360700\nA02360800\nA02360900\nA02369900\nA07080118\nC01031900\nC07000000\nC07010000\nC07010100\nC07010200\nC07010300\nC07010400\nC07019900\nC07020000\nC07020100\nC07020101\nC07020102\nC07020103\nC07020104\nC07020105\nC07020106\nC07020199\nC07020200\nC07020201\nC07020202\nC07020203\nC07020204\nC07020205\nC07020206\nC07020299\nC07020300\nC07020301\nC07020302\nC07020303\nC07020304\nC07020305\nC07020399\nC07020400\nC07020401\nC07020402\nC07020403\nC07020404\nC07020405\nC07020406\nC07020407\nC07020408\nC07020499\nC07020500\nC07020600\nC07020700\nC07020800\nC07020900\nC07029900\nC07990000\nC11051700\nC12020000\nC12020100\nC12020200\nC12020300\nC12020400\nC12029900\nC20020800\nC24050000"

bidding_tags/main.go

@@ -99,11 +99,13 @@ func main() {
 // dealInc 处理增量数据
 func dealInc() {
 	now := time.Now()
-	yesterday := time.Date(now.Year(), now.Month(), now.Day()+GF.Env.Days, 0, 0, 0, 0, now.Location())
+	start := time.Date(now.Year(), now.Month(), now.Day()+GF.Env.Start, 0, 0, 0, 0, now.Location())
+	end := time.Date(now.Year(), now.Month(), now.Day()+GF.Env.End, 0, 0, 0, 0, now.Location())
 
 	where := map[string]interface{}{
 		"comeintime": map[string]interface{}{
-			"$gte": yesterday.Unix(),
+			"$gte": start.Unix(),
+			"$lt":  end.Unix(),
 		},
 	}
 	dealTopInformation(where, timeTypeInc)
@@ -113,7 +115,8 @@ func dealInc() {
 func dealAll() {
 	where := map[string]interface{}{
 		"comeintime": map[string]interface{}{
-			"$gte": 1638288000, //2021-12-01
+			"$gte": 1718812800, //2021-06-20
+			"$lt":  1721664000, //2021-07-23
 		},
 	}
 
@@ -168,6 +171,7 @@ func dealTopInformation(where interface{}, timeType int) {
 
 			// 有匹配新的标签，需要更新MongoDB以及ES 数据
 			if hasNew {
+				topinformation = removeDuplicates(topinformation) //去重
 				//ToDo 1.更新MongoDB
 				biddingID := mongodb.BsonIdToSId(tmp["_id"])
 				updateData := map[string]interface{}{
@@ -189,11 +193,11 @@ func dealTopInformation(where interface{}, timeType int) {
 				}
 
 				err := Esa.UpdateDocument("bidding", biddingID, updateData)
-				if err != nil {
+				if err != nil && err.Error() != "Document not updated: noop" {
 					log.Println("esa update err", biddingID, err)
 				}
 				err = Esb.UpdateDocument("bidding", biddingID, updateData)
-				if err != nil {
+				if err != nil && err.Error() != "Document not updated: noop" {
 					log.Println("esb update err", biddingID, err)
 				}
 			}

bidding_tags/tools.go

@@ -11,3 +11,16 @@ func IsInStringArray(str string, arr []string) bool {
 	// 如果找到了则返回 true，否则返回 false
 	return pos < len(arr) && arr[pos] == str
 }
+
+// removeDuplicates 去除字符串数组中的重复元素
+func removeDuplicates(strs []string) []string {
+	result := make([]string, 0)
+	uniqueMap := make(map[string]struct{})
+	for _, str := range strs {
+		if _, ok := uniqueMap[str]; !ok {
+			uniqueMap[str] = struct{}{}
+			result = append(result, str)
+		}
+	}
+	return result
+}