更新 网站 热门栏目数据

website_columns/config.go

@@ -15,6 +15,7 @@ type MgoConf struct {
 	Username string
 	Password string
 	Size     int
+	Direct   bool
 }
 
 // CronConf 定时任务
@@ -25,6 +26,7 @@ type CronConf struct {
 	Delete  int
 	Columns []string
 	HotNum  int
+	HotDay  int
 }
 
 type EsConf struct {

website_columns/config.toml

@@ -1,16 +1,17 @@
 [mongob]  ## bidding 链接
-#    host = "172.17.145.163:27083"
-#    db = "qfw"
-#    coll = "bidding"        ## 标讯数据标
-#    username = "SJZY_RWbid_ES"
-#    password = "SJZY@B4i4D5e6S"
+    host = "127.0.0.1:27083"
+    db = "qfw"
+    coll = "bidding"        ## 标讯数据标
+    username = "SJZY_RWbid_ES"
+    password = "SJZY@B4i4D5e6S"
+    direct = true
 
     ## 测试环境
-    host = "192.168.3.206:27002"
-    db = "qfw_data"
-    coll = "bidding"        ## 标讯数据标
-    username = "root"
-    password = "root"
+#    host = "192.168.3.206:27002"
+#    db = "qfw_data"
+#    coll = "bidding"        ## 标讯数据标
+#    username = "root"
+#    password = "root"
 
 
 [mongop] ##  项目信息
@@ -38,6 +39,7 @@
         "政府采购","拟在建项目","审批项目","推荐项目","业主委托项目","热门项目","新兴项目",
     "国家级项目","省级项目"]
     hotnum = 25 ## 每天搜索次数
+    hotday = 15 ## 汇总15天的数据
 
 
 

website_columns/init.go

@@ -73,6 +73,7 @@ func InitMgo() {
 		Size:        10,
 		UserName:    GF.MongoB.Username,
 		Password:    GF.MongoB.Password,
+		Direct:      GF.MongoB.Direct,
 	}
 	MgoB.InitPool()
 
@@ -82,6 +83,7 @@ func InitMgo() {
 		Size:        10,
 		UserName:    GF.MongoP.Username,
 		Password:    GF.MongoP.Password,
+		Direct:      GF.MongoP.Direct,
 	}
 	MgoP.InitPool()
 

website_columns/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/net/gclient"
+	"github.com/gogf/gf/v2/os/gtime"
 	"github.com/robfig/cron/v3"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -148,82 +149,99 @@ func dealBidding() {
 
 // getHot 获取热门数据
 func getHot() {
-	var hotMap = make(map[string]bool)
+	var existsMap = make(map[string]bool) //bidding_hots 已经存在的ID
 	//获取已有热门数据
 	hots, _ := MgoB.Find("bidding_hots", nil, nil, map[string]interface{}{"_id": 1}, false, -1, -1)
 	if len(*hots) > 0 {
 		for _, v := range *hots {
-			hotMap[mongodb.BsonIdToSId(v["_id"])] = true
+			existsMap[mongodb.BsonIdToSId(v["_id"])] = true
 		}
 	}
 
-	getRes := make([]string, 0)
-	end := time.Date(2023, 12, 5, 0, 0, 0, 0, time.Local)
-	now := time.Now()
-	var recentDays []string
-	for i := 0; i < 15; i++ {
-		day := now.AddDate(0, 0, -i)
-		// 如果日期在 2023-12-05 及之后，则添加到 recentDays
-		if !day.Before(end) {
-			recentDays = append(recentDays, day.Format("2006-01-02"))
+	var hotMap = make(map[string]int)
+	//1.获取昨日数据
+	file := gtime.Now().AddDate(0, 0, -1).Format("Y-m-d")
+	res := gclient.New().GetContent(context.Background(), "http://172.17.145.164:18880/jyartvisit/"+file+".res")
+	arrs := strings.Split(res, "\n")
+	for _, va := range arrs {
+		vs := strings.Split(va, " ")
+		if len(vs) == 2 {
+			insert := map[string]interface{}{
+				"bidding_id": mongodb.StringTOBsonId(vs[0]),
+				"num":        g.NewVar(vs[1]).Int(),
+				"date":       file,
+			}
+			err := MgoB.InsertOrUpdate(GF.MongoB.DB, "bidding_hot_data", insert)
+			if err != nil {
+				log.Error("getHot", zap.Error(err))
+			}
 		}
 	}
 
-	for _, v := range recentDays {
-		res := gclient.New().GetContent(context.Background(), "http://172.17.145.164:18880/jyartvisit/"+v+".res")
-		arrs := strings.Split(res, "\n")
-		getRes = append(getRes, arrs...)
+	//2. 汇总15天内，符合条件热门数据
+	where := map[string]interface{}{
+		"date": map[string]interface{}{
+			"$gte": gtime.Now().AddDate(0, 0, -GF.Cron.HotDay).Format("Y-m-d"),
+		},
 	}
 
-	for _, v := range getRes {
-		vs := strings.Split(v, " ")
-		if len(vs) == 2 {
+	hotData, _ := MgoB.Find("bidding_hot_data", where, nil, nil, false, -1, -1)
+
+	for _, data := range *hotData {
+		biddingID := util.ObjToString(data["bidding_id"])
+		// 将字符串转换为 ObjectId
+		objectID, err := primitive.ObjectIDFromHex(biddingID)
+		if err != nil {
+			fmt.Println("Error parsing ObjectId:", err)
+			continue
+		}
+		// 从 ObjectId 中提取时间戳
+		timestamp := objectID.Timestamp()
+		// 判断时间是否在最近一年内
+		oneYearAgo := time.Now().AddDate(-1, 0, 0)
+		isWithinOneYear := timestamp.After(oneYearAgo)
+		if !isWithinOneYear {
+			continue
+		}
+		num := util.IntAll(data["num"])
+		if da, ok := hotMap[biddingID]; ok {
+			hotMap[biddingID] = num + da
+		} else {
+			hotMap[biddingID] = num
+		}
+	}
+
+	for k, v := range hotMap {
+		//之前存在过的数据，不再入 bidding_hots
+		if existsMap[k] {
+			continue
+		}
+		if v > GF.Cron.HotNum {
+			bidding, _ := MgoB.FindById("bidding", k, nil)
+			biddingData := *bidding
+			if biddingData == nil {
+				continue
+			}
+
 			insert := map[string]interface{}{
-				"_id":        mongodb.StringTOBsonId(vs[0]),
-				"num":        g.NewVar(vs[1]).Int(),
+				"bidding_id": k,
+				"num":        v,
 				"createtime": time.Now().Unix(),
+				"date":       time.Now().Format("2006-01-02"),
 			}
-			err := MgoB.InsertOrUpdate(GF.MongoB.DB, "bidding_hot_data", insert)
+			err := MgoB.InsertOrUpdate(GF.MongoB.DB, "bidding_hots", insert)
 			if err != nil {
 				log.Error("getHot", zap.Error(err))
 			}
-			if !hotMap[vs[0]] {
-				// 将字符串转换为 ObjectId
-				objectID, err := primitive.ObjectIDFromHex(vs[0])
-				if err != nil {
-					fmt.Println("Error parsing ObjectId:", err)
-					continue
-				}
+			biddingData["hot_data"] = 1
+			rea := TagBidding(biddingData)
+			reb := calculateFlag(rea, columns) //拿到十进制数字，标签栏目结果
 
-				// 从 ObjectId 中提取时间戳
-				timestamp := objectID.Timestamp()
-				//// 计算时间差
-				//timeDifference := time.Now().Sub(timestamp)
-				// 判断时间是否在最近一年内
-				oneYearAgo := time.Now().AddDate(-1, 0, 0)
-				isWithinOneYear := timestamp.After(oneYearAgo)
-				//最近一年数据并且 数量大于100
-				if isWithinOneYear && g.NewVar(vs[1]).Int() > GF.Cron.HotNum {
-					err := MgoB.InsertOrUpdate(GF.MongoB.DB, "bidding_hots", insert)
-					if err != nil {
-						log.Error("getHot", zap.Error(err))
-					}
-					bidding, _ := MgoB.FindById("bidding", vs[0], nil)
-					biddingData := *bidding
-					if biddingData == nil {
-						continue
-					}
-					biddingData["hot_data"] = 1
-					rea := TagBidding(biddingData)
-					reb := calculateFlag(rea, columns) //拿到十进制数字，标签栏目结果
-					log.Info("getHot", zap.Any("reb", reb), zap.String("id", vs[0]))
-					updatePool <- []map[string]interface{}{
-						{"_id": insert["_id"]},
-						{"$set": bson.M{
-							"nav_column": reb,
-						}},
-					}
-				}
+			updatePool <- []map[string]interface{}{
+				{"_id": mongodb.StringTOBsonId(k)},
+				{"$set": bson.M{
+					"nav_column": reb,
+				}},
 			}
 		}
 	}