fix:添加

config.yaml

@@ -17,7 +17,16 @@ mongodb:
   replSet: ""
   collection: "bidding"
   collection_back: "bidding_back"
-  projectset_proposed: "projectset_proposed"
+  collection_hot: "bidding_hots"
+  userName: "jyDevGroup"
+  password: "jy@DevGroup"
+
+proposedDb:
+  address: "172.17.4.85:27080"
+  size: 5
+  dbName: "qfw"
+  replSet: ""
+  collection: "projectset_proposed"
   userName: "jyDevGroup"
   password: "jy@DevGroup"
 
@@ -34,7 +43,7 @@ runData:
   compressCron: "0 0 1 * * *" #每天凌晨1点
   columnDataCron: "0 0 4 * * *" #每天凌晨4点
   columnCron: "0 1 8-16 * * *" #八点到16点 每小时执行一次
-  dataSign: "0 0 3 * * *" #每天凌晨3点
+  dataSign: "0 0 8 * * *" #每天凌晨8点
 
 script:
   savePool: 5

db/mgodb.go

@@ -7,20 +7,22 @@ import (
 )
 
 type mgoConf struct {
-	Address             string
-	Size                int
-	DbName              string
-	ReplSet             string
-	UserName            string
-	Password            string
-	Collection          string
-	Collection_back     string
-	Projectset_proposed string
+	Address         string
+	Size            int
+	DbName          string
+	ReplSet         string
+	UserName        string
+	Password        string
+	Collection      string
+	Collection_back string
+	Collection_hot  string
 }
 
 var (
-	MongdbConf mgoConf
-	Mgo        m.MongodbSim
+	MongdbConf      mgoConf
+	Mgo             m.MongodbSim
+	ProposedMgoConf mgoConf
+	ProposedMgo     m.MongodbSim
 )
 
 func init() {
@@ -39,5 +41,19 @@ func init() {
 		ReplSet:     MongdbConf.ReplSet,
 	}
 	Mgo.InitPool()
+
+	err = g.Cfg().MustGet(initCtx, "proposedDb").Struct(&ProposedMgoConf)
+	if err != nil {
+		panic(err)
+	}
+	ProposedMgo = m.MongodbSim{
+		MongodbAddr: ProposedMgoConf.Address,
+		Size:        ProposedMgoConf.Size,
+		DbName:      ProposedMgoConf.DbName,
+		UserName:    ProposedMgoConf.UserName,
+		Password:    ProposedMgoConf.Password,
+		ReplSet:     ProposedMgoConf.ReplSet,
+	}
+	ProposedMgo.InitPool()
 	g.Log().Debug(initCtx, "初始化 mongodb 完成")
 }

main.go

@@ -34,7 +34,7 @@ func main() {
 		cronJobName = append(cronJobName, "ColumnCronJob")
 	}
 
-	//8-16每一小时更新一次
+	//每日8点同步前天数据
 	if dataSignCron := g.Cfg().MustGet(mainCtx, "runData.dataSign").String(); dataSignCron != "" {
 		if _, err := gcron.Add(gctx.New(), g.Cfg().MustGet(mainCtx, "runData.dataSign").String(), func(ctx context.Context) {
 			manager.GobalManager.RunDataSign(ctx)
@@ -43,6 +43,7 @@ func main() {
 		}
 		cronJobName = append(cronJobName, "dataSignCronJob")
 	}
+
 	if len(cronJobName) > 0 {
 		gcron.Start(cronJobName...)
 		select {}

manager/columnCron.go

@@ -2,11 +2,15 @@ package manager
 
 import (
 	date2 "app.yhyue.com/moapp/jybase/date"
+	"app.yhyue.com/moapp/jybase/mongodb"
 	"context"
 	"fmt"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/util/gconv"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"math/rand"
+	"runBidSite/db"
+	"strings"
 	"time"
 )
 
@@ -66,6 +70,10 @@ var (
 	}
 )
 
+const (
+	SeoRedisKey = "jybid_classifyList_seo_id"
+)
+
 func init() {
 	JobPkg = InitColumnBidJob(context.Background())
 }
@@ -85,23 +93,16 @@ func InitColumnBidJob(ctx context.Context) *columnJob {
 		tableName: g.Cfg().MustGet(ctx, "script.columnBid.tableName").String(),
 		//pool:             make(chan *ColumnMsg, g.Cfg().MustGet(context.Background(), "script.cacheSize", 100000).Int()),
 	}
-	StartSeoId = time.Now().Unix() + 205
-	result, err := g.DB().Query(ctx, fmt.Sprintf(`SELECT seo_id
-    	FROM %s 
-    	ORDER BY seo_id DESC LIMIT 1`, job.tableName))
-	if err == nil {
-		StartSeoId = 10000 //首次默认从10000开始
-		if result.Len() > 0 {
-			for _, v := range result.List() {
-				StartSeoId = gconv.Int64(v["seo_id"])
-			}
-		}
+	StartSeoId = 10000 //首次默认从10000开始
+	res, err := g.Redis("other").Get(context.Background(), SeoRedisKey)
+	if err == nil && res.Int64() > 0 {
+		StartSeoId = res.Int64()
 	}
 	g.Log().Infof(ctx, "初始化加载InitColumnBidJob StartSeoId:%d", StartSeoId)
 	return job
 }
 
-func (job *columnJob) Match(msg *FullMsg, bidListId *map[string]bool) *ColumnMsg {
+func (job *columnJob) Match(msg *FullMsg) *ColumnMsg {
 	r := &ColumnMsg{
 		BidId:       msg.Bid_id,
 		Area:        msg.Area,
@@ -113,15 +114,15 @@ func (job *columnJob) Match(msg *FullMsg, bidListId *map[string]bool) *ColumnMsg
 		Class2:      msg.Class2,
 		PublishTime: msg.Publish_time,
 	}
-	return job.verifySave(r, msg.Title, bidListId)
+	return job.verifySave(r, msg.Title)
 }
 
-func (job *columnJob) verifySave(d *ColumnMsg, title string, bidListId *map[string]bool) *ColumnMsg {
+func (job *columnJob) verifySave(d *ColumnMsg, title string) *ColumnMsg {
 	var key columnType
 	//热门
 	key.classify1 = d.Area
 	key.SType = HostBidSort
-	if TimeCheck(key) < job.hotBid && (*bidListId)[d.BidId] {
+	if TimeCheck(key) < job.hotBid {
 		redisStore(key, title)
 		d.SeoId = getSeoId(d.PublishTime)
 		d.Status = -HostBidSort
@@ -130,7 +131,7 @@ func (job *columnJob) verifySave(d *ColumnMsg, title string, bidListId *map[stri
 	//区域行业
 	key.classify2 = d.Class1
 	key.SType = AreaIndustrySort
-	if TimeCheck(key) < job.regionalIndustry && d.Class1 != "" && (*bidListId)[d.BidId] {
+	if TimeCheck(key) < job.regionalIndustry && d.Class1 != "" {
 		redisStore(key, title)
 		d.SeoId = getSeoId(d.PublishTime)
 		d.Status = -AreaIndustrySort
@@ -140,7 +141,7 @@ func (job *columnJob) verifySave(d *ColumnMsg, title string, bidListId *map[stri
 	//区域城市
 	key.classify2 = d.City
 	key.SType = AreaCitySort
-	if TimeCheck(key) < job.regionalCity && d.City != "" && !SpecialCity[key.classify1] && (*bidListId)[d.BidId] {
+	if TimeCheck(key) < job.regionalCity && d.City != "" && !SpecialCity[key.classify1] {
 		redisStore(key, title)
 		d.Status = -AreaCitySort
 		d.SeoId = getSeoId(d.PublishTime)
@@ -182,10 +183,11 @@ func TimeCheck(key columnType) int {
 }
 
 func getSeoId(publishTime int64) string {
-	ptime := gconv.String(publishTime)
+	tm := time.Unix(publishTime, 0)
+	ptime := tm.Format("2006-01-02 15:04:05")
 	pre := ptime[2:4] + ptime[5:7] + ptime[8:10]
 	rand.Seed(time.Now().UnixNano())
-	StartSeoId += int64(rand.Intn(200) + 2)
+	StartSeoId += int64(rand.Intn(100) + 2)
 	return fmt.Sprintf("%s%d", pre, StartSeoId)
 }
 
@@ -212,7 +214,7 @@ func (job *columnJob) ColumnCron(ctx context.Context) {
 			case -HostBidSort: //热门
 				ct.classify1 = d.Area
 				ct.SType = HostBidSort
-				if dataStatistics[ct] < job.hotBid {
+				if dataStatistics[ct] < job.upHotBid {
 					dataStatistics[ct]++
 					d.Status = HostBidSort
 					dArr = append(dArr, d)
@@ -305,3 +307,40 @@ func SaveBidSign(ctx context.Context, arr []*BidSign) {
 		g.Log().Errorf(ctx, "save 异常", err)
 	}
 }
+
+func SaveHotSign(ctx context.Context, ids []string) int {
+	var mgoIds []primitive.ObjectID
+	result, err := g.DB().Query(ctx, fmt.Sprintf(`SELECT bid_id FROM new_bidList  WHERE bid_id in (%s)`, strings.Join(ids, ",")))
+	if err == nil && result.Len() > 0 {
+		for _, v := range result.List() {
+			if bidId := gconv.String(v["bid_id"]); bidId != "" {
+				mgoIds = append(mgoIds, mongodb.StringTOBsonId(bidId))
+			}
+		}
+	}
+	if len(mgoIds) == 0 {
+		return 0
+	}
+	bidData, ok1 := db.Mgo.Find(db.MongdbConf.Collection, map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$in": mgoIds,
+		},
+	}, "", "_id,area,city,nav_column,publishtime", false, -1, -1)
+	if ok1 && len(*bidData) > 0 {
+		var bidSignArr []*BidSign
+		for _, m2 := range *bidData {
+			bs := &BidSign{
+				Bid_id:       mongodb.BsonIdToSId(m2["_id"]),
+				Area:         gconv.String(m2["area"]),
+				City:         gconv.String(m2["city"]),
+				Sign:         convertToBitmap(gconv.Int(m2["nav_column"])),
+				Publish_time: time.Unix(gconv.Int64(m2["publishtime"]), 0),
+			}
+			bidSignArr = append(bidSignArr, bs)
+		}
+		if len(bidSignArr) > 0 {
+			SaveBidSign(ctx, bidSignArr)
+		}
+	}
+	return len(mgoIds)
+}

manager/manager.go

@@ -74,37 +74,66 @@ func (m runManager) RunColumnCron() {
 
 func (m *runManager) RunDataSign(ctx context.Context) {
 	st, et := eTime()
-	GetProjectSign(ctx, st, et)
-	GetBidSign(ctx, st, et)
-	return
+	GetProjectSign(ctx, st, et) //前一天拟建标签
+	GetBidSign(ctx, st, et)     //前一天标讯标签
+	GetHotSign(ctx)             //热门标签
+}
+
+// 热门标签统计
+func GetHotSign(ctx context.Context) {
+	var (
+		total, count int
+	)
+	//当天统计的新增热门数据
+	tm := time.Now()
+	htm := time.Date(tm.Year(), tm.Month(), tm.Day(), 0, 0, 0, 0, tm.Location())
+	data, ok := db.Mgo.Find(db.MongdbConf.Collection_hot, map[string]interface{}{
+		"createtime": map[string]interface{}{
+			"$gte": htm.Unix(),
+		},
+	}, "", "bidding_id", false, -1, -1)
+	if ok && len(*data) > 0 {
+		var ids []string
+		total = len(*data) - 1
+		for k, m := range *data {
+			ids = append(ids, fmt.Sprintf(`'%s'`, gconv.String(m["bidding_id"])))
+			if len(ids) == 100 || (k == total && len(ids) > 0) {
+				count += SaveHotSign(ctx, ids)
+				ids = []string{}
+			}
+		}
+		g.Log().Infof(ctx, "GetHotSign date :%s 总需插入%d， 实际加载%d条数据", time.Now().String(), total+1, count)
+	}
 }
 
 // 项目标签统计
 func GetProjectSign(ctx context.Context, st, et int64) {
-	sess := db.Mgo.GetMgoConn()
-	defer db.Mgo.DestoryMongoConn(sess)
+	sess := db.ProposedMgo.GetMgoConn()
+	defer db.ProposedMgo.DestoryMongoConn(sess)
 	query := bson.M{
 		"nav_column": bson.M{"$exists": true},
-		"pici":       bson.M{"$gt": st, "$lte": et},
+		"pici":       bson.M{"$gte": st, "$lte": et},
 	}
-	g.Log().Infof(ctx, "开始加载执行RunProjectSign:time：%v query:%v", time.Now(), query)
-	it := sess.DB(db.MongdbConf.DbName).C(db.MongdbConf.Projectset_proposed).Find(query).Select(map[string]interface{}{
+	it := sess.DB(db.ProposedMgoConf.DbName).C(db.ProposedMgoConf.Collection).Find(query).Select(map[string]interface{}{
 		"_id":             1,
 		"nature_code":     1,
 		"ownerclass_code": 1,
 		"category_code":   1,
 		"projectname":     1,
 		"nav_column":      1,
-		"lasttime":        1,
+		"firsttime":       1,
 	}).Iter()
 	var (
 		projectSignArr []*ProjectSign
+		total          int
 	)
 	row := map[string]interface{}{}
 	for it.Next(&row) {
 		if row["nav_column"] == nil || gconv.Int(row["nav_column"]) == 0 {
+			row = map[string]interface{}{}
 			continue
 		}
+		total++
 		cMap := gconv.Map(gutil.Copy(row))
 		bs := &ProjectSign{
 			Bid_id:       mongodb.BsonIdToSId(cMap["_id"]),
@@ -113,7 +142,7 @@ func GetProjectSign(ctx context.Context, st, et int64) {
 			Ownerclass:   gconv.String(cMap["ownerclass_code"]),
 			Category:     gconv.String(cMap["category_code"]),
 			Sign:         convertToBitmap(gconv.Int(cMap["nav_column"])),
-			Publish_time: time.Unix(gconv.Int64(cMap["lasttime"]), 0),
+			Publish_time: time.Unix(gconv.Int64(cMap["firsttime"]), 0),
 		}
 		projectSignArr = append(projectSignArr, bs)
 		if len(projectSignArr) > 100 {
@@ -125,6 +154,7 @@ func GetProjectSign(ctx context.Context, st, et int64) {
 	if len(projectSignArr) > 0 {
 		SaveProjectSign(ctx, projectSignArr)
 	}
+	g.Log().Infof(ctx, "GetProjectSign date :%s 总加载%d条数据", time.Now().String(), total)
 }
 
 // 标讯标签统计
@@ -134,9 +164,8 @@ func GetBidSign(ctx context.Context, st, et int64) {
 	defer db.Mgo.DestoryMongoConn(sess)
 	query := bson.M{
 		"nav_column": bson.M{"$exists": true},
-		"pici":       bson.M{"$gt": st, "$lte": et},
+		"pici":       bson.M{"$gte": st, "$lte": et},
 	}
-	g.Log().Infof(ctx, "开始加载执行RunProjectSign:time：%v query:%v", time.Now(), query)
 	it := sess.DB(db.MongdbConf.DbName).C(db.MongdbConf.Collection).Find(query).Select(map[string]interface{}{
 		"_id":             1,
 		"nature_code":     1,
@@ -148,10 +177,12 @@ func GetBidSign(ctx context.Context, st, et int64) {
 	}).Iter()
 	var (
 		bidSignArr []*BidSign
+		total      int
 	)
 	row := map[string]interface{}{}
 	for it.Next(&row) {
 		if row["nav_column"] == nil || gconv.Int(row["nav_column"]) == 0 || !(*bidListId)[mongodb.BsonIdToSId(row["_id"])] {
+			row = map[string]interface{}{}
 			continue
 		}
 		cMap := gconv.Map(gutil.Copy(row))
@@ -162,6 +193,7 @@ func GetBidSign(ctx context.Context, st, et int64) {
 			Sign:         convertToBitmap(gconv.Int(cMap["nav_column"])),
 			Publish_time: time.Unix(gconv.Int64(cMap["publishtime"]), 0),
 		}
+		total++
 		bidSignArr = append(bidSignArr, bs)
 		if len(bidSignArr) > 100 {
 			SaveBidSign(ctx, bidSignArr)
@@ -172,6 +204,7 @@ func GetBidSign(ctx context.Context, st, et int64) {
 	if len(bidSignArr) > 0 {
 		SaveBidSign(ctx, bidSignArr)
 	}
+	g.Log().Infof(ctx, "GetBidSign date :%s 总加载%d条数据", time.Now().String(), total)
 }
 
 func getNewBidList(ctx context.Context) *map[string]bool {
@@ -248,6 +281,7 @@ func (m *runManager) RunColumnData(ctx context.Context) {
 	)
 	for it.Next(&row) {
 		if gconv.Int(row["extracttype"]) == -1 {
+			row = map[string]interface{}{}
 			continue
 		}
 		cMap := gconv.Map(gutil.Copy(row))
@@ -265,7 +299,8 @@ func (m *runManager) RunColumnData(ctx context.Context) {
 			Subtype:      gconv.String(cMap["subtype"]),
 			Publish_time: gconv.Int64(cMap["publishtime"]),
 		}
-		if msg.IsColumnEmpty() {
+		if msg.IsColumnEmpty() || !(*bidListId)[msg.Bid_id] {
+			row = map[string]interface{}{}
 			continue
 		}
 		if subs := gconv.String(cMap["s_subscopeclass"]); subs != "" {
@@ -275,7 +310,7 @@ func (m *runManager) RunColumnData(ctx context.Context) {
 				msg.Class2 = classArr[1]
 			}
 		}
-		d := JobPkg.Match(msg, bidListId)
+		d := JobPkg.Match(msg)
 		if d != nil {
 			total++
 			dataArr = append(dataArr, *d)
@@ -294,7 +329,10 @@ func (m *runManager) RunColumnData(ctx context.Context) {
 	if len(dataArr) > 0 {
 		g.DB().Insert(ctx, JobPkg.tableName, dataArr)
 	}
-	g.Log().Infof(ctx, "总加载%d条数据 pici %v 插入条数：%d query :%v", total, lastId, len(TitleRemoval), query)
+	if StartSeoId > 0 {
+		g.Redis("other").Set(context.Background(), SeoRedisKey, StartSeoId)
+	}
+	g.Log().Infof(ctx, "总加载%d条数据 pici %v 插入条数：%d StartSeoId:%d query :%v", total, lastId, len(TitleRemoval), StartSeoId, query)
 }
 
 type ProjectSign struct {