wip:分类数量定时任务

rpc/partnerlib/crontab/crontab.go

@@ -15,5 +15,7 @@ func StartTask() {
 		go service.GetDocinInfoTask(ctx)
 		//更新豆丁docs基本信息
 		go service.UpdateDocinInfoTask(ctx)
+		//文档分类文档数据量更新
+		go service.GetDocsClassTask(ctx)
 	}
 }

rpc/partnerlib/crontab/service/docinInfo.go

@@ -122,7 +122,7 @@ func GetMD5(id, productName, desc, FilePostfix string) string {
 func GetDocFileType(fileSuffix string) (i int) {
 	//1 doc 2 pdf 3 xls 4 ppt 5 txt 6 其他
 	switch strings.ToLower(fileSuffix) {
-	case "doc":
+	case "doc", "docx":
 		i = 1
 	case "pdf":
 		i = 2

rpc/partnerlib/crontab/service/docinUpdate.go

@@ -5,6 +5,7 @@ import (
 	"app.yhyue.com/moapp/jy_docs/rpc/partnerlib/model"
 	sm "app.yhyue.com/moapp/jy_docs/services/model"
 	"app.yhyue.com/moapp/jy_docs/services/partner"
+	"app.yhyue.com/moapp/jybase/common"
 	"app.yhyue.com/moapp/jybase/date"
 	"encoding/json"
 	"fmt"
@@ -29,8 +30,8 @@ func UpdateDocinInfos(b []byte) (err error, lastId int64, expectTotal, actualTot
 					doc.Price = int(v.Price * entity.Multiple)
 					doc.PriceVip = v.PriceVip
 					doc.ProductType = int64(v.Ifcharge + entity.Charge)
-					doc.DownOrUp = v.State
-					if err := partner.DocsUpdate(doc); err == nil {
+					doc.DownOrUp = common.If(v.State == 0, 1, v.State).(int)
+					if err := partner.DocsUpdate(1, doc); err == nil {
 						actualTotal++
 					}
 				}

rpc/partnerlib/crontab/service/docsUpdateClass.go

@@ -0,0 +1,42 @@
+package service
+
+import (
+	"app.yhyue.com/moapp/jy_docs/rpc/partnerlib/entity"
+	jyDocRpcUtil "app.yhyue.com/moapp/jy_docs/services/util"
+	"fmt"
+	"log"
+)
+
+type Result struct {
+	Code string
+	Name string
+}
+
+// 获取分类详情
+func GetClassInfo() (results []*Result) {
+	var (
+		jydb = jyDocRpcUtil.GetJyDocsDB()
+	)
+	err := jydb.Raw(fmt.Sprintf("SELECT dcs.code,dc.name FROM %s dcs LEFT JOIN %s dc ON dcs.code = dc.code AND dcs.state = dc.state ", entity.DocClassStatisticsTable, entity.DocClassTable)).Find(&results).Error
+	if err != nil {
+		log.Println("获取分类信息异常：", err.Error())
+	}
+	return
+}
+
+// 更新分类文档数据量
+func UpdateDocsSize(classInfo *Result) {
+	var (
+		jydb  = jyDocRpcUtil.GetJyDocsDB()
+		count int64
+	)
+	err := jydb.Table(entity.DocTable).Where(" FIND_IN_SET(?,docTags)", classInfo.Name).Count(&count).Error
+	if err != nil {
+		log.Println(fmt.Sprintf("获取%s分类的文档量异常：%s", classInfo.Name, err.Error()))
+	} else if count > 0 {
+		err = jydb.Table(entity.DocClassStatisticsTable).Where("code=?", classInfo.Code).UpdateColumn("total", count).Error
+		if err != nil {
+			log.Println(fmt.Sprintf("获取%s分类的文档量异常：%s", classInfo.Name, err.Error()))
+		}
+	}
+}

rpc/partnerlib/crontab/service/service.go

@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"github.com/gogf/gf/v2/util/gconv"
 	"log"
+	"sync"
 	"time"
 )
 
@@ -33,6 +34,14 @@ func SyncDocinInfo(cron config.Cron) {
 		})
 		b, err, _ = h.HttpFunc()
 		lastId    = startId
+		setCache  = func(cron config.Cron) {
+			//缓存 保存最后一次更新id
+			if b := partner.SetDocsStartId(entity.RedisCode, cron.StartIdKey, cron.StartId, -1); !b {
+				log.Println("同步 Docin 数据 更新最新id缓存异常： ", cron.StartId)
+			} else {
+				log.Println("同步 Docin 数据 更新最新id缓存成功： ", cron.StartId)
+			}
+		}
 	)
 	go h.SaveDocinLogger(b, err, "req")
 	if err == nil {
@@ -48,6 +57,7 @@ func SyncDocinInfo(cron config.Cron) {
 			} else if lastId > 0 {
 				//周期内数据未同步完成
 				cron.StartId = lastId
+				go setCache(cron)
 				if cron.Count == et {
 					if st := Cron.NewDocsList.SleepTime; st > 0 {
 						time.Sleep(time.Duration(st) * time.Millisecond)
@@ -80,10 +90,15 @@ func UpdateDocinInfo(cron config.Cron) {
 		count     = cron.Count
 		startDate = cron.StartDate
 		endDate   = cron.EndDate
+		setCache  = func(cron config.Cron) {
+			//缓存数据处理
+			partner.SetUpdateTaskInfo(entity.RedisCode, cron.StartDateKey, endDate, -1)
+			partner.DelDocsStartId(entity.RedisCode, cron.StartIdKey)
+		}
 	)
 	log.Println("update Docin info task start :", date.NowFormat(date.Date_Full_Layout))
 	if endDate == "" {
-		endDate = date.NowFormat(date.Date_yyyyMMdd)
+		endDate = time.Now().AddDate(0, 0, 1).Format(date.Date_yyyyMMdd)
 	}
 	if startDate == "" || endDate == "" || count == 0 || gconv.Int64(startDate) >= gconv.Int64(endDate) {
 		go warn.SendMsgByWXURL(fmt.Sprintf("timetask 保存 docin 文档失败--参数异常：startId:%d --startDate:%s --endDate:%s--count:%d", startId, startDate, endDate, count))
@@ -110,6 +125,7 @@ func UpdateDocinInfo(cron config.Cron) {
 				go warn.SendMsgByWXURL(fmt.Sprintf("更新豆丁数据到tidb失败：%s,当前任务执行参数：%v", err.Error(), cron))
 			} else if lastId > 0 {
 				cron.StartId = lastId
+				go setCache(cron)
 				//周期内数据未同步完成
 				if count == et {
 					if st := Cron.UpdateDocsList.SleepTime; st > 0 {
@@ -125,9 +141,35 @@ func UpdateDocinInfo(cron config.Cron) {
 	//缓存数据处理
 	partner.SetUpdateTaskInfo(entity.RedisCode, cron.StartDateKey, endDate, -1)
 	partner.DelDocsStartId(entity.RedisCode, cron.StartIdKey)
-	//休眠
-	if st := Cron.UpdateDocsList.SleepTime; st > 0 {
-		time.Sleep(time.Duration(st) * time.Millisecond)
-	}
 	log.Println(fmt.Sprintf("update Docin info task end :%s-当前任务更新 预计数据量：%d 实际数据量：%d", date.NowFormat(date.Date_Full_Layout), entity.UpdateExpectTotal, entity.UpdateActualTotal))
 }
+
+// 文档分类文档数据量更新
+func GetDocsClassTask(ctx context.Context) {
+	defer common.Catch()
+	cron.CrontabByTicker(ctx, Cron.UpdateDocsClass, DocsClassUpdate)
+}
+
+func DocsClassUpdate(cron config.Cron) {
+	log.Println("update Docs class count task start :", date.NowFormat(date.Date_Full_Layout), cron)
+	//获取分类详情
+	if res := GetClassInfo(); len(res) > 0 {
+		var (
+			pool = make(chan bool, 6)
+			wg   = &sync.WaitGroup{}
+		)
+		for _, cv := range res {
+			pool <- true
+			wg.Add(1)
+			go func(cv *Result) {
+				defer func() {
+					wg.Done()
+					<-pool
+				}()
+				UpdateDocsSize(cv)
+			}(cv)
+		}
+		wg.Wait()
+	}
+	log.Println(fmt.Sprintf("update Docs class count task end :%s", date.NowFormat(date.Date_Full_Layout)))
+}

rpc/partnerlib/entity/entity.go

@@ -3,17 +3,19 @@ package entity
 import "sync"
 
 const (
-	DateFormat         = "20060102"
-	RedisCode          = "newother"
-	AppId              = "10000"
-	InBatchesCount     = 20
-	ElasticJYDoc       = "jydoc"
-	Multiple           = 10
-	DocTable           = "doc"
-	DocStatisticsTable = "doc_statistics"
-	DateFullLayout     = "20060102150405"
-	PartnerName        = "docin"
-	DocUserOrderTab    = "doc_user_order"
+	DateFormat              = "20060102"
+	RedisCode               = "newother"
+	AppId                   = "10000"
+	InBatchesCount          = 20
+	ElasticJYDoc            = "jydoc"
+	Multiple                = 10
+	DocTable                = "doc"
+	DocStatisticsTable      = "doc_statistics"
+	DateFullLayout          = "20060102150405"
+	PartnerName             = "docin"
+	DocUserOrderTab         = "doc_user_order"
+	DocClassTable           = "doc_class"
+	DocClassStatisticsTable = "doc_class_statistics"
 )
 
 var (

rpc/partnerlib/etc/crontab.yaml

@@ -1,4 +1,4 @@
-IsRun: false
+IsRun: true
 NewDocsList:
   Name: syncDocs
   First: false
@@ -9,12 +9,17 @@ NewDocsList:
   SleepTime: 10
 UpdateDocsList:
   Name: updateDocs
-  First: true
+  First: false
   Time: "0 0 2 ? * 7" # 豆丁文档更新量比较少，每周六早上两点开始
   StartId: 0 # 首次任务执行：0；以后配置成1
   StartIdKey: docin_update_start_id
   Count: 50
-  StartDate: "20240411" # 首次任务执行：20240411
+  StartDate: "20240610" # 首次任务执行：20240411
   StartDateKey: docin_update_start_date
   EndDate: "" # 默认空
   SleepTime: 10
+UpdateDocsClass:
+  Name: updateClass
+  First: false
+  Time: "0 0 1 1 * *" # 文档分类文档数据量更新，每月一号一点开始
+

rpc/partnerlib/internal/config/crontab.go

@@ -1,9 +1,10 @@
 package config
 
 type CrontabInfo struct {
-	IsRun          bool
-	NewDocsList    Cron
-	UpdateDocsList Cron
+	IsRun           bool
+	NewDocsList     Cron
+	UpdateDocsList  Cron
+	UpdateDocsClass Cron
 }
 
 type Cron struct {
@@ -12,7 +13,7 @@ type Cron struct {
 	Time         string
 	StartId      int64  `json:",optional"`
 	StartIdKey   string `json:",optional"`
-	Count        int
+	Count        int    `json:",optional"`
 	StartDate    string `json:",optional"`
 	StartDateKey string `json:",optional"`
 	EndDate      string `json:",optional"`

rpc/partnerlib/model/partner.go

@@ -7,8 +7,8 @@ type DocinInfoRes struct {
 	Count     int    //请求文档量
 	ListCount int    //列表大小
 	Data      []Docs //列表内容
-	StartDate string //起始修改日期
-	EndDate   string //结束修改日期
+	StartDate int64  //起始修改日期
+	EndDate   int64  //结束修改日期
 }
 
 type Docs struct {
@@ -27,5 +27,5 @@ type Docs struct {
 	DownloadCount int     //下载量
 	Ifcharge      int     //是否付费，0:免费,1:收费
 	Desc          string  //文档详情
-	State         int     //文档状态，0:上架,1:下架
+	State         int     //文档状态，0:上架,1:下架--和剑鱼相反
 }

rpc/partnerlib/service/docDownload.go

@@ -11,6 +11,7 @@ import (
 	"app.yhyue.com/moapp/jyfs/rpc/filesystem"
 	"encoding/json"
 	"fmt"
+	"strings"
 )
 
 type DRes struct {
@@ -23,9 +24,10 @@ func DocDownload(in *partnerlib.UserDownloadRequest) (res *partnerlib.UDRes, err
 	var (
 		b      []byte
 		isJson bool
+		id     = strings.ReplaceAll(in.DocId, fmt.Sprintf("%s-", entity.PartnerName), "")
 	)
 	h := NewHH(IC.I.Docin.Name, IC.I.Docin.Host, IC.I.Docin.DocDownload.Name, IC.I.Docin.DocDownload.Pathname, IC.I.Docin.DocDownload.Method, map[string]interface{}{
-		"productId": in.DocId,
+		"productId": id,
 		"userId":    in.PositionId,
 	})
 	b, err, isJson = h.HttpFunc()
@@ -38,7 +40,6 @@ func DocDownload(in *partnerlib.UserDownloadRequest) (res *partnerlib.UDRes, err
 				res.Msg = entity.Res[res.Msg]
 				return
 			}
-
 		} else {
 			//获取文件信息
 			docInfo := partner.DocsFindOne(in.DocId)
@@ -59,7 +60,7 @@ func DocDownload(in *partnerlib.UserDownloadRequest) (res *partnerlib.UDRes, err
 					RawFileContent: b,
 				}).OssDocId; res.OssDocId != "" {
 					//更新doc
-					if errUpdate := partner.DocsUpdate(model.Doc{OssDocId: res.OssDocId}); errUpdate != nil {
+					if errUpdate := partner.DocsUpdate(0, model.Doc{OssDocId: res.OssDocId}); errUpdate != nil {
 						warn.SendMsgByWXURL(fmt.Sprintf("文档id：%s,文档名称：%s 上传tidb doc 异常：%s", docInfo.Id, docInfo.DocName, err.Error()))
 					}
 					return

services/partner/docsUpdate.go

@@ -8,19 +8,16 @@ import (
 	"log"
 )
 
-func DocsUpdate(doc model.Doc) (err error) {
+func DocsUpdate(state int, doc model.Doc) (err error) {
 	updateMap := map[string]interface{}{"updateDate": doc.UpdateDate.Format(date.Date_Full_Layout)}
-	if doc.Price > 0 {
+	switch state {
+	case 0:
+		updateMap["ossDocId"] = doc.OssDocId
+	case 1:
 		updateMap["price"] = doc.Price
-	}
-	if doc.ProductType > 0 {
 		updateMap["productType"] = doc.ProductType
-	}
-	if doc.ProductType > 0 {
 		updateMap["downOrUp"] = doc.DownOrUp
-	}
-	if doc.OssDocId != "" {
-		updateMap["ossDocId"] = doc.OssDocId
+		updateMap["priceVip"] = doc.PriceVip
 	}
 	err = docRpcUtil.GetJyDocsDB().Table(entity.DocTable).Where("id=?", doc.Id).Updates(updateMap).Error
 	if err != nil {