package service import ( cron "app.yhyue.com/moapp/jy_docs/rpc/partnerlib/crontab/handle" "app.yhyue.com/moapp/jy_docs/rpc/partnerlib/entity" . "app.yhyue.com/moapp/jy_docs/rpc/partnerlib/init" "app.yhyue.com/moapp/jy_docs/rpc/partnerlib/internal/config" "app.yhyue.com/moapp/jy_docs/rpc/partnerlib/service" "app.yhyue.com/moapp/jy_docs/rpc/partnerlib/warn" "app.yhyue.com/moapp/jy_docs/services/partner" "app.yhyue.com/moapp/jybase/common" "app.yhyue.com/moapp/jybase/date" "context" "fmt" "github.com/gogf/gf/v2/util/gconv" "log" "time" ) // 获取豆丁docs基本信息 func GetDocinInfoTask(ctx context.Context) { defer common.Catch() cron.CrontabByTicker(ctx, Cron.NewDocsList, SyncDocinInfo) } func SyncDocinInfo(cron config.Cron) { log.Println("sync Docin info task start :", date.NowFormat(date.Date_Full_Layout)) var ( startId = cron.StartId h = service.NewHH(I.Docin.Name, I.Docin.Host, I.Docin.DocList.Name, I.Docin.DocList.Pathname, I.Docin.DocList.Method, map[string]interface{}{ "startId": startId, "count": Cron.NewDocsList.Count, }) b, err, _ = h.HttpFunc() lastId = startId ) go h.SaveDocinLogger(b, err, "req") if err == nil { if len(b) > 0 { var et, at int //豆丁同步数据到mongo go h.SaveDocinLogger(b, err, "res") err, lastId, et, at = InsertDocinInfos(b) entity.SyncExpectTotal = entity.SyncExpectTotal + et entity.SyncActualTotal = entity.SyncActualTotal + at if err != nil { go warn.SendMsgByWXURL(fmt.Sprintf("同步豆丁数据到tidb失败:%s,当前任务执行参数:%v", err.Error(), cron)) } else if lastId > 0 { //周期内数据未同步完成 cron.StartId = lastId if cron.Count == et { if st := Cron.NewDocsList.SleepTime; st > 0 { time.Sleep(time.Duration(st) * time.Millisecond) } SyncDocinInfo(cron) } } } } else { go warn.SendMsgByWXURL(fmt.Sprintf("获取豆丁数据列表失败:%s,当前任务执行参数:%v", err.Error(), cron)) } //缓存 保存最后一次更新id if b := partner.SetDocsStartId(entity.RedisCode, cron.StartIdKey, cron.StartId, -1); !b { log.Println("同步 Docin 数据 更新最新id缓存异常: ", cron.StartId) } else { log.Println("同步 Docin 数据 更新最新id缓存成功: ", cron.StartId) } log.Println(fmt.Sprintf("sync Docin info task end :%s-当前任务更新 预计数据量:%d 实际数据量:%d", date.NowFormat(date.Date_Full_Layout), entity.SyncExpectTotal, entity.SyncActualTotal)) } // 更新豆丁docs基本信息 func UpdateDocinInfoTask(ctx context.Context) { defer common.Catch() cron.CrontabByTicker(ctx, Cron.UpdateDocsList, UpdateDocinInfo) } func UpdateDocinInfo(cron config.Cron) { var ( startId = cron.StartId count = cron.Count startDate = cron.StartDate endDate = cron.EndDate ) log.Println("update Docin info task start :", date.NowFormat(date.Date_Full_Layout)) if endDate == "" { endDate = date.NowFormat(date.Date_yyyyMMdd) } if startDate == "" || endDate == "" || count == 0 || gconv.Int64(startDate) >= gconv.Int64(endDate) { go warn.SendMsgByWXURL(fmt.Sprintf("timetask 保存 docin 文档失败--参数异常:startId:%d --startDate:%s --endDate:%s--count:%d", startId, startDate, endDate, count)) return } h := service.NewHH(I.Docin.Name, I.Docin.Host, I.Docin.UpdateList.Name, I.Docin.UpdateList.Pathname, I.Docin.UpdateList.Method, map[string]interface{}{ "startId": startId, "count": count, "startDate": startDate, "endDate": endDate, }) b, err, _ := h.HttpFunc() lastId := startId go h.SaveDocinLogger(b, err, "req") if err == nil { if len(b) > 0 { var et, at int //豆丁更新数据到mongo go h.SaveDocinLogger(b, err, "res") err, lastId, et, at = UpdateDocinInfos(b) entity.UpdateExpectTotal = entity.UpdateExpectTotal + et entity.UpdateActualTotal = entity.UpdateActualTotal + at if err != nil { go warn.SendMsgByWXURL(fmt.Sprintf("更新豆丁数据到tidb失败:%s,当前任务执行参数:%v", err.Error(), cron)) } else if lastId > 0 { cron.StartId = lastId //周期内数据未同步完成 if count == et { if st := Cron.UpdateDocsList.SleepTime; st > 0 { time.Sleep(time.Duration(st) * time.Millisecond) } UpdateDocinInfo(cron) } } } } else { go warn.SendMsgByWXURL(fmt.Sprintf("获取豆丁更新数据失败:%s,当前任务执行参数:%v", err.Error(), cron)) } //缓存数据处理 partner.SetUpdateTaskInfo(entity.RedisCode, cron.StartDateKey, endDate, -1) partner.DelDocsStartId(entity.RedisCode, cron.StartIdKey) //休眠 if st := Cron.UpdateDocsList.SleepTime; st > 0 { time.Sleep(time.Duration(st) * time.Millisecond) } log.Println(fmt.Sprintf("update Docin info task end :%s-当前任务更新 预计数据量:%d 实际数据量:%d", date.NowFormat(date.Date_Full_Layout), entity.UpdateExpectTotal, entity.UpdateActualTotal)) }