|
|
@@ -0,0 +1,946 @@
|
|
|
+package main
|
|
|
+
|
|
|
+import (
|
|
|
+ "fmt"
|
|
|
+ mgo "mongodb"
|
|
|
+ qu "qfw/util"
|
|
|
+ "strconv"
|
|
|
+ "sync"
|
|
|
+ "time"
|
|
|
+
|
|
|
+ "github.com/donnie4w/go-logger/logger"
|
|
|
+)
|
|
|
+
|
|
|
+type Task struct {
|
|
|
+ Code string //爬虫代码
|
|
|
+ Site string //站点
|
|
|
+ Channel string //栏目
|
|
|
+ ErrType string //异常类型：6：运行异常;5：下载异常;4：发布时间异常;3：乱码;2：状态码异常;1：数据量异常
|
|
|
+ ErrInfo map[string]map[string]interface{} //异常集合
|
|
|
+ Description string //描述
|
|
|
+}
|
|
|
+
|
|
|
+var (
|
|
|
+ StartTime int64 //上一个工作日的起始时间
|
|
|
+ EndTime int64 //上一个工作日的结束时间
|
|
|
+ TaskMap map[string]*Task //任务集合
|
|
|
+ UpdateStateCron string //每天关闭任务的时间
|
|
|
+ CreateTaskCron string //每天创建任务的时间
|
|
|
+ CloseTaskCron string //每天关闭任务的时间
|
|
|
+ CloseNum int //关闭几天的任务
|
|
|
+ DayNum int //更新数据天数
|
|
|
+ UserTaskNum map[string]map[string]int //记录每人每天新建任务量
|
|
|
+)
|
|
|
+
|
|
|
+//创建任务
|
|
|
+func CreateTaskProcess() {
|
|
|
+ InitInfo() //初始化
|
|
|
+ GetDownloadFailedData() //1、统计spider_highlistdata前一天下载失败的爬虫数据(统计完成后修改状态state:0)
|
|
|
+ GetRegatherFailedData() //2、统计regatherdata前一天重采失败的爬虫数据
|
|
|
+ GetDTPErrData() //3、统计spider_warn异常数据(发布时间异常、乱码)
|
|
|
+ GetStatusCodeErrorData() //4、统计spider_sitecheck 站点异常爬虫(404)
|
|
|
+ GetDownloadNumErrData() //5、统计download前一天下载量异常的爬虫数据(每天1点统计下载量，目前统计完成需要1个小时)
|
|
|
+ SaveResult() //保存统计信息
|
|
|
+ CreateLuaTask() //创建任务
|
|
|
+ SaveUserCreateTaskNum() //保存每人创建的任务量
|
|
|
+}
|
|
|
+
|
|
|
+//初始化
|
|
|
+func InitInfo() {
|
|
|
+ defer qu.Catch()
|
|
|
+ TaskMap = map[string]*Task{}
|
|
|
+ UserTaskNum = map[string]map[string]int{}
|
|
|
+ InitTime() //初始化时间
|
|
|
+}
|
|
|
+
|
|
|
+//关闭任务
|
|
|
+func CloseTask() {
|
|
|
+ qu.Catch()
|
|
|
+ logger.Debug("---清理未更新任务---")
|
|
|
+ decreaseDay, day := 0, 0
|
|
|
+ var cleanDay string
|
|
|
+ for {
|
|
|
+ decreaseDay--
|
|
|
+ weekDay := time.Now().AddDate(0, 0, decreaseDay).Weekday().String()
|
|
|
+ if weekDay != "Saturday" && weekDay != "Sunday" {
|
|
|
+ day++
|
|
|
+ }
|
|
|
+ if day == CloseNum {
|
|
|
+ cleanDay = time.Now().AddDate(0, 0, decreaseDay).Format("2006-01-02")
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+ the_time, _ := time.ParseInLocation(qu.Date_Short_Layout, cleanDay, time.Local)
|
|
|
+ unix_time := the_time.Unix() //凌晨时间戳
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "i_state": 0,
|
|
|
+ "l_complete": map[string]interface{}{
|
|
|
+ "$lt": unix_time + 86400,
|
|
|
+ },
|
|
|
+ }
|
|
|
+ logger.Debug("query:", query)
|
|
|
+ set := map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
+ "i_state": 6,
|
|
|
+ },
|
|
|
+ }
|
|
|
+ MgoE.Update("task", query, set, false, true)
|
|
|
+ logger.Debug("---清理未更新任务完毕---")
|
|
|
+}
|
|
|
+
|
|
|
+//1、统计三级页下载失败数据
|
|
|
+func GetDownloadFailedData() {
|
|
|
+ defer qu.Catch()
|
|
|
+ logger.Debug("---开始统计下载失败信息---")
|
|
|
+ sess := MgoS.GetMgoConn()
|
|
|
+ defer MgoS.DestoryMongoConn(sess)
|
|
|
+ ch := make(chan bool, 5)
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+ lock := &sync.Mutex{}
|
|
|
+ field := map[string]interface{}{
|
|
|
+ "spidercode": 1,
|
|
|
+ "href": 1,
|
|
|
+ "site": 1,
|
|
|
+ "channel": 1,
|
|
|
+ }
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "comeintime": map[string]interface{}{
|
|
|
+ "$gte": StartTime,
|
|
|
+ "$lte": EndTime,
|
|
|
+ },
|
|
|
+ "state": -1,
|
|
|
+ }
|
|
|
+ it := sess.DB("spider").C("spider_highlistdata").Find(&query).Select(&field).Iter()
|
|
|
+ count, _ := sess.DB("spider").C("spider_highlistdata").Find(&query).Count()
|
|
|
+ logger.Debug("共有下载失败数据", count, "条")
|
|
|
+ n := 0
|
|
|
+ //arr := [][]map[string]interface{}{}
|
|
|
+ for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
|
|
|
+ ch <- true
|
|
|
+ wg.Add(1)
|
|
|
+ go func(tmp map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-ch
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+ code := qu.ObjToString(tmp["spidercode"])
|
|
|
+ href := qu.ObjToString(tmp["href"])
|
|
|
+ site := qu.ObjToString(tmp["site"])
|
|
|
+ channel := qu.ObjToString(tmp["channel"])
|
|
|
+ lock.Lock()
|
|
|
+ if t := TaskMap[code]; t != nil {
|
|
|
+ if info := t.ErrInfo["6"]; info != nil {
|
|
|
+ num := qu.IntAll(info["num"])
|
|
|
+ num++
|
|
|
+ info["num"] = num
|
|
|
+ hrefs := info["hrefs"].([]string)
|
|
|
+ if len(hrefs) < 3 {
|
|
|
+ hrefs = append(hrefs, href)
|
|
|
+ info["hrefs"] = hrefs
|
|
|
+ t.Description += href + "\n"
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ t.ErrInfo["6"] = map[string]interface{}{ //ErrInfo新增下载异常信息
|
|
|
+ "num": 1,
|
|
|
+ "hrefs": []string{href},
|
|
|
+ }
|
|
|
+ t.Description += "下载异常：\n" + href + "\n"
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ t := &Task{
|
|
|
+ Code: code,
|
|
|
+ Site: site,
|
|
|
+ Channel: channel,
|
|
|
+ ErrType: "6",
|
|
|
+ ErrInfo: map[string]map[string]interface{}{},
|
|
|
+ Description: "下载异常：\n" + href + "\n",
|
|
|
+ }
|
|
|
+ t.ErrInfo = map[string]map[string]interface{}{
|
|
|
+ "6": map[string]interface{}{
|
|
|
+ "num": 1,
|
|
|
+ "hrefs": []string{href},
|
|
|
+ },
|
|
|
+ }
|
|
|
+ TaskMap[code] = t
|
|
|
+ }
|
|
|
+
|
|
|
+ //更新state状态重新下载
|
|
|
+ // update := []map[string]interface{}{}
|
|
|
+ // update = append(update, map[string]interface{}{"_id": tmp["_id"]})
|
|
|
+ // update = append(update, map[string]interface{}{"$set": map[string]interface{}{"state": 0, "times": 0}})
|
|
|
+ // arr = append(arr, update)
|
|
|
+ // if len(arr) > 500 {
|
|
|
+ // tmps := arr
|
|
|
+ // MgoS.UpdateBulk("spider_highlistdata", tmps...)
|
|
|
+ // arr = [][]map[string]interface{}{}
|
|
|
+ // }
|
|
|
+ lock.Unlock()
|
|
|
+ }(tmp)
|
|
|
+ if n%100 == 0 {
|
|
|
+ qu.Debug("current:", n)
|
|
|
+ }
|
|
|
+ tmp = map[string]interface{}{}
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ // lock.Lock()
|
|
|
+ // if len(arr) > 0 {
|
|
|
+ // MgoS.UpdateBulk("spider_highlistdata", arr...)
|
|
|
+ // arr = [][]map[string]interface{}{}
|
|
|
+ // }
|
|
|
+ // lock.Unlock()
|
|
|
+ logger.Debug("---统计下载失败信息完成---")
|
|
|
+}
|
|
|
+
|
|
|
+//2、统计重采失败数据
|
|
|
+func GetRegatherFailedData() {
|
|
|
+ defer qu.Catch()
|
|
|
+ logger.Debug("---开始统计重采失败信息---")
|
|
|
+ sess := MgoS.GetMgoConn()
|
|
|
+ defer MgoS.DestoryMongoConn(sess)
|
|
|
+ ch := make(chan bool, 5)
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+ lock := &sync.Mutex{}
|
|
|
+ field := map[string]interface{}{
|
|
|
+ "spidercode": 1,
|
|
|
+ "href": 1,
|
|
|
+ "site": 1,
|
|
|
+ "channel": 1,
|
|
|
+ }
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "state": 1,
|
|
|
+ "from": "lua",
|
|
|
+ "comeintime": map[string]interface{}{
|
|
|
+ "$gte": StartTime,
|
|
|
+ "$lte": EndTime,
|
|
|
+ },
|
|
|
+ }
|
|
|
+ it := sess.DB("spider").C("regatherdata").Find(&query).Select(&field).Iter()
|
|
|
+ count, _ := sess.DB("spider").C("regatherdata").Find(&query).Count()
|
|
|
+ logger.Debug("共有重采失败数据", count, "条")
|
|
|
+ n := 0
|
|
|
+ for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
|
|
|
+ ch <- true
|
|
|
+ wg.Add(1)
|
|
|
+ go func(tmp map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-ch
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+ code := qu.ObjToString(tmp["spidercode"])
|
|
|
+ href := qu.ObjToString(tmp["href"])
|
|
|
+ site := qu.ObjToString(tmp["site"])
|
|
|
+ channel := qu.ObjToString(tmp["channel"])
|
|
|
+ lock.Lock()
|
|
|
+ if t := TaskMap[code]; t != nil {
|
|
|
+ if info := t.ErrInfo["5"]; info != nil {
|
|
|
+ num := qu.IntAll(info["num"])
|
|
|
+ num++
|
|
|
+ info["num"] = num
|
|
|
+ hrefs := info["hrefs"].([]string)
|
|
|
+ if len(hrefs) < 3 {
|
|
|
+ hrefs = append(hrefs, href)
|
|
|
+ info["hrefs"] = hrefs
|
|
|
+ t.Description += href + "\n"
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ t.ErrInfo["5"] = map[string]interface{}{ //ErrInfo新增下载异常信息
|
|
|
+ "num": 1,
|
|
|
+ "hrefs": []string{href},
|
|
|
+ }
|
|
|
+ t.Description += "运行报错：\n" + href + "\n"
|
|
|
+ }
|
|
|
+
|
|
|
+ } else {
|
|
|
+ t := &Task{
|
|
|
+ Code: code,
|
|
|
+ Site: site,
|
|
|
+ Channel: channel,
|
|
|
+ ErrType: "5",
|
|
|
+ ErrInfo: map[string]map[string]interface{}{},
|
|
|
+ Description: "运行报错：\n" + href + "\n",
|
|
|
+ }
|
|
|
+ t.ErrInfo = map[string]map[string]interface{}{
|
|
|
+ "5": map[string]interface{}{
|
|
|
+ "num": 1,
|
|
|
+ "hrefs": []string{href},
|
|
|
+ },
|
|
|
+ }
|
|
|
+ TaskMap[code] = t
|
|
|
+ }
|
|
|
+ lock.Unlock()
|
|
|
+ }(tmp)
|
|
|
+ if n%100 == 0 {
|
|
|
+ qu.Debug("current:", n)
|
|
|
+ }
|
|
|
+ tmp = map[string]interface{}{}
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ // for _, task := range TaskMap {
|
|
|
+ // qu.Debug("code:", task.Code)
|
|
|
+ // qu.Debug("site:", task.Site)
|
|
|
+ // qu.Debug("channel:", task.Channel)
|
|
|
+ // qu.Debug("errtype:", task.ErrType)
|
|
|
+ // qu.Debug("description:", task.Description)
|
|
|
+ // qu.Debug("info:", task.ErrInfo)
|
|
|
+ // qu.Debug("-------------------------------------------")
|
|
|
+ // tmap := map[string]interface{}{}
|
|
|
+ // ab, _ := json.Marshal(&task)
|
|
|
+ // json.Unmarshal(ab, &tmap)
|
|
|
+ // MgoE.Save("save_aa", tmap)
|
|
|
+ // }
|
|
|
+ logger.Debug("---统计重采失败信息完成---")
|
|
|
+}
|
|
|
+
|
|
|
+//3、统计detail、title、publishtime异常数据
|
|
|
+func GetDTPErrData() {
|
|
|
+ defer qu.Catch()
|
|
|
+ logger.Debug("---开始统计信息异常数据---")
|
|
|
+ sess := MgoS.GetMgoConn()
|
|
|
+ defer MgoS.DestoryMongoConn(sess)
|
|
|
+ ch := make(chan bool, 5)
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+ lock := &sync.Mutex{}
|
|
|
+ field := map[string]interface{}{
|
|
|
+ "code": 1,
|
|
|
+ "href": 1,
|
|
|
+ "site": 1,
|
|
|
+ "channel": 1,
|
|
|
+ "field": 1,
|
|
|
+ }
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "comeintime": map[string]interface{}{
|
|
|
+ "$gte": StartTime,
|
|
|
+ "$lte": EndTime,
|
|
|
+ },
|
|
|
+ "level": 2, //2：error数据 1：warn数据
|
|
|
+ }
|
|
|
+ it := sess.DB("spider").C("spider_warn").Find(&query).Select(&field).Iter()
|
|
|
+ count, _ := sess.DB("spider").C("spider_warn").Find(&query).Count()
|
|
|
+ logger.Debug("共有信息异常数据", count, "条")
|
|
|
+ n := 0
|
|
|
+ for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
|
|
|
+ ch <- true
|
|
|
+ wg.Add(1)
|
|
|
+ go func(tmp map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-ch
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+ code := qu.ObjToString(tmp["code"])
|
|
|
+ href := qu.ObjToString(tmp["href"])
|
|
|
+ site := qu.ObjToString(tmp["site"])
|
|
|
+ channel := qu.ObjToString(tmp["channel"])
|
|
|
+ field := qu.ObjToString(tmp["field"])
|
|
|
+ errnum := "3" //detail、 title异常
|
|
|
+ destmp := "正文标题异常：\n"
|
|
|
+ if field == "publishtime" { //发布时间异常
|
|
|
+ errnum = "4"
|
|
|
+ destmp = "发布时间异常：\n"
|
|
|
+ }
|
|
|
+ lock.Lock()
|
|
|
+ if t := TaskMap[code]; t != nil {
|
|
|
+ if info := t.ErrInfo[errnum]; info != nil {
|
|
|
+ num := qu.IntAll(info["num"])
|
|
|
+ num++
|
|
|
+ info["num"] = num
|
|
|
+ hrefs := info["hrefs"].([]string)
|
|
|
+ if len(hrefs) < 3 {
|
|
|
+ hrefs = append(hrefs, href)
|
|
|
+ info["hrefs"] = hrefs
|
|
|
+ t.Description += href + "\n"
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ t.ErrInfo[errnum] = map[string]interface{}{
|
|
|
+ "num": 1,
|
|
|
+ "hrefs": []string{href},
|
|
|
+ }
|
|
|
+ t.Description += destmp + href + "\n"
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ t := &Task{
|
|
|
+ Code: code,
|
|
|
+ Site: site,
|
|
|
+ Channel: channel,
|
|
|
+ ErrType: errnum,
|
|
|
+ ErrInfo: map[string]map[string]interface{}{},
|
|
|
+ Description: destmp + href + "\n",
|
|
|
+ }
|
|
|
+ t.ErrInfo = map[string]map[string]interface{}{
|
|
|
+ errnum: map[string]interface{}{
|
|
|
+ "num": 1,
|
|
|
+ "hrefs": []string{href},
|
|
|
+ },
|
|
|
+ }
|
|
|
+ TaskMap[code] = t
|
|
|
+ }
|
|
|
+ lock.Unlock()
|
|
|
+ }(tmp)
|
|
|
+ if n%100 == 0 {
|
|
|
+ qu.Debug("current:", n)
|
|
|
+ }
|
|
|
+ tmp = map[string]interface{}{}
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ logger.Debug("---统计信息异常数据完成---")
|
|
|
+}
|
|
|
+
|
|
|
+//4、状态码404
|
|
|
+func GetStatusCodeErrorData() {
|
|
|
+ defer qu.Catch()
|
|
|
+ logger.Debug("---开始统计栏目地址404数据---")
|
|
|
+ field := map[string]interface{}{
|
|
|
+ "url": 1,
|
|
|
+ "code": 1,
|
|
|
+ "site": 1,
|
|
|
+ "channel": 1,
|
|
|
+ }
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "comeintime": map[string]interface{}{
|
|
|
+ "$gte": StartTime,
|
|
|
+ "$lte": EndTime,
|
|
|
+ },
|
|
|
+ "statuscode": 404,
|
|
|
+ }
|
|
|
+ list, _ := MgoS.Find("spider_sitecheck", query, nil, field, false, -1, -1)
|
|
|
+ logger.Debug("共有404地址", len(*list), "条")
|
|
|
+ for _, tmp := range *list {
|
|
|
+ code := qu.ObjToString(tmp["code"])
|
|
|
+ href := qu.ObjToString(tmp["url"])
|
|
|
+ site := qu.ObjToString(tmp["site"])
|
|
|
+ channel := qu.ObjToString(tmp["channel"])
|
|
|
+ if t := TaskMap[code]; t != nil {
|
|
|
+ t.ErrInfo["2"] = map[string]interface{}{ //ErrInfo新增下载异常信息
|
|
|
+ "num": 404,
|
|
|
+ "hrefs": []string{href},
|
|
|
+ }
|
|
|
+ t.Description += "网站监测：404\n" + href + "\n"
|
|
|
+ } else {
|
|
|
+ t := &Task{
|
|
|
+ Code: code,
|
|
|
+ Site: site,
|
|
|
+ Channel: channel,
|
|
|
+ ErrType: "2",
|
|
|
+ ErrInfo: map[string]map[string]interface{}{},
|
|
|
+ Description: "网站监测：404\n" + href + "\n",
|
|
|
+ }
|
|
|
+ t.ErrInfo = map[string]map[string]interface{}{
|
|
|
+ "2": map[string]interface{}{
|
|
|
+ "num": 404,
|
|
|
+ "hrefs": []string{href},
|
|
|
+ },
|
|
|
+ }
|
|
|
+ TaskMap[code] = t
|
|
|
+ }
|
|
|
+ }
|
|
|
+ logger.Debug("---统计栏目地址404数据完成---")
|
|
|
+}
|
|
|
+
|
|
|
+//5、统计下载量异常数据
|
|
|
+func GetDownloadNumErrData() {
|
|
|
+ defer qu.Catch()
|
|
|
+ logger.Debug("---开始统计下载量异常数据---")
|
|
|
+ sess := MgoS.GetMgoConn()
|
|
|
+ defer MgoS.DestoryMongoConn(sess)
|
|
|
+ ch := make(chan bool, 5)
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+ lock := &sync.Mutex{}
|
|
|
+ field := map[string]interface{}{
|
|
|
+ "downloadNum": 1,
|
|
|
+ "code": 1,
|
|
|
+ "averageDownload": 1,
|
|
|
+ "site": 1,
|
|
|
+ "channel": 1,
|
|
|
+ }
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "isok": false,
|
|
|
+ }
|
|
|
+ it := sess.DB("spider").C("spider_download").Find(&query).Select(&field).Iter()
|
|
|
+ count, _ := sess.DB("spider").C("spider_download").Find(&query).Count()
|
|
|
+ logger.Debug("共有下载量异常数据", count, "条")
|
|
|
+ n := 0
|
|
|
+ arr := [][]map[string]interface{}{}
|
|
|
+ for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
|
|
|
+ ch <- true
|
|
|
+ wg.Add(1)
|
|
|
+ go func(tmp map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-ch
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+ code := qu.ObjToString(tmp["code"])
|
|
|
+ site := qu.ObjToString(tmp["site"])
|
|
|
+ channel := qu.ObjToString(tmp["channel"])
|
|
|
+ average := qu.IntAll(tmp["averageDownload"])
|
|
|
+ date := "" //日期
|
|
|
+ dnum := 0 //下载量
|
|
|
+ for d, n := range tmp["downloadNum"].(map[string]interface{}) {
|
|
|
+ date = d
|
|
|
+ dnum = qu.IntAll(n)
|
|
|
+ }
|
|
|
+ lock.Lock()
|
|
|
+ if t := TaskMap[code]; t != nil {
|
|
|
+ t.ErrInfo["1"] = map[string]interface{}{ //ErrInfo新增下载异常信息
|
|
|
+ "num": dnum,
|
|
|
+ "date": date,
|
|
|
+ "average": average,
|
|
|
+ }
|
|
|
+ t.Description += "下载量异常：\n" + date + "：" + fmt.Sprint(dnum) + "\n"
|
|
|
+ } else {
|
|
|
+ t := &Task{
|
|
|
+ Code: code,
|
|
|
+ Site: site,
|
|
|
+ Channel: channel,
|
|
|
+ ErrType: "1",
|
|
|
+ ErrInfo: map[string]map[string]interface{}{},
|
|
|
+ Description: "下载量异常：\n" + date + "：" + fmt.Sprint(dnum) + "\n",
|
|
|
+ }
|
|
|
+ t.ErrInfo = map[string]map[string]interface{}{
|
|
|
+ "1": map[string]interface{}{
|
|
|
+ "num": dnum,
|
|
|
+ "date": date,
|
|
|
+ "average": average,
|
|
|
+ },
|
|
|
+ }
|
|
|
+ TaskMap[code] = t
|
|
|
+ }
|
|
|
+ //更新isok
|
|
|
+ update := []map[string]interface{}{}
|
|
|
+ update = append(update, map[string]interface{}{"_id": tmp["_id"]})
|
|
|
+ update = append(update, map[string]interface{}{"$set": map[string]interface{}{"isok": true}})
|
|
|
+ arr = append(arr, update)
|
|
|
+ if len(arr) > 500 {
|
|
|
+ tmps := arr
|
|
|
+ MgoS.UpdateBulk("spider_download", tmps...)
|
|
|
+ arr = [][]map[string]interface{}{}
|
|
|
+ }
|
|
|
+ lock.Unlock()
|
|
|
+ }(tmp)
|
|
|
+ if n%100 == 0 {
|
|
|
+ qu.Debug("current:", n)
|
|
|
+ }
|
|
|
+ tmp = map[string]interface{}{}
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ lock.Lock()
|
|
|
+ if len(arr) > 0 {
|
|
|
+ MgoS.UpdateBulk("spider_download", arr...)
|
|
|
+ arr = [][]map[string]interface{}{}
|
|
|
+ }
|
|
|
+ lock.Unlock()
|
|
|
+ logger.Debug("---统计下载量异常数据完成---")
|
|
|
+}
|
|
|
+
|
|
|
+//保存统计信息
|
|
|
+func SaveResult() {
|
|
|
+ defer qu.Catch()
|
|
|
+ logger.Debug("---开始保存信息---")
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+ lock := &sync.Mutex{}
|
|
|
+ ch := make(chan bool, 10)
|
|
|
+ savearr := []map[string]interface{}{}
|
|
|
+ for _, task := range TaskMap {
|
|
|
+ wg.Add(1)
|
|
|
+ ch <- true
|
|
|
+ go func(t *Task) {
|
|
|
+ defer func() {
|
|
|
+ <-ch
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+ result := map[string]interface{}{}
|
|
|
+ result["code"] = t.Code
|
|
|
+ result["site"] = t.Site
|
|
|
+ result["channel"] = t.Channel
|
|
|
+ result["errtype"] = t.ErrType
|
|
|
+ result["errinfo"] = t.ErrInfo
|
|
|
+ result["description"] = t.Description
|
|
|
+ result["comeintime"] = time.Now().Unix()
|
|
|
+ result["updatetime"] = time.Now().Unix()
|
|
|
+ lua, _ := MgoE.FindOne("luaconfig", map[string]interface{}{"code": t.Code})
|
|
|
+ if lua != nil && len(*lua) > 0 {
|
|
|
+ result["modifyid"] = (*lua)["createuserid"]
|
|
|
+ result["modify"] = (*lua)["createuser"]
|
|
|
+ result["event"] = (*lua)["event"]
|
|
|
+ }
|
|
|
+ lock.Lock()
|
|
|
+ if len(result) > 0 {
|
|
|
+ savearr = append(savearr, result)
|
|
|
+ }
|
|
|
+ if len(savearr) > 500 {
|
|
|
+ tmps := savearr
|
|
|
+ MgoE.SaveBulk("luataskinfo", tmps...)
|
|
|
+ savearr = []map[string]interface{}{}
|
|
|
+ }
|
|
|
+ lock.Unlock()
|
|
|
+ }(task)
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ lock.Lock()
|
|
|
+ if len(savearr) > 0 {
|
|
|
+ MgoE.SaveBulk("luataskinfo", savearr...)
|
|
|
+ savearr = []map[string]interface{}{}
|
|
|
+ }
|
|
|
+ lock.Unlock()
|
|
|
+ TaskMap = map[string]*Task{} //重置
|
|
|
+ logger.Debug("---保存信息完成---")
|
|
|
+}
|
|
|
+
|
|
|
+//创建任务
|
|
|
+/*
|
|
|
+1、新任务待确认
|
|
|
+ 1.原任务待确认-->更新原待确认任务的紧急度、最迟完成时间、times；如果times>5则更新为待处理任务；追加信息description、addinfoid
|
|
|
+ 2.原任务待处理-->description、addinfoid信息更新到原任务
|
|
|
+ 3.原任务处理中-->description、addinfoid信息更新到原任务
|
|
|
+ 3.原任务待审核-->新建待确认任务
|
|
|
+ 3.原任务未通过-->更新信息后改为待处理；爬虫状态改为待完成；追加信息description、addinfoid
|
|
|
+
|
|
|
+2、新任务待处理
|
|
|
+ 1.原任务待确认-->关闭原任务；新建待处理任务，追加信息description、addinfoid；
|
|
|
+ 2.原任务待处理-->description、addinfoid信息更新到原任务
|
|
|
+ 3.原任务处理中-->description、addinfoid信息更新到原任务
|
|
|
+ 3.原任务待审核-->新建待确认任务；
|
|
|
+ 3.原任务未通过-->更新信息后改为待处理；爬虫状态改为待完成；追加信息description、addinfoid
|
|
|
+*/
|
|
|
+func CreateLuaTask() {
|
|
|
+ defer qu.Catch()
|
|
|
+ logger.Debug("---开始创建任务---")
|
|
|
+ sess := MgoE.GetMgoConn()
|
|
|
+ defer MgoE.DestoryMongoConn(sess)
|
|
|
+ ch := make(chan bool, 1)
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+ field := map[string]interface{}{
|
|
|
+ "comeintime": 0,
|
|
|
+ "updatetime": 0,
|
|
|
+ }
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "comeintime": map[string]interface{}{
|
|
|
+ "$gte": GetTime(0),
|
|
|
+ },
|
|
|
+ }
|
|
|
+ it := sess.DB("editor").C("luataskinfo").Find(&query).Select(&field).Iter()
|
|
|
+ count, _ := sess.DB("editor").C("luataskinfo").Find(&query).Count()
|
|
|
+ logger.Debug("共有异常爬虫数据量", count, "条")
|
|
|
+ n := 0
|
|
|
+ for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
|
|
|
+ ch <- true
|
|
|
+ wg.Add(1)
|
|
|
+ func(tmp map[string]interface{}) { //目前不用多线程
|
|
|
+ defer func() {
|
|
|
+ <-ch
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+ id := mgo.BsonIdToSId(tmp["_id"])
|
|
|
+ code := qu.ObjToString(tmp["code"])
|
|
|
+ site := qu.ObjToString(tmp["site"])
|
|
|
+ channel := qu.ObjToString(tmp["channel"])
|
|
|
+ description := qu.ObjToString(tmp["description"])
|
|
|
+ errtype := qu.ObjToString(tmp["errtype"])
|
|
|
+ errinfo := tmp["errinfo"].(map[string]interface{})
|
|
|
+ modifyid := qu.ObjToString(tmp["modifyid"])
|
|
|
+ modify := qu.ObjToString(tmp["modify"])
|
|
|
+ event := qu.IntAll(tmp["event"])
|
|
|
+ //初始化一些任务的变量
|
|
|
+ n_istate := 0 //当前任务的状态(默认待处理)
|
|
|
+ n_imin := 0 //最小下载量
|
|
|
+ n_itimes := 0 //任务出现特别紧急的次数
|
|
|
+ n_idn := 0 //下载量
|
|
|
+ n_sdt := "" //下载量对应的日期
|
|
|
+ n_surgency := "1" //紧急程度
|
|
|
+ //
|
|
|
+ dnerr := errinfo["1"]
|
|
|
+ if errtype == "1" && dnerr != nil { //只有任务类型是数据量异常时，才记录数据量信息
|
|
|
+ info := errinfo["1"].(map[string]interface{})
|
|
|
+ n_imin = qu.IntAll(info["average"])
|
|
|
+ n_idn = qu.IntAll(info["num"])
|
|
|
+ n_sdt = qu.ObjToString(info["date"])
|
|
|
+ }
|
|
|
+ switch errtype {
|
|
|
+ case "6", "5": //下载异常、重采异常
|
|
|
+ info := errinfo[errtype].(map[string]interface{})
|
|
|
+ num := qu.IntAll(info["num"])
|
|
|
+ if num > 5 || (len(errinfo) == 2 && dnerr == nil) || len(errinfo) >= 3 {
|
|
|
+ n_istate = 1
|
|
|
+ n_surgency = "4"
|
|
|
+ }
|
|
|
+ case "4", "3", "2": //下载报错、发布时间异常、detail、title异常、404
|
|
|
+ n_istate = 1
|
|
|
+ n_surgency = "4"
|
|
|
+ case "1": //下载量异常
|
|
|
+ n_istate = 0
|
|
|
+ n_surgency = "1"
|
|
|
+ }
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "s_code": code,
|
|
|
+ "i_state": map[string]interface{}{
|
|
|
+ "$in": []int{0, 1, 2, 3, 5},
|
|
|
+ },
|
|
|
+ }
|
|
|
+ list, _ := MgoE.Find("task", query, nil, nil, false, -1, -1)
|
|
|
+ if list != nil && len(*list) > 0 { //已有任务
|
|
|
+ task := (*list)[0] //记录已有任务的集合
|
|
|
+ if len(*list) > 2 { //一个爬虫不能同时存在状态为0,1,2,3,5的超过2个任务
|
|
|
+ logger.Error("Code:", code, "任务异常")
|
|
|
+ MgoE.Save("luacreatetaskerr", map[string]interface{}{
|
|
|
+ "code": code,
|
|
|
+ "comeintime": time.Now().Unix(),
|
|
|
+ "tasknum": len(*list),
|
|
|
+ })
|
|
|
+ return
|
|
|
+ } else if len(*list) == 2 { //会出现一个爬虫既有待审核的任务也有待确认的任务,此情况只处理待确认的任务即可
|
|
|
+ tmpstate := qu.IntAll(task["i_state"])
|
|
|
+ if n_istate == 0 { //新建任务为待确认
|
|
|
+ if tmpstate == 3 || tmpstate == 5 { //如果task=(*list)[0]取的是待审核或未通过的任务，则将task替换为待确认任务数据集
|
|
|
+ task = map[string]interface{}{}
|
|
|
+ task = (*list)[1]
|
|
|
+ }
|
|
|
+ } else if n_istate == 1 { //新建任务为待处理
|
|
|
+ if tmpstate == 0 { //如果task=(*list)[0]取的是待确认任务，则将task替换为待审核或未通过的任务数据集
|
|
|
+ task = map[string]interface{}{}
|
|
|
+ task = (*list)[1]
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //已有任务部分变量
|
|
|
+ o_istate := qu.IntAll(task["i_state"]) //已有任务的状态
|
|
|
+ addinfoid := []string{} //记录追加信息的infoid
|
|
|
+ if addinfoidtmp, ok := task["addinfoid"].([]interface{}); ok {
|
|
|
+ addinfoid = qu.ObjArrToStringArr(addinfoidtmp)
|
|
|
+ }
|
|
|
+ o_stype := qu.ObjToString(task["s_type"]) //已有任务的类型
|
|
|
+ if errtype > o_stype { //任务类型替换为权重最高的
|
|
|
+ o_stype = errtype
|
|
|
+ }
|
|
|
+ o_sdescript := qu.ObjToString(task["s_descript"]) //已有任务的描述
|
|
|
+ o_lcomplete := qu.Int64All(task["l_complete"]) //已有任务的最迟完成时间
|
|
|
+ o_surgency := qu.ObjToString(task["s_urgency"]) //已有任务的紧急度
|
|
|
+ o_iurgency, _ := strconv.Atoi(o_surgency) //已有任务的紧急度int类型
|
|
|
+ o_itimes := qu.IntAll(task["i_times"]) //已有任务出现特别紧急的次数
|
|
|
+ //处理任务流程
|
|
|
+ switch o_istate {
|
|
|
+ case 0: //原任务为待确认
|
|
|
+ // ctask := true //是否创建新任务(只针对原任务为待确认新任务为待处理)
|
|
|
+ // if n_istate == 0 || (n_istate == 1 && len(*list) == 2) {
|
|
|
+ o_iurgency++ //紧急度加一级
|
|
|
+ if o_iurgency >= 4 { //出现特别紧急的状态，记录次数itimes
|
|
|
+ o_itimes++
|
|
|
+ o_iurgency = 4
|
|
|
+ }
|
|
|
+ o_surgency = fmt.Sprint(o_iurgency)
|
|
|
+ o_lcomplete = CompleteTime(o_surgency) //更新完成时间
|
|
|
+ o_sdescript += time.Now().Format(qu.Date_Short_Layout) + "追加描述：------------------------------\n" + description //更新描述
|
|
|
+ addinfoid = append(addinfoid, id) //记录追加infoid
|
|
|
+ //ctask = false //n_istate==1&&len(*list)==2为true：新任务为待处理，已有任务除了有待确认还有其他类型任务，此时更新已有的待确认任务
|
|
|
+ //}
|
|
|
+ if n_istate == 0 { //新任务为待确认
|
|
|
+ if o_itimes >= 5 && len(*list) == 1 { //特别紧急的次数出现5次，自动创建待处理的任务(排除有待审核任务的可能)
|
|
|
+ o_istate = 1
|
|
|
+ }
|
|
|
+ q := map[string]interface{}{
|
|
|
+ "_id": task["_id"],
|
|
|
+ }
|
|
|
+ s := map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
+ "addinfoid": addinfoid,
|
|
|
+ "i_state": o_istate,
|
|
|
+ "s_descript": o_sdescript,
|
|
|
+ "l_complete": o_lcomplete,
|
|
|
+ "s_urgency": o_surgency,
|
|
|
+ "i_times": o_itimes,
|
|
|
+ "s_type": o_stype,
|
|
|
+ "i_min": n_imin,
|
|
|
+ "i_num": n_idn,
|
|
|
+ "s_downloadtime": n_sdt,
|
|
|
+ },
|
|
|
+ }
|
|
|
+ //更新task
|
|
|
+ MgoE.Update("task", q, s, false, false)
|
|
|
+ } else if n_istate == 1 { //新任务为待处理
|
|
|
+ if len(*list) == 1 { //已有任务只有待确认，新建待处理任务，关闭原任务
|
|
|
+ //新建任务(这里使用o_sdescript而不是description，是为了追加描述信息)
|
|
|
+ SaveTask(code, site, channel, modifyid, modify, o_sdescript, n_surgency, n_sdt, errtype, n_istate, n_imin, n_idn, event, n_itimes, addinfoid)
|
|
|
+ //关闭原任务
|
|
|
+ MgoE.Update("task", map[string]interface{}{"_id": task["_id"]}, map[string]interface{}{"$set": map[string]interface{}{"i_state": 6}}, false, false)
|
|
|
+ } else { //除了待确认有其他状态的任务，更新待确认任务
|
|
|
+ q := map[string]interface{}{
|
|
|
+ "_id": task["_id"],
|
|
|
+ }
|
|
|
+ s := map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
+ "addinfoid": addinfoid,
|
|
|
+ //"i_state": o_istate,
|
|
|
+ "s_descript": o_sdescript,
|
|
|
+ "l_complete": o_lcomplete,
|
|
|
+ "s_urgency": o_surgency,
|
|
|
+ "i_times": o_itimes,
|
|
|
+ "s_type": o_stype,
|
|
|
+ "i_min": n_imin,
|
|
|
+ "i_num": n_idn,
|
|
|
+ "s_downloadtime": n_sdt,
|
|
|
+ },
|
|
|
+ }
|
|
|
+ //更新task
|
|
|
+ MgoE.Update("task", q, s, false, false)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ case 1, 2: //原任务为待处理、处理中
|
|
|
+ //信息更新
|
|
|
+ o_sdescript += time.Now().Format(qu.Date_Short_Layout) + "追加描述：------------------------------\n" + description //更新描述
|
|
|
+ addinfoid = append(addinfoid, id)
|
|
|
+ q := map[string]interface{}{
|
|
|
+ "_id": task["_id"],
|
|
|
+ }
|
|
|
+ s := map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
+ "addinfoid": addinfoid,
|
|
|
+ "s_descript": o_sdescript,
|
|
|
+ "s_type": o_stype,
|
|
|
+ "i_min": n_imin,
|
|
|
+ "i_num": n_idn,
|
|
|
+ "s_downloadtime": n_sdt,
|
|
|
+ },
|
|
|
+ }
|
|
|
+ //更新task
|
|
|
+ MgoE.Update("task", q, s, false, false)
|
|
|
+ case 3: //原任务为待审核
|
|
|
+ //新建待确认任务
|
|
|
+ n_istate = 0
|
|
|
+ SaveTask(code, site, channel, modifyid, modify, description, n_surgency, n_sdt, errtype, n_istate, n_imin, n_idn, event, n_itimes, []string{id})
|
|
|
+ case 5: //原任务为未通过
|
|
|
+ //信息更新
|
|
|
+ o_sdescript += time.Now().Format(qu.Date_Short_Layout) + "追加描述：------------------------------\n" + description //更新描述
|
|
|
+ addinfoid = append(addinfoid, id) //记录追加infoid
|
|
|
+ o_istate = 1
|
|
|
+ q := map[string]interface{}{
|
|
|
+ "_id": task["_id"],
|
|
|
+ }
|
|
|
+ s := map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
+ "addinfoid": addinfoid,
|
|
|
+ "i_state": o_istate,
|
|
|
+ "s_descript": o_sdescript,
|
|
|
+ "s_type": o_stype,
|
|
|
+ "i_min": n_imin,
|
|
|
+ "i_num": n_idn,
|
|
|
+ "s_downloadtime": n_sdt,
|
|
|
+ },
|
|
|
+ }
|
|
|
+ //更新task
|
|
|
+ MgoE.Update("task", q, s, false, false)
|
|
|
+ //更新lua
|
|
|
+ MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"state": 0}}, false, false)
|
|
|
+ }
|
|
|
+ } else { //没有任务新建
|
|
|
+ SaveTask(code, site, channel, modifyid, modify, description, n_surgency, n_sdt, errtype, n_istate, n_imin, n_idn, event, n_itimes, []string{id})
|
|
|
+ }
|
|
|
+ }(tmp)
|
|
|
+ if n%100 == 0 {
|
|
|
+ qu.Debug("current:", n)
|
|
|
+ }
|
|
|
+ tmp = map[string]interface{}{}
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ logger.Debug("---任务创建完成---")
|
|
|
+}
|
|
|
+
|
|
|
+func SaveTask(code, site, channel, modifyid, modify, description, urgency, downloadtime, errtype string, state, min, downloadnum, event, times int, addinfoid []string) {
|
|
|
+ defer qu.Catch()
|
|
|
+ result := map[string]interface{}{}
|
|
|
+ if stateNum := UserTaskNum[modify]; stateNum == nil {
|
|
|
+ tmp := map[string]int{fmt.Sprint(state): 1}
|
|
|
+ UserTaskNum[modify] = tmp
|
|
|
+ } else {
|
|
|
+ stateNum[fmt.Sprint(state)]++
|
|
|
+ }
|
|
|
+ if state == 1 { //待处理任务，紧急程度定为特别紧急
|
|
|
+ urgency = "4"
|
|
|
+ }
|
|
|
+ result["s_code"] = code
|
|
|
+ result["s_site"] = site
|
|
|
+ result["s_channel"] = channel
|
|
|
+ result["s_modifyid"] = modifyid
|
|
|
+ result["s_modify"] = modify
|
|
|
+ result["s_descript"] = description
|
|
|
+ result["i_min"] = min
|
|
|
+ result["i_num"] = downloadnum //下载量
|
|
|
+ result["s_urgency"] = urgency
|
|
|
+ result["i_state"] = state
|
|
|
+ result["i_event"] = event
|
|
|
+ result["s_downloadtime"] = downloadtime //下载量对应的日期
|
|
|
+ result["l_comeintime"] = time.Now().Unix()
|
|
|
+ result["l_complete"] = CompleteTime(urgency)
|
|
|
+ //result["s_date"] = time.Now().Format(qu.Date_Short_Layout) //任务创建字符串日期
|
|
|
+ result["i_times"] = times //为了方便编辑器对次数的排序，记录当前的次数
|
|
|
+ result["s_type"] = errtype //任务类型
|
|
|
+ result["addinfoid"] = addinfoid //信息id
|
|
|
+ result["s_source"] = "程序"
|
|
|
+ MgoE.Save("task", result)
|
|
|
+}
|
|
|
+
|
|
|
+func SaveUserCreateTaskNum() {
|
|
|
+ defer qu.Catch()
|
|
|
+ for user, sn := range UserTaskNum {
|
|
|
+ save := map[string]interface{}{}
|
|
|
+ save["user"] = user
|
|
|
+ save["comeintime"] = time.Now().Unix()
|
|
|
+ for s, n := range sn {
|
|
|
+ save[s] = n
|
|
|
+ }
|
|
|
+ MgoE.Save("luausertask", save)
|
|
|
+ }
|
|
|
+ UserTaskNum = map[string]map[string]int{}
|
|
|
+}
|
|
|
+
|
|
|
+//重置前一周内未下载成功的数据(一天3次未下成功的数据可以连续下一周)
|
|
|
+func ResetDataState() {
|
|
|
+ defer qu.Catch()
|
|
|
+ logger.Info("-----更新数据状态-----")
|
|
|
+ sess := MgoS.GetMgoConn()
|
|
|
+ defer MgoS.DestoryMongoConn(sess)
|
|
|
+ ch := make(chan bool, 3)
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+ lock := &sync.Mutex{}
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "comeintime": map[string]interface{}{
|
|
|
+ "$gte": GetTime(-DayNum),
|
|
|
+ "$lt": GetTime(0),
|
|
|
+ },
|
|
|
+ "state": -1,
|
|
|
+ }
|
|
|
+ field := map[string]interface{}{
|
|
|
+ "_id": 1,
|
|
|
+ }
|
|
|
+ it := sess.DB("spider").C("spider_highlistdata").Find(&query).Select(&field).Iter()
|
|
|
+ count, _ := sess.DB("spider").C("spider_highlistdata").Find(&query).Count()
|
|
|
+ logger.Info("更新数据状态数量：", count)
|
|
|
+ n := 0
|
|
|
+ arr := [][]map[string]interface{}{}
|
|
|
+ for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
|
|
|
+ ch <- true
|
|
|
+ wg.Add(1)
|
|
|
+ go func(tmp map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-ch
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+ update := []map[string]interface{}{}
|
|
|
+ update = append(update, map[string]interface{}{"_id": tmp["_id"]})
|
|
|
+ update = append(update, map[string]interface{}{"$set": map[string]interface{}{"times": 0, "state": 0}})
|
|
|
+ lock.Lock()
|
|
|
+ arr = append(arr, update)
|
|
|
+ if len(arr) > 500 {
|
|
|
+ tmps := arr
|
|
|
+ MgoS.UpdateBulk("spider_highlistdata", tmps...)
|
|
|
+ arr = [][]map[string]interface{}{}
|
|
|
+ }
|
|
|
+ lock.Unlock()
|
|
|
+ }(tmp)
|
|
|
+ tmp = map[string]interface{}{}
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ lock.Lock()
|
|
|
+ if len(arr) > 0 {
|
|
|
+ MgoS.UpdateBulk("spider_highlistdata", arr...)
|
|
|
+ arr = [][]map[string]interface{}{}
|
|
|
+ }
|
|
|
+ lock.Unlock()
|
|
|
+ logger.Info("-----更新数据状态完毕-----")
|
|
|
+}
|
