新版爬虫异常信息统计定时任务

src/luatask/newtask.go

@@ -5,6 +5,7 @@ import (
 	"github.com/donnie4w/go-logger/logger"
 	qu "qfw/util"
 	"sync"
+	"time"
 	"util"
 )
 
@@ -392,25 +393,27 @@ func getSpiderListDownloadNum() {
 			href := qu.ObjToString(tmp["href"])
 			lock.Lock()
 			defer lock.Unlock()
-			tmpState := repeatHrefMap[href]
-			if tmpState == 1 { //该href已记录下载成功，后续不做任务记录
-				return
-			} else if tmpState == 0 { //未曾记录该href
-				if sp := NewCodeInfoMap[code]; sp != nil {
-					if state == 1 {
+			if sp := NewCodeInfoMap[code]; sp != nil {
+				tmpState := repeatHrefMap[href]
+				if tmpState == 1 { //该href已记录下载成功，后续不做任务记录
+					return
+				} else if tmpState == 0 { //未曾记录该href
+					if sp := NewCodeInfoMap[code]; sp != nil {
+						if state == 1 {
+							sp.Detail_DownloadSuccessNum++
+						} else {
+							state = -1
+							sp.Detail_DownloadFailNum++
+						}
+						sp.Detail_DownloadNum++
+						repeatHrefMap[href] = state
+					}
+				} else if tmpState == -1 && state == 1 { //已记录状态是下载失败,当前下载成功，记录该href最终为下载成功
+					if sp := NewCodeInfoMap[code]; sp != nil {
 						sp.Detail_DownloadSuccessNum++
-					} else {
-						state = -1
-						sp.Detail_DownloadFailNum++
+						sp.Detail_DownloadFailNum--
+						repeatHrefMap[href] = state
 					}
-					sp.Detail_DownloadNum++
-					repeatHrefMap[href] = state
-				}
-			} else if tmpState == -1 && state == 1 { //已记录状态是下载失败,当前下载成功，记录该href最终为下载成功
-				if sp := NewCodeInfoMap[code]; sp != nil {
-					sp.Detail_DownloadSuccessNum++
-					sp.Detail_DownloadFailNum--
-					repeatHrefMap[href] = state
 				}
 			}
 		}(tmp)
@@ -461,13 +464,13 @@ func getSpiderDownloadRateDataNew() {
 			lock.Lock()
 			if sp := NewCodeInfoMap[code]; sp != nil {
 				sp.List_NoDataTimes = zero
-				sp.List_AllInTimes = alltimes
+				sp.List_RunTimes = alltimes
 				sp.List_AllInTimes = oh_percent
 			}
 			lock.Unlock()
 		}(tmp)
 		if n%1000 == 0 {
-			logger.Info("current:", n)
+			logger.Info(n)
 		}
 		tmp = map[string]interface{}{}
 	}
@@ -480,6 +483,7 @@ func saveCodeInfo() {
 	lock := &sync.Mutex{}
 	wg := &sync.WaitGroup{}
 	ch := make(chan bool, 5)
+	comeintime := time.Now().Unix()
 	arr := []map[string]interface{}{}
 	for _, spider := range NewCodeInfoMap {
 		ch <- true
@@ -489,6 +493,7 @@ func saveCodeInfo() {
 				<-ch
 				wg.Done()
 			}()
+			sp.Comeintime = comeintime
 			spByte, err := json.Marshal(sp)
 			if err != nil {
 				logger.Info("Json Marshal Error", sp.Code)