新增详情页心跳异常提醒

src/logs/task.log

@@ -702899,3 +702899,6 @@
 2022/08/22 09:41:14 task.go:1888: info  -----更新数据状态-----
 2022/08/22 09:41:14 task.go:1953: info  更新数据状态数量： 0
 2022/08/22 09:41:14 task.go:2014: info  -----更新spider_historydata数据状态完毕-----
+2022/12/12 15:38:43 wxworkwarn.go:290: info  query: map[del:false event:map[$in:[]]]
+2022/12/12 15:38:53 wxworkwarn.go:290: info  query: map[del:false event:map[$in:[]]]
+2022/12/12 15:39:24 wxworkwarn.go:290: info  query: map[del:false event:map[$in:[7100 7110 7200 7300 7400 7210 7310]]]

src/luatask/task.go

@@ -244,7 +244,7 @@ func GetCodeBaseInfo() {
 			},
 		},
 	}
-	fieles := map[string]interface{}{
+	fields := map[string]interface{}{
 		"event":             1,
 		"param_common":      1,
 		"platform":          1,
@@ -261,7 +261,7 @@ func GetCodeBaseInfo() {
 	}
 	count := util.MgoEB.Count("luaconfig", query)
 	logger.Debug("共加载线上爬虫个数:", count)
-	it := sess.DB(util.MgoEB.DbName).C("luaconfig").Find(&query).Select(&fieles).Iter()
+	it := sess.DB(util.MgoEB.DbName).C("luaconfig").Find(&query).Select(&fields).Iter()
 	n := 0
 	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
 		wg.Add(1)

src/timetask/wxworkwarn.go

@@ -3,10 +3,12 @@ package timetask
 import (
 	"bytes"
 	"fmt"
+	"github.com/donnie4w/go-logger/logger"
 	"math"
 	"net/http"
 	qu "qfw/util"
 	"strings"
+	"sync"
 	"time"
 	"util"
 )
@@ -70,6 +72,7 @@ var TextModel = `{
     }
 }`
 
+//初始化人员信息
 func GetLuaUserInfo() {
 	LuaModifyUserInfoMap = map[string]*UserInfo{}
 	LuaAuditorInfoMap = map[string]*UserInfo{}
@@ -111,6 +114,7 @@ func GetLuaUserInfo() {
 func SendInfoToWxWork_Tomodifyuser() {
 	SendLuaInfo()
 	SendPythonInfo()
+	SendPrivateInfo()
 }
 
 func SendLuaInfo() {
@@ -263,6 +267,79 @@ func SendPythonInfo() {
 	defer resp2.Body.Close()
 }
 
+//每日统计列表页、详情页分开采集模式节点爬虫详情页采集心跳异常信息
+func SendPrivateInfo() {
+	defer qu.Catch()
+	var eventArr []int //节点集合
+	eventNumMap := map[int]int{}
+	for event, model := range util.CodeEventModel {
+		if model == 1 {
+			eventArr = append(eventArr, event)
+		}
+	}
+	query := map[string]interface{}{
+		"event": map[string]interface{}{
+			"$in": eventArr,
+		},
+		"del": false,
+	}
+	fields := map[string]interface{}{
+		"event":  1,
+		"detail": 1,
+	}
+	logger.Info("query:", query)
+	lock := &sync.Mutex{}
+	wg := &sync.WaitGroup{}
+	ch := make(chan bool, 2)
+	sess := util.MgoS.GetMgoConn()
+	defer util.MgoS.DestoryMongoConn(sess)
+	it := sess.DB(util.MgoS.DbName).C("spider_heart").Find(&query).Select(&fields).Iter()
+	n := 0
+	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
+		wg.Add(1)
+		ch <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			detailTime := qu.Int64All(tmp["detail"])
+			if time.Now().Unix()-detailTime > 3600*3 { //统计超过3小时前爬虫个数
+				event := qu.IntAll(tmp["event"])
+				lock.Lock()
+				eventNumMap[event] += 1
+				lock.Unlock()
+			}
+		}(tmp)
+		if n%100 == 0 {
+			logger.Debug(n)
+		}
+		tmp = map[string]interface{}{}
+	}
+	wg.Wait()
+	logger.Info(eventNumMap)
+	content := ""
+	for event, num := range eventNumMap {
+		if num >= 10 {
+			content += "节点：" + fmt.Sprint(event) + "详情页执行心跳异常个数：" + fmt.Sprint(num) + "；"
+		}
+	}
+	if content != "" {
+		toMyself := fmt.Sprintf(TextModel, content, "15637845493")
+		logger.Info("toMyself", toMyself)
+		resp, err := http.Post(
+			"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
+			"application/json",
+			bytes.NewBuffer([]byte(toMyself)),
+		)
+		if err != nil {
+			fmt.Println("request error:", err)
+			return
+		}
+		defer resp.Body.Close()
+	}
+}
+
 // 统计爬虫审核人员待审核爬虫
 func SendInfoToWxWork_ToAuditor() {
 	defer qu.Catch()