重点爬虫任务告警

src/config.json

@@ -137,6 +137,8 @@
 	"updateluausercron": "0 30 23 ? * *",
 	"updatesitecron": "0 30 9 ? * *",
 	"newstartaskcron": "0 0 7 ? * *",
+	"importsitetaskcron1": "0 10 9 ? * *",
+	"importsitetaskcron2": "0 35 17 ? * *",
 	"closenum": 2,
 	"daynum": 6,
 	"infoformat": {

src/luatask/sitecount.go

@@ -33,13 +33,6 @@ type SiteInfo struct {
 	Comeintime            int64  `json:"comeintime"`            //href去重，当天发布采集成功数据量
 }
 
-var SiteInfoModel = `{
-    "msgtype": "file",
-    "file": {
-         "media_id": "%s"
-    }
-}`
-
 //var AllHref map[string]string
 
 //重点网站每日采集量统计
@@ -471,7 +464,7 @@ func SendSiteInfoToWxWork(file *xlsx.File) {
 	} else {
 		qu.Debug("unmarshal result err")
 	}
-	msg := fmt.Sprintf(SiteInfoModel, mediaId)
+	msg := fmt.Sprintf(util.FileModel, mediaId)
 	resp1, err := http.Post(
 		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
 		"application/json",

src/main.go

@@ -60,6 +60,9 @@ func main() {
 	//lua小组周报
 	c.AddFunc("0 30 17 ? * FRI", timetask.SpiderWeeklyReportForLua)
 	//c.AddFunc(CodeSummaryCron, SummaryCode)    //上架爬虫信息汇总
+	//重点网站任务统计
+	c.AddFunc(util.ImportSiteTaskCron1, timetask.ImportSiteTask)
+	c.AddFunc(util.ImportSiteTaskCron2, timetask.ImportSiteTask)
 	ch := make(chan bool, 1)
 	<-ch
 }

src/timetask/importsite.go

@@ -0,0 +1,76 @@
+package timetask
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	qu "qfw/util"
+	"time"
+	"util"
+)
+
+var ImportantCodeWarn = `
+     >积压任务个数：<font color=\"info\">%d个</font>
+     >超期任务个数：<font color=\"info\">%d个</font>
+     >完成任务个数：<font color=\"info\">%d个</font>
+`
+
+func ImportSiteTask() {
+	defer qu.Catch()
+	today := util.GetTime(0)
+	//积压告警（每日新增且未能修复的问题爬虫）
+	q1 := map[string]interface{}{
+		"l_create_time": map[string]interface{}{
+			"$gte": today,
+		},
+		"b_repair": false,
+	}
+	count1 := util.MgoEB.Count("spider_important_warning", q1)
+	//超期告警（每日在规定最迟修复时间内未修复爬虫量）
+	q2 := map[string]interface{}{
+		"b_repair": false,
+		"l_max_repair_time": map[string]interface{}{
+			"$gte": today,
+			"$lt":  time.Now().Unix(),
+		},
+	}
+	count2 := util.MgoEB.Count("spider_important_warning", q2)
+	//完成告警（每日完成修复量）
+	q3 := map[string]interface{}{
+		"l_update_time": map[string]interface{}{
+			"$gte": today,
+		},
+		"b_repair": true,
+	}
+	count3 := util.MgoEB.Count("spider_important_warning", q3)
+	////user
+	//mobileArr := []string{}
+	//for mobile, _ := range luaUserTextMap {
+	//	mobileArr = append(mobileArr, "\""+mobile+"\"")
+	//}
+	//toUserMsg := fmt.Sprintf(util.UserTextModel, "", strings.Join(mobileArr, ","))
+	//qu.Debug("toUserMsg", toUserMsg)
+	//resp1, err := http.Post(
+	//	"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
+	//	"application/json",
+	//	bytes.NewBuffer([]byte(toUserMsg)),
+	//)
+	//if err != nil {
+	//	fmt.Println("request error:", err)
+	//	return
+	//}
+	//defer resp1.Body.Close()
+	//msg
+	msg := fmt.Sprintf(util.MarkdownModel, fmt.Sprintf(ImportantCodeWarn, count1, count2, count3))
+	qu.Debug("重点网站告警", msg)
+	resp, err := http.Post(
+		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=62c891a7-5a21-4487-bcaf-0ea2067bea15",
+		"application/json",
+		bytes.NewBuffer([]byte(msg)),
+	)
+	if err != nil {
+		fmt.Println("request error:", err)
+		return
+	}
+	defer resp.Body.Close()
+}

src/timetask/report.go

@@ -645,7 +645,7 @@ func SpiderWeeklyReportForLua() {
 	}
 	taskCount := util.MgoEB.Count("task", q)
 	text := fmt.Sprintf(LuaWeeklyReport, allCount, newCount, allCount-newCount, needNewCount+needHistoryCount, taskCount)
-	msg := fmt.Sprintf(MarkdownModel, text)
+	msg := fmt.Sprintf(util.MarkdownModel, text)
 	qu.Debug("msg", msg)
 	resp, err := http.Post(
 		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=7b082bc8-31d6-41b8-a6d9-1f731505ad0e",

src/timetask/summary.go

@@ -23,18 +23,6 @@ var (
 	Publishtime                     string
 )
 
-var LuaPythonNumModel = `{
-    "msgtype": "text",
-    "text": {
-		"content": "%s"
-	}
-}`
-var MarkdownModel = `{
-    "msgtype": "markdown",
-    "markdown": {
-        "content": "%s"
-    }
-}`
 var NumContentModel = `
      >平台：<font color=\"warning\">%s</font>
      >列表页采集量：<font color=\"warning\">%d</font>
@@ -233,7 +221,7 @@ func SendLuaPythonAllNum() {
 	defer qu.Catch()
 	luaContent := fmt.Sprintf(NumContentModel, "Lua", LuaListDownloadAllNum, LuaListDownloadSuccessAllNum, LuaBiddingDownloadAllNum)
 	pythonContent := fmt.Sprintf(NumContentModel, "python", PythonListDownloadAllNum, PythonListDownloadSuccessAllNum, PythonBiddingDownloadAllNum)
-	resultContent := fmt.Sprintf(MarkdownModel, Publishtime+"，Lua、Python各维度采集量统计结果如下：\n"+luaContent+pythonContent)
+	resultContent := fmt.Sprintf(util.MarkdownModel, Publishtime+"，Lua、Python各维度采集量统计结果如下：\n"+luaContent+pythonContent)
 	qu.Debug(resultContent)
 	//保存记录
 	util.MgoS.Save("spider_luapythoncount", map[string]interface{}{

src/timetask/wxworkwarn.go

@@ -67,13 +67,6 @@ var PythonUserContentModel = `
 //        "content": "%s"
 //    }
 //}`
-var TextModel = `{
-    "msgtype": "text",
-    "text": {
-        "content": "%s",
-        "mentioned_mobile_list":[%s]
-    }
-}`
 
 //初始化人员信息
 func GetLuaUserInfo() {
@@ -203,9 +196,9 @@ func SendLuaInfo() {
 		mobileArr = append(mobileArr, "\""+mobile+"\"")
 		resultContent += fmt.Sprintf(LuaUserContentModel, t.Username, t.FailedTaskCount, t.FailedTaskOverdueDay, t.HeartErrCount, t.NoCollectDataDay)
 	}
-	msg := fmt.Sprintf(MarkdownModel, resultContent)
+	msg := fmt.Sprintf(util.MarkdownModel, resultContent)
 	qu.Debug("msg", msg)
-	toUserMsg := fmt.Sprintf(TextModel, "", strings.Join(mobileArr, ","))
+	toUserMsg := fmt.Sprintf(util.UserTextModel, "", strings.Join(mobileArr, ","))
 	qu.Debug("toUserMsg", toUserMsg)
 	resp1, err := http.Post(
 		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
@@ -253,9 +246,9 @@ func SendPythonInfo() {
 		mobileArr = append(mobileArr, "\""+mobile+"\"")
 		resultContent += fmt.Sprintf(PythonUserContentModel, t.Username, t.ToBeCompleted, t.Failed)
 	}
-	msg := fmt.Sprintf(MarkdownModel, resultContent)
+	msg := fmt.Sprintf(util.MarkdownModel, resultContent)
 	qu.Debug("msg", msg)
-	toUserMsg := fmt.Sprintf(TextModel, "", strings.Join(mobileArr, ","))
+	toUserMsg := fmt.Sprintf(util.UserTextModel, "", strings.Join(mobileArr, ","))
 	qu.Debug("toUserMsg", toUserMsg)
 
 	resp1, err := http.Post(
@@ -338,7 +331,7 @@ func SendPrivateInfo() {
 		}
 	}
 	if content != "" {
-		toMyself := fmt.Sprintf(TextModel, content, "18624860222") //白小虎
+		toMyself := fmt.Sprintf(util.UserTextModel, content, "18624860222") //白小虎
 		logger.Info("toMyself", toMyself)
 		resp, err := http.Post(
 			"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
@@ -380,7 +373,7 @@ func SendInfoToWxWork_ToAuditor() {
 		tmpContent += userInfo.Username + "需审核的爬虫有：" + fmt.Sprint(count) + "个；"
 		tmpModifyList = append(tmpModifyList, userInfo.Mobile)
 	}
-	msg := fmt.Sprintf(TextModel, tmpContent+"请及时处理！", strings.Join(tmpModifyList, ","))
+	msg := fmt.Sprintf(util.UserTextModel, tmpContent+"请及时处理！", strings.Join(tmpModifyList, ","))
 	qu.Debug("msg", msg)
 	resp1, err := http.Post(
 		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",

src/util/config.go

@@ -34,6 +34,8 @@ var (
 	UpdateLuaUserCron          string        //每天更新外包爬虫到内部人员
 	UpdateSiteCron             string        //每天更新站点信息
 	NewStarTaskCron            string        //新版爬虫维护任务
+	ImportSiteTaskCron1        string        //重点网站任务
+	ImportSiteTaskCron2        string        //重点网站任务
 	CloseNum                   int           //关闭几天的任务
 	DayNum                     int           //更新数据天数
 	CodeEventModel             map[int]int   //节点对应的采集模式0：老模式；1：新模式
@@ -119,6 +121,8 @@ func InitOther() {
 	UpdateLuaUserCron = qu.ObjToString(Config["updateluausercron"])
 	UpdateSiteCron = qu.ObjToString(Config["updatesitecron"])
 	NewStarTaskCron = qu.ObjToString(Config["newstartaskcron"])
+	ImportSiteTaskCron1 = qu.ObjToString(Config["importsitetaskcron1"])
+	ImportSiteTaskCron2 = qu.ObjToString(Config["importsitetaskcron2"])
 
 	CloseNum = qu.IntAll(Config["closenum"])
 	DayNum = qu.IntAll(Config["daynum"])

src/util/warnmodel.go

@@ -0,0 +1,30 @@
+package util
+
+var (
+	MarkdownModel = `{
+    "msgtype": "markdown",
+    "markdown": {
+        "content": "%s"
+    }
+}`
+	TextModel = `{
+    "msgtype": "text",
+    "text": {
+		"content": "%s"
+	}
+}`
+	UserTextModel = `{
+    "msgtype": "text",
+    "text": {
+        "content": "%s",
+        "mentioned_mobile_list":[%s]
+    }
+}`
+
+	FileModel = `{
+    "msgtype": "file",
+    "file": {
+         "media_id": "%s"
+    }
+}`
+)