新增python爬虫信息告警

src/luatask/task.go

@@ -1592,7 +1592,7 @@ func CreateTaskProcess() {
 							if spider.Model == 1 { //新模式(7100、7110、7200、7210、7300、7310、7400)根据异常总量和占比建任务
 								moreThanLimit := false
 								//1、异常条数;2、异常占比
-								if spider.DownloadFailedNum > FailedNumLimit {
+								if spider.DownloadFailedNum >= FailedNumLimit {
 									moreThanLimit = true
 								} else if spider.DownloadAllNum > 0 && (float64(spider.DownloadFailedNum)/float64(spider.DownloadAllNum)) > FailedPercentLimit {
 									moreThanLimit = true

src/main.go

@@ -18,7 +18,7 @@ import (
 func init() {
 	qu.ReadConfig(&util.Config)
 	qu.ReadConfig("./worktime.json", &util.Workfig)
-	qu.ReadConfig("./user.json", &timetask.LuaUserMap)
+	qu.ReadConfig("./user.json", &timetask.UserMap)
 	util.InitWorkfig()
 	timetask.GetLuaUserInfo()
 	//
@@ -46,7 +46,7 @@ func main() {
 	//
 	c.AddFunc(util.FileWarnCron, timetask.GetFileWarn) //异常附件数据警告
 	//
-	c.AddFunc(util.QyworkRemindModifyuserCron, timetask.Sendinfotowxwork_Tomodifyuser) //企业微信日常警告,爬虫开发人员告警信息
+	c.AddFunc(util.QyworkRemindModifyuserCron, timetask.SendInfoToWxWork_Tomodifyuser) //企业微信日常警告,爬虫开发人员告警信息
 	c.AddFunc(util.QyworkRemindAuditorCron, timetask.SendInfoToWxWork_ToAuditor)       //企业微信日常警告,审核人员告警信息
 
 	c.AddFunc(util.MoveListDataCron, timetask.MoveListData) //列表页数据迁移

src/timetask/wxworkwarn.go

@@ -11,11 +11,13 @@ import (
 	"util"
 )
 
-var LuaUserMap map[string]map[string]string
-var LuaModifyUserInfoMap map[string]*LuaUserInfo //开发人员信息集合
-var LuaAuditorInfoMap map[string]*LuaUserInfo    //审核人员信息集合
+var UserMap map[string]map[string]string
+var PythonModifyUserInfoMap map[string]*UserInfo //开发人员信息集合
+var LuaModifyUserInfoMap map[string]*UserInfo    //开发人员信息集合
+var LuaAuditorInfoMap map[string]*UserInfo       //审核人员信息集合
 
-type UserTextInfo struct {
+//lua
+type LuaUserTextInfo struct {
 	Username             string
 	FailedTaskCount      int
 	FailedTaskOverdueDay int
@@ -23,21 +25,37 @@ type UserTextInfo struct {
 	NoCollectDataDay     int
 }
 
-type LuaUserInfo struct {
+//python
+type PythonUserTextInfo struct {
+	Username      string
+	ToBeCompleted int //待完成爬虫个数
+	Failed        int //未通过爬虫个数
+}
+
+type UserInfo struct {
 	Username string
 	Mobile   string
 	Auth     string
 }
 
 //
-var TitleContentModel = `
+var LuaTitleContentModel = `
 	截止目前，爬虫共有未通过任务<font color=\"warning\">%d个</font>，异常心跳爬虫<font color=\"warning\">%d个</font>。请及时处理！\n
 `
-var UserContentModel = `
+var PythonTitleContentModel = `
+	截止目前，共有待完成爬虫<font color=\"warning\">%d个</font>，未通过爬虫<font color=\"warning\">%d个</font>，待审核爬虫<font color=\"warning\">%d个</font>。请及时处理！\n
+`
+
+var LuaUserContentModel = `
      >人员：<font color=\"warning\">%s</font>
      >未通过任务：<font color=\"warning\">%d个</font><font color=\"info\">(最早任务已逾期%d天)</font>
      >异常心跳爬虫：<font color=\"warning\">%d个</font><font color=\"info\">(已有爬虫%d天未采集数据)</font>\n
 `
+var PythonUserContentModel = `
+     >人员：<font color=\"warning\">%s</font>
+     >待完成爬虫：<font color=\"warning\">%d个</font><font color=\"info\"></font>
+     >未通过爬虫：<font color=\"warning\">%d个</font><font color=\"info\"></font>\n
+`
 var MarkdownModel = `{
     "msgtype": "markdown",
     "markdown": {
@@ -53,41 +71,59 @@ var TextModel = `{
 }`
 
 func GetLuaUserInfo() {
-	LuaModifyUserInfoMap = map[string]*LuaUserInfo{}
-	LuaAuditorInfoMap = map[string]*LuaUserInfo{}
-	for eu, info := range LuaUserMap {
+	LuaModifyUserInfoMap = map[string]*UserInfo{}
+	LuaAuditorInfoMap = map[string]*UserInfo{}
+	PythonModifyUserInfoMap = map[string]*UserInfo{}
+	for eu, info := range UserMap {
+		role := info["role"]
 		auth := info["auth"]
-		if auth == "1" { //开发人员
-			LuaModifyUserInfoMap[eu] = &LuaUserInfo{
-				Username: info["username"],
-				Mobile:   info["mobile"],
-				Auth:     auth,
+		if role == "lua" {
+			if auth == "1" { //开发人员
+				LuaModifyUserInfoMap[eu] = &UserInfo{
+					Username: info["username"],
+					Mobile:   info["mobile"],
+					Auth:     auth,
+				}
 			}
-		}
-		if auth == "3" || eu == "ssc" { //审核人员
-			LuaAuditorInfoMap[eu] = &LuaUserInfo{
-				Username: info["username"],
-				Mobile:   info["mobile"],
-				Auth:     auth,
+			if auth == "3" || eu == "ssc" { //审核人员
+				LuaAuditorInfoMap[eu] = &UserInfo{
+					Username: info["username"],
+					Mobile:   info["mobile"],
+					Auth:     auth,
+				}
+			}
+		} else {
+			if auth == "1" {
+				PythonModifyUserInfoMap[eu] = &UserInfo{
+					Username: info["username"],
+					Mobile:   info["mobile"],
+					Auth:     auth,
+				}
 			}
 		}
 	}
 	qu.Debug(LuaModifyUserInfoMap)
 	qu.Debug(LuaAuditorInfoMap)
+	qu.Debug(PythonModifyUserInfoMap)
 }
 
 // 统计爬虫开发人员未完成爬虫和任务
-func Sendinfotowxwork_Tomodifyuser() {
+func SendInfoToWxWork_Tomodifyuser() {
+	SendLuaInfo()
+	SendPythonInfo()
+}
+
+func SendLuaInfo() {
 	defer qu.Catch()
-	qu.Debug("企业微信发送提示信息")
-	failedTaskCount, heartCodeCount := 0, 0   //总未通过任务个数，总待处理心跳异常爬虫个数
-	userTextMap := map[string]*UserTextInfo{} //key:mobile
-	for eu, userInfo := range LuaModifyUserInfoMap {
-		textInfo := &UserTextInfo{}
+	qu.Debug("lua企业微信发送提示信息")
+	failedTaskCount, heartCodeCount := 0, 0         //总未通过任务个数，总待处理心跳异常爬虫个数
+	luaUserTextMap := map[string]*LuaUserTextInfo{} //key:mobile
+	for user, userInfo := range LuaModifyUserInfoMap {
+		textInfo := &LuaUserTextInfo{}
 		textInfo.Username = userInfo.Username
 		//1、未通过任务信息
 		list_task, _ := util.MgoE.Find("task",
-			map[string]interface{}{"s_modify": eu, "i_state": 5},
+			map[string]interface{}{"s_modify": user, "i_state": 5},
 			map[string]interface{}{"l_complete": 1},
 			map[string]interface{}{"l_complete": 1},
 			false, -1, -1)
@@ -101,7 +137,7 @@ func Sendinfotowxwork_Tomodifyuser() {
 		}
 		//2、爬虫心跳信息
 		query := map[string]interface{}{
-			"modifyuser": eu,
+			"modifyuser": user,
 			"del":        false,
 			//"list": map[string]interface{}{
 			//	"$lte": util.GetTime(0),
@@ -141,19 +177,70 @@ func Sendinfotowxwork_Tomodifyuser() {
 			ncDay := int(math.Floor(float64(time.Now().Unix()-listTime) / float64(86400)))
 			textInfo.NoCollectDataDay = ncDay //个人未通过任务最早逾期天数赋值
 		}
-		userTextMap[userInfo.Mobile] = textInfo
+		luaUserTextMap[userInfo.Mobile] = textInfo
+	}
+	//拼接content
+	resultContent := fmt.Sprintf(LuaTitleContentModel, failedTaskCount, heartCodeCount)
+	mobileArr := []string{}
+	for mobile, t := range luaUserTextMap {
+		mobileArr = append(mobileArr, "\""+mobile+"\"")
+		resultContent += fmt.Sprintf(LuaUserContentModel, t.Username, t.FailedTaskCount, t.FailedTaskOverdueDay, t.HeartErrCount, t.NoCollectDataDay)
+	}
+	msg := fmt.Sprintf(MarkdownModel, resultContent)
+	qu.Debug("msg", msg)
+	toUserMsg := fmt.Sprintf(TextModel, "", strings.Join(mobileArr, ","))
+	qu.Debug("toUserMsg", toUserMsg)
+	resp1, err := http.Post(
+		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
+		"application/json",
+		bytes.NewBuffer([]byte(toUserMsg)),
+	)
+	if err != nil {
+		fmt.Println("request error:", err)
+		return
+	}
+	defer resp1.Body.Close()
+	resp2, err := http.Post(
+		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
+		"application/json",
+		bytes.NewBuffer([]byte(msg)),
+	)
+	if err != nil {
+		fmt.Println("request error:", err)
+		return
+	}
+	defer resp2.Body.Close()
+}
+
+func SendPythonInfo() {
+	defer qu.Catch()
+	qu.Debug("python企业微信发送提示信息")
+	toBeCompletedAllCount, failedAllCount := 0, 0
+	toBeReviewedAllCount := util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 1, "platform": "python"})
+	pythonUserTextMap := map[string]*PythonUserTextInfo{} //key:mobile
+	for user, userInfo := range PythonModifyUserInfoMap {
+		textInfo := &PythonUserTextInfo{}
+		textInfo.Username = userInfo.Username
+		//1、待完成爬虫个数
+		textInfo.ToBeCompleted = util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 0, "modifyuser": user})
+		toBeCompletedAllCount += textInfo.ToBeCompleted
+		//2、未通过爬虫个数
+		textInfo.Failed = util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 2, "modifyuser": user})
+		failedAllCount += textInfo.Failed
+		pythonUserTextMap[userInfo.Mobile] = textInfo
 	}
 	//拼接content
-	resultContent := fmt.Sprintf(TitleContentModel, failedTaskCount, heartCodeCount)
+	resultContent := fmt.Sprintf(PythonTitleContentModel, toBeCompletedAllCount, failedAllCount, toBeReviewedAllCount)
 	mobileArr := []string{}
-	for mobile, t := range userTextMap {
+	for mobile, t := range pythonUserTextMap {
 		mobileArr = append(mobileArr, "\""+mobile+"\"")
-		resultContent += fmt.Sprintf(UserContentModel, t.Username, t.FailedTaskCount, t.FailedTaskOverdueDay, t.HeartErrCount, t.NoCollectDataDay)
+		resultContent += fmt.Sprintf(PythonUserContentModel, t.Username, t.ToBeCompleted, t.Failed)
 	}
 	msg := fmt.Sprintf(MarkdownModel, resultContent)
 	qu.Debug("msg", msg)
 	toUserMsg := fmt.Sprintf(TextModel, "", strings.Join(mobileArr, ","))
 	qu.Debug("toUserMsg", toUserMsg)
+
 	resp1, err := http.Post(
 		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
 		"application/json",

src/user.json

@@ -2,21 +2,43 @@
 	"ssc":{
 		"username": "施顺才",
 		"mobile": "13523457747",
+		"role": "lua",
 		"auth": "1"
 	},
 	"lyf":{
 		"username": "刘一帆",
 		"mobile": "15896901897",
+		"role": "lua",
 		"auth": "1"
 	},
 	"jiaoyubo":{
 		"username": "焦宇波",
 		"mobile": "15516197109",
+		"role": "lua",
 		"auth": "1"
 	},
 	"cjk":{
 		"username": "陈佳康",
 		"mobile": "15736702898",
+		"role": "lua",
 		"auth": "3"
+	},
+	"dongzhaorui":{
+		"username": "董钊瑞",
+		"mobile": "15238795000",
+		"role": "python",
+		"auth": "1"
+	},
+	"lizongze":{
+		"username": "李宗泽",
+		"mobile": "18272665277",
+		"role": "python",
+		"auth": "1"
+	},
+	"baixiaohu":{
+		"username": "白小虎",
+		"mobile": "18624860222",
+		"role": "python",
+		"auth": "1"
 	}
 }