package timetask import ( "bytes" "fmt" "github.com/donnie4w/go-logger/logger" "math" "net/http" qu "qfw/util" "strings" "sync" "time" "util" ) var UserMap map[string]map[string]string var PythonModifyUserInfoMap map[string]*UserInfo //开发人员信息集合 var LuaModifyUserInfoMap map[string]*UserInfo //开发人员信息集合 var LuaAuditorInfoMap map[string]*UserInfo //审核人员信息集合 //lua type LuaUserTextInfo struct { Username string FailedTaskCount int FailedTaskOverdueDay int HeartErrCount int NoCollectDataDay int } //python type PythonUserTextInfo struct { Username string ToBeCompleted int //待完成爬虫个数 Failed int //未通过爬虫个数 } type UserInfo struct { Username string UserID string Email string Mobile string Auth string } // var LuaTitleContentModel = ` 截止目前,爬虫共有未通过任务%d个,异常心跳爬虫%d个。请及时处理!\n ` var PythonTitleContentModel = ` 截止目前,共有待完成爬虫%d个,未通过爬虫%d个,待审核爬虫%d个。请及时处理!\n ` var LuaUserContentModel = ` >人员:%s >未通过任务:%d个(最早任务已逾期%d天) >异常心跳爬虫:%d个(已有爬虫%d天未采集数据)\n ` var PythonUserContentModel = ` >人员:%s >待完成爬虫:%d个 >未通过爬虫:%d个\n ` var MarkdownModel = `{ "msgtype": "markdown", "markdown": { "content": "%s" } }` var TextModel = `{ "msgtype": "text", "text": { "content": "%s", "mentioned_mobile_list":[%s] } }` //初始化人员信息 func GetLuaUserInfo() { LuaModifyUserInfoMap = map[string]*UserInfo{} LuaAuditorInfoMap = map[string]*UserInfo{} PythonModifyUserInfoMap = map[string]*UserInfo{} for eu, info := range UserMap { role := info["role"] auth := info["auth"] if role == "lua" { if auth == "1" { //开发人员 LuaModifyUserInfoMap[eu] = &UserInfo{ Username: info["username"], UserID: info["userid"], Email: info["email"], Mobile: info["mobile"], Auth: auth, } } if auth == "3" || eu == "ssc" { //审核人员 LuaAuditorInfoMap[eu] = &UserInfo{ Username: info["username"], Mobile: info["mobile"], Auth: auth, } } } else { if auth == "1" { PythonModifyUserInfoMap[eu] = &UserInfo{ Username: info["username"], Mobile: info["mobile"], Auth: auth, } } } } // index := 0 for name, _ := range LuaModifyUserInfoMap { LuaUserIndexMap[index] = name index++ } LuaUserNum = index qu.Debug(LuaModifyUserInfoMap) qu.Debug(LuaAuditorInfoMap) qu.Debug(PythonModifyUserInfoMap) qu.Debug(LuaUserIndexMap, LuaUserNum) } // 统计爬虫开发人员未完成爬虫和任务 func SendInfoToWxWork_Tomodifyuser() { SendLuaInfo() SendPythonInfo() SendPrivateInfo() } func SendLuaInfo() { defer qu.Catch() qu.Debug("lua企业微信发送提示信息") failedTaskCount, heartCodeCount := 0, 0 //总未通过任务个数,总待处理心跳异常爬虫个数 luaUserTextMap := map[string]*LuaUserTextInfo{} //key:mobile for user, userInfo := range LuaModifyUserInfoMap { textInfo := &LuaUserTextInfo{} textInfo.Username = userInfo.Username //1、未通过任务信息 list_task, _ := util.MgoEB.Find("task", map[string]interface{}{"s_modify": user, "i_state": 5}, map[string]interface{}{"l_complete": 1}, map[string]interface{}{"l_complete": 1}, false, -1, -1) taskLen := len(*list_task) textInfo.FailedTaskCount = taskLen //个人未通过任务个数赋值 failedTaskCount += taskLen //总未通过个数++ if taskLen > 0 { complete := qu.Int64All((*list_task)[0]["l_complete"]) //未通过任务中最迟完成时间最早的任务 odDay := int(math.Floor(float64(time.Now().Unix()-complete) / float64(86400))) textInfo.FailedTaskOverdueDay = odDay //个人未通过任务最早逾期天数赋值 } //2、爬虫心跳信息 query := map[string]interface{}{ "modifyuser": user, "del": false, //"list": map[string]interface{}{ // "$lte": util.GetTime(0), //}, "$or": []interface{}{ map[string]interface{}{ "event": map[string]interface{}{ "$nin": []int{7000, 7520}, //"$nin": []int{7500, 7510}, }, "list": map[string]interface{}{ "$lte": util.GetTime(0), }, }, map[string]interface{}{ "event": 7520, //"event": map[string]interface{}{ // "$in": []int{7500, 7510}, //}, "list": map[string]interface{}{ "$lte": util.GetTime(-1), }, }, }, } qu.Debug("heart query:", query) list_code, _ := util.MgoS.Find("spider_heart", query, map[string]interface{}{"list": 1}, map[string]interface{}{"list": 1}, false, -1, -1) codeLen := len(*list_code) textInfo.HeartErrCount = codeLen //个人异常心跳爬虫个数赋值 heartCodeCount += codeLen //总异常心跳爬虫个数++ if codeLen > 0 { listTime := qu.Int64All((*list_code)[0]["list"]) //未通过任务中最迟完成时间最早的任务 ncDay := int(math.Floor(float64(time.Now().Unix()-listTime) / float64(86400))) textInfo.NoCollectDataDay = ncDay //个人未通过任务最早逾期天数赋值 } luaUserTextMap[userInfo.Mobile] = textInfo } //拼接content resultContent := fmt.Sprintf(LuaTitleContentModel, failedTaskCount, heartCodeCount) mobileArr := []string{} for mobile, t := range luaUserTextMap { mobileArr = append(mobileArr, "\""+mobile+"\"") resultContent += fmt.Sprintf(LuaUserContentModel, t.Username, t.FailedTaskCount, t.FailedTaskOverdueDay, t.HeartErrCount, t.NoCollectDataDay) } msg := fmt.Sprintf(MarkdownModel, resultContent) qu.Debug("msg", msg) toUserMsg := fmt.Sprintf(TextModel, "", strings.Join(mobileArr, ",")) qu.Debug("toUserMsg", toUserMsg) resp1, err := http.Post( "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e", "application/json", bytes.NewBuffer([]byte(toUserMsg)), ) if err != nil { fmt.Println("request error:", err) return } defer resp1.Body.Close() resp2, err := http.Post( "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e", "application/json", bytes.NewBuffer([]byte(msg)), ) if err != nil { fmt.Println("request error:", err) return } defer resp2.Body.Close() } func SendPythonInfo() { defer qu.Catch() qu.Debug("python企业微信发送提示信息") toBeCompletedAllCount, failedAllCount := 0, 0 toBeReviewedAllCount := util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 1, "platform": "python"}) pythonUserTextMap := map[string]*PythonUserTextInfo{} //key:mobile for user, userInfo := range PythonModifyUserInfoMap { textInfo := &PythonUserTextInfo{} textInfo.Username = userInfo.Username //1、待完成爬虫个数 textInfo.ToBeCompleted = util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 0, "modifyuser": user}) toBeCompletedAllCount += textInfo.ToBeCompleted //2、未通过爬虫个数 textInfo.Failed = util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 2, "modifyuser": user}) failedAllCount += textInfo.Failed pythonUserTextMap[userInfo.Mobile] = textInfo } //拼接content resultContent := fmt.Sprintf(PythonTitleContentModel, toBeCompletedAllCount, failedAllCount, toBeReviewedAllCount) mobileArr := []string{} for mobile, t := range pythonUserTextMap { mobileArr = append(mobileArr, "\""+mobile+"\"") resultContent += fmt.Sprintf(PythonUserContentModel, t.Username, t.ToBeCompleted, t.Failed) } msg := fmt.Sprintf(MarkdownModel, resultContent) qu.Debug("msg", msg) toUserMsg := fmt.Sprintf(TextModel, "", strings.Join(mobileArr, ",")) qu.Debug("toUserMsg", toUserMsg) resp1, err := http.Post( "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e", "application/json", bytes.NewBuffer([]byte(toUserMsg)), ) if err != nil { fmt.Println("request error:", err) return } defer resp1.Body.Close() resp2, err := http.Post( "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e", "application/json", bytes.NewBuffer([]byte(msg)), ) if err != nil { fmt.Println("request error:", err) return } defer resp2.Body.Close() } //每日统计列表页、详情页分开采集模式节点爬虫详情页采集心跳异常信息 func SendPrivateInfo() { defer qu.Catch() var eventArr []int //节点集合 eventNumMap := map[int]int{} for event, model := range util.CodeEventModel { if model == 1 { eventArr = append(eventArr, event) } } query := map[string]interface{}{ "event": map[string]interface{}{ "$in": eventArr, }, "del": false, } fields := map[string]interface{}{ "event": 1, "detail": 1, } logger.Info("query:", query) lock := &sync.Mutex{} wg := &sync.WaitGroup{} ch := make(chan bool, 2) sess := util.MgoS.GetMgoConn() defer util.MgoS.DestoryMongoConn(sess) it := sess.DB(util.MgoS.DbName).C("spider_heart").Find(&query).Select(&fields).Iter() n := 0 for tmp := make(map[string]interface{}); it.Next(tmp); n++ { wg.Add(1) ch <- true go func(tmp map[string]interface{}) { defer func() { <-ch wg.Done() }() detailTime := qu.Int64All(tmp["detail"]) if time.Now().Unix()-detailTime > 3600*3 { //统计超过3小时前爬虫个数 event := qu.IntAll(tmp["event"]) lock.Lock() eventNumMap[event] += 1 lock.Unlock() } }(tmp) if n%100 == 0 { logger.Debug(n) } tmp = map[string]interface{}{} } wg.Wait() logger.Info(eventNumMap) content := "" for event, num := range eventNumMap { if num >= 10 { content += "节点:" + fmt.Sprint(event) + "详情页执行心跳异常个数:" + fmt.Sprint(num) + ";" } } if content != "" { toMyself := fmt.Sprintf(TextModel, content, "15637845493") logger.Info("toMyself", toMyself) resp, err := http.Post( "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e", "application/json", bytes.NewBuffer([]byte(toMyself)), ) if err != nil { fmt.Println("request error:", err) return } defer resp.Body.Close() } } // 统计爬虫审核人员待审核爬虫 func SendInfoToWxWork_ToAuditor() { defer qu.Catch() qu.Debug("企业微信发送提示信息") tmpContent := "当前" tmpModifyList := []string{} qu.Debug(LuaAuditorInfoMap) for eu, userInfo := range LuaAuditorInfoMap { query := map[string]interface{}{ "state": 1, "platform": "golua平台", } if eu == "niehaiyang" { //王丹婷审核的施顺才的爬虫 query["modifyuser"] = map[string]interface{}{ "$eq": "ssc", } } else { query["modifyuser"] = map[string]interface{}{ "$ne": "ssc", } } count := util.MgoEB.Count("luaconfig", query) tmpContent += userInfo.Username + "需审核的爬虫有:" + fmt.Sprint(count) + "个;" tmpModifyList = append(tmpModifyList, userInfo.Mobile) } msg := fmt.Sprintf(TextModel, tmpContent+"请及时处理!", strings.Join(tmpModifyList, ",")) qu.Debug("msg", msg) resp1, err := http.Post( "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e", "application/json", bytes.NewBuffer([]byte(msg)), ) if err != nil { fmt.Println("request error:", err) return } defer resp1.Body.Close() }