package timetask
import (
"bytes"
"fmt"
"github.com/donnie4w/go-logger/logger"
"math"
"net/http"
qu "qfw/util"
"strings"
"sync"
"time"
"util"
)
var UserMap map[string]map[string]string
var PythonModifyUserInfoMap map[string]*UserInfo //开发人员信息集合
var LuaModifyUserInfoMap map[string]*UserInfo //开发人员信息集合
var LuaAuditorInfoMap map[string]*UserInfo //审核人员信息集合
//lua
type LuaUserTextInfo struct {
Username string
FailedTaskCount int
FailedTaskOverdueDay int
HeartErrCount int
NoCollectDataDay int
}
//python
type PythonUserTextInfo struct {
Username string
ToBeCompleted int //待完成爬虫个数
Failed int //未通过爬虫个数
}
type UserInfo struct {
Username string
UserID string
Email string
Mobile string
Auth string
}
//
var LuaTitleContentModel = `
截止目前,爬虫共有未通过任务%d个,异常心跳爬虫%d个。请及时处理!\n
`
var PythonTitleContentModel = `
截止目前,共有待完成爬虫%d个,未通过爬虫%d个,待审核爬虫%d个。请及时处理!\n
`
var LuaUserContentModel = `
>人员:%s
>未通过任务:%d个(最早任务已逾期%d天)
>异常心跳爬虫:%d个(已有爬虫%d天未采集数据)\n
`
var PythonUserContentModel = `
>人员:%s
>待完成爬虫:%d个
>未通过爬虫:%d个\n
`
var MarkdownModel = `{
"msgtype": "markdown",
"markdown": {
"content": "%s"
}
}`
var TextModel = `{
"msgtype": "text",
"text": {
"content": "%s",
"mentioned_mobile_list":[%s]
}
}`
//初始化人员信息
func GetLuaUserInfo() {
LuaModifyUserInfoMap = map[string]*UserInfo{}
LuaAuditorInfoMap = map[string]*UserInfo{}
PythonModifyUserInfoMap = map[string]*UserInfo{}
for eu, info := range UserMap {
role := info["role"]
auth := info["auth"]
if role == "lua" {
if auth == "1" { //开发人员
LuaModifyUserInfoMap[eu] = &UserInfo{
Username: info["username"],
UserID: info["userid"],
Email: info["email"],
Mobile: info["mobile"],
Auth: auth,
}
}
if auth == "3" || eu == "ssc" { //审核人员
LuaAuditorInfoMap[eu] = &UserInfo{
Username: info["username"],
Mobile: info["mobile"],
Auth: auth,
}
}
} else {
if auth == "1" {
PythonModifyUserInfoMap[eu] = &UserInfo{
Username: info["username"],
Mobile: info["mobile"],
Auth: auth,
}
}
}
}
//
index := 0
for name, _ := range LuaModifyUserInfoMap {
LuaUserIndexMap[index] = name
index++
}
LuaUserNum = index
qu.Debug(LuaModifyUserInfoMap)
qu.Debug(LuaAuditorInfoMap)
qu.Debug(PythonModifyUserInfoMap)
qu.Debug(LuaUserIndexMap, LuaUserNum)
}
// 统计爬虫开发人员未完成爬虫和任务
func SendInfoToWxWork_Tomodifyuser() {
SendLuaInfo()
SendPythonInfo()
SendPrivateInfo()
}
func SendLuaInfo() {
defer qu.Catch()
qu.Debug("lua企业微信发送提示信息")
failedTaskCount, heartCodeCount := 0, 0 //总未通过任务个数,总待处理心跳异常爬虫个数
luaUserTextMap := map[string]*LuaUserTextInfo{} //key:mobile
for user, userInfo := range LuaModifyUserInfoMap {
textInfo := &LuaUserTextInfo{}
textInfo.Username = userInfo.Username
//1、未通过任务信息
list_task, _ := util.MgoEB.Find("task",
map[string]interface{}{"s_modify": user, "i_state": 5},
map[string]interface{}{"l_complete": 1},
map[string]interface{}{"l_complete": 1},
false, -1, -1)
taskLen := len(*list_task)
textInfo.FailedTaskCount = taskLen //个人未通过任务个数赋值
failedTaskCount += taskLen //总未通过个数++
if taskLen > 0 {
complete := qu.Int64All((*list_task)[0]["l_complete"]) //未通过任务中最迟完成时间最早的任务
odDay := int(math.Floor(float64(time.Now().Unix()-complete) / float64(86400)))
textInfo.FailedTaskOverdueDay = odDay //个人未通过任务最早逾期天数赋值
}
//2、爬虫心跳信息
query := map[string]interface{}{
"modifyuser": user,
"del": false,
//"list": map[string]interface{}{
// "$lte": util.GetTime(0),
//},
"$or": []interface{}{
map[string]interface{}{
"event": map[string]interface{}{
"$nin": []int{7000, 7520},
//"$nin": []int{7500, 7510},
},
"list": map[string]interface{}{
"$lte": util.GetTime(0),
},
},
map[string]interface{}{
"event": 7520,
//"event": map[string]interface{}{
// "$in": []int{7500, 7510},
//},
"list": map[string]interface{}{
"$lte": util.GetTime(-1),
},
},
},
}
qu.Debug("heart query:", query)
list_code, _ := util.MgoS.Find("spider_heart",
query,
map[string]interface{}{"list": 1},
map[string]interface{}{"list": 1},
false, -1, -1)
codeLen := len(*list_code)
textInfo.HeartErrCount = codeLen //个人异常心跳爬虫个数赋值
heartCodeCount += codeLen //总异常心跳爬虫个数++
if codeLen > 0 {
listTime := qu.Int64All((*list_code)[0]["list"]) //未通过任务中最迟完成时间最早的任务
ncDay := int(math.Floor(float64(time.Now().Unix()-listTime) / float64(86400)))
textInfo.NoCollectDataDay = ncDay //个人未通过任务最早逾期天数赋值
}
luaUserTextMap[userInfo.Mobile] = textInfo
}
//拼接content
resultContent := fmt.Sprintf(LuaTitleContentModel, failedTaskCount, heartCodeCount)
mobileArr := []string{}
for mobile, t := range luaUserTextMap {
mobileArr = append(mobileArr, "\""+mobile+"\"")
resultContent += fmt.Sprintf(LuaUserContentModel, t.Username, t.FailedTaskCount, t.FailedTaskOverdueDay, t.HeartErrCount, t.NoCollectDataDay)
}
msg := fmt.Sprintf(MarkdownModel, resultContent)
qu.Debug("msg", msg)
toUserMsg := fmt.Sprintf(TextModel, "", strings.Join(mobileArr, ","))
qu.Debug("toUserMsg", toUserMsg)
resp1, err := http.Post(
"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
"application/json",
bytes.NewBuffer([]byte(toUserMsg)),
)
if err != nil {
fmt.Println("request error:", err)
return
}
defer resp1.Body.Close()
resp2, err := http.Post(
"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
"application/json",
bytes.NewBuffer([]byte(msg)),
)
if err != nil {
fmt.Println("request error:", err)
return
}
defer resp2.Body.Close()
}
func SendPythonInfo() {
defer qu.Catch()
qu.Debug("python企业微信发送提示信息")
toBeCompletedAllCount, failedAllCount := 0, 0
toBeReviewedAllCount := util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 1, "platform": "python"})
pythonUserTextMap := map[string]*PythonUserTextInfo{} //key:mobile
for user, userInfo := range PythonModifyUserInfoMap {
textInfo := &PythonUserTextInfo{}
textInfo.Username = userInfo.Username
//1、待完成爬虫个数
textInfo.ToBeCompleted = util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 0, "modifyuser": user})
toBeCompletedAllCount += textInfo.ToBeCompleted
//2、未通过爬虫个数
textInfo.Failed = util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 2, "modifyuser": user})
failedAllCount += textInfo.Failed
pythonUserTextMap[userInfo.Mobile] = textInfo
}
//拼接content
resultContent := fmt.Sprintf(PythonTitleContentModel, toBeCompletedAllCount, failedAllCount, toBeReviewedAllCount)
mobileArr := []string{}
for mobile, t := range pythonUserTextMap {
mobileArr = append(mobileArr, "\""+mobile+"\"")
resultContent += fmt.Sprintf(PythonUserContentModel, t.Username, t.ToBeCompleted, t.Failed)
}
msg := fmt.Sprintf(MarkdownModel, resultContent)
qu.Debug("msg", msg)
toUserMsg := fmt.Sprintf(TextModel, "", strings.Join(mobileArr, ","))
qu.Debug("toUserMsg", toUserMsg)
resp1, err := http.Post(
"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
"application/json",
bytes.NewBuffer([]byte(toUserMsg)),
)
if err != nil {
fmt.Println("request error:", err)
return
}
defer resp1.Body.Close()
resp2, err := http.Post(
"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
"application/json",
bytes.NewBuffer([]byte(msg)),
)
if err != nil {
fmt.Println("request error:", err)
return
}
defer resp2.Body.Close()
}
//每日统计列表页、详情页分开采集模式节点爬虫详情页采集心跳异常信息
func SendPrivateInfo() {
defer qu.Catch()
var eventArr []int //节点集合
eventNumMap := map[int]int{}
for event, model := range util.CodeEventModel {
if model == 1 {
eventArr = append(eventArr, event)
}
}
query := map[string]interface{}{
"event": map[string]interface{}{
"$in": eventArr,
},
"del": false,
}
fields := map[string]interface{}{
"event": 1,
"detail": 1,
}
logger.Info("query:", query)
lock := &sync.Mutex{}
wg := &sync.WaitGroup{}
ch := make(chan bool, 2)
sess := util.MgoS.GetMgoConn()
defer util.MgoS.DestoryMongoConn(sess)
it := sess.DB(util.MgoS.DbName).C("spider_heart").Find(&query).Select(&fields).Iter()
n := 0
for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
wg.Add(1)
ch <- true
go func(tmp map[string]interface{}) {
defer func() {
<-ch
wg.Done()
}()
detailTime := qu.Int64All(tmp["detail"])
if time.Now().Unix()-detailTime > 3600*3 { //统计超过3小时前爬虫个数
event := qu.IntAll(tmp["event"])
lock.Lock()
eventNumMap[event] += 1
lock.Unlock()
}
}(tmp)
if n%100 == 0 {
logger.Debug(n)
}
tmp = map[string]interface{}{}
}
wg.Wait()
logger.Info(eventNumMap)
content := ""
for event, num := range eventNumMap {
if num >= 10 {
content += "节点:" + fmt.Sprint(event) + "详情页执行心跳异常个数:" + fmt.Sprint(num) + ";"
}
}
if content != "" {
toMyself := fmt.Sprintf(TextModel, content, "15637845493")
logger.Info("toMyself", toMyself)
resp, err := http.Post(
"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
"application/json",
bytes.NewBuffer([]byte(toMyself)),
)
if err != nil {
fmt.Println("request error:", err)
return
}
defer resp.Body.Close()
}
}
// 统计爬虫审核人员待审核爬虫
func SendInfoToWxWork_ToAuditor() {
defer qu.Catch()
qu.Debug("企业微信发送提示信息")
tmpContent := "当前"
tmpModifyList := []string{}
qu.Debug(LuaAuditorInfoMap)
for eu, userInfo := range LuaAuditorInfoMap {
query := map[string]interface{}{
"state": 1,
"platform": "golua平台",
}
if eu == "niehaiyang" { //王丹婷审核的施顺才的爬虫
query["modifyuser"] = map[string]interface{}{
"$eq": "ssc",
}
} else {
query["modifyuser"] = map[string]interface{}{
"$ne": "ssc",
}
}
count := util.MgoEB.Count("luaconfig", query)
tmpContent += userInfo.Username + "需审核的爬虫有:" + fmt.Sprint(count) + "个;"
tmpModifyList = append(tmpModifyList, userInfo.Mobile)
}
msg := fmt.Sprintf(TextModel, tmpContent+"请及时处理!", strings.Join(tmpModifyList, ","))
qu.Debug("msg", msg)
resp1, err := http.Post(
"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
"application/json",
bytes.NewBuffer([]byte(msg)),
)
if err != nil {
fmt.Println("request error:", err)
return
}
defer resp1.Body.Close()
}