spider_task/src/timetask/wxworkwarn.go

package timetask

import (
	"bytes"
	"fmt"
	"github.com/donnie4w/go-logger/logger"
	"math"
	"net/http"
	qu "qfw/util"
	"strings"
	"sync"
	"time"
	"util"
)

var UserMap map[string]map[string]string
var PythonModifyUserInfoMap map[string]*UserInfo //开发人员信息集合
var LuaModifyUserInfoMap map[string]*UserInfo    //开发人员信息集合
var LuaAuditorInfoMap map[string]*UserInfo       //审核人员信息集合

//lua
type LuaUserTextInfo struct {
	Username             string
	FailedTaskCount      int
	FailedTaskOverdueDay int
	HeartErrCount        int
	NoCollectDataDay     int
}

//python
type PythonUserTextInfo struct {
	Username      string
	ToBeCompleted int //待完成爬虫个数
	Failed        int //未通过爬虫个数
}

type UserInfo struct {
	Username string
	UserID   string
	Email    string
	Mobile   string
	Auth     string
}

//
var LuaTitleContentModel = `
	截止目前，爬虫共有未通过任务<font color=\"warning\">%d个</font>，异常心跳爬虫<font color=\"warning\">%d个</font>。请及时处理！\n
`
var PythonTitleContentModel = `
	截止目前，共有待完成爬虫<font color=\"warning\">%d个</font>，未通过爬虫<font color=\"warning\">%d个</font>，待审核爬虫<font color=\"warning\">%d个</font>。请及时处理！\n
`

var LuaUserContentModel = `
     >人员：<font color=\"warning\">%s</font>
     >未通过任务：<font color=\"warning\">%d个</font><font color=\"info\">(最早任务已逾期%d天)</font>
     >异常心跳爬虫：<font color=\"warning\">%d个</font><font color=\"info\">(已有爬虫%d天未采集数据)</font>\n
`
var PythonUserContentModel = `
     >人员：<font color=\"warning\">%s</font>
     >待完成爬虫：<font color=\"warning\">%d个</font><font color=\"info\"></font>
     >未通过爬虫：<font color=\"warning\">%d个</font><font color=\"info\"></font>\n
`
var MarkdownModel = `{
    "msgtype": "markdown",
    "markdown": {
        "content": "%s"
    }
}`
var TextModel = `{
    "msgtype": "text",
    "text": {
        "content": "%s",
        "mentioned_mobile_list":[%s]
    }
}`

//初始化人员信息
func GetLuaUserInfo() {
	LuaModifyUserInfoMap = map[string]*UserInfo{}
	LuaAuditorInfoMap = map[string]*UserInfo{}
	PythonModifyUserInfoMap = map[string]*UserInfo{}
	for eu, info := range UserMap {
		role := info["role"]
		auth := info["auth"]
		if role == "lua" {
			if auth == "1" { //开发人员
				LuaModifyUserInfoMap[eu] = &UserInfo{
					Username: info["username"],
					UserID:   info["userid"],
					Email:    info["email"],
					Mobile:   info["mobile"],
					Auth:     auth,
				}
			}
			if auth == "3" || eu == "ssc" { //审核人员
				LuaAuditorInfoMap[eu] = &UserInfo{
					Username: info["username"],
					Mobile:   info["mobile"],
					Auth:     auth,
				}
			}
		} else {
			if auth == "1" {
				PythonModifyUserInfoMap[eu] = &UserInfo{
					Username: info["username"],
					Mobile:   info["mobile"],
					Auth:     auth,
				}
			}
		}
	}
	//
	index := 0
	for name, _ := range LuaModifyUserInfoMap {
		LuaUserIndexMap[index] = name
		index++
	}
	LuaUserNum = index
	qu.Debug(LuaModifyUserInfoMap)
	qu.Debug(LuaAuditorInfoMap)
	qu.Debug(PythonModifyUserInfoMap)
	qu.Debug(LuaUserIndexMap, LuaUserNum)
}

// 统计爬虫开发人员未完成爬虫和任务
func SendInfoToWxWork_Tomodifyuser() {
	SendLuaInfo()
	SendPythonInfo()
	SendPrivateInfo()
}

func SendLuaInfo() {
	defer qu.Catch()
	qu.Debug("lua企业微信发送提示信息")
	failedTaskCount, heartCodeCount := 0, 0         //总未通过任务个数，总待处理心跳异常爬虫个数
	luaUserTextMap := map[string]*LuaUserTextInfo{} //key:mobile
	for user, userInfo := range LuaModifyUserInfoMap {
		textInfo := &LuaUserTextInfo{}
		textInfo.Username = userInfo.Username
		//1、未通过任务信息
		list_task, _ := util.MgoEB.Find("task",
			map[string]interface{}{"s_modify": user, "i_state": 5},
			map[string]interface{}{"l_complete": 1},
			map[string]interface{}{"l_complete": 1},
			false, -1, -1)
		taskLen := len(*list_task)
		textInfo.FailedTaskCount = taskLen //个人未通过任务个数赋值
		failedTaskCount += taskLen         //总未通过个数++
		if taskLen > 0 {
			complete := qu.Int64All((*list_task)[0]["l_complete"]) //未通过任务中最迟完成时间最早的任务
			odDay := int(math.Floor(float64(time.Now().Unix()-complete) / float64(86400)))
			textInfo.FailedTaskOverdueDay = odDay //个人未通过任务最早逾期天数赋值
		}
		//2、爬虫心跳信息
		query := map[string]interface{}{
			"modifyuser": user,
			"del":        false,
			//"list": map[string]interface{}{
			//	"$lte": util.GetTime(0),
			//},
			"$or": []interface{}{
				map[string]interface{}{
					"event": map[string]interface{}{
						"$nin": []int{7000, 7520},
						//"$nin": []int{7500, 7510},
					},
					"list": map[string]interface{}{
						"$lte": util.GetTime(0),
					},
				},
				map[string]interface{}{
					"event": 7520,
					//"event": map[string]interface{}{
					//	"$in": []int{7500, 7510},
					//},
					"list": map[string]interface{}{
						"$lte": util.GetTime(-1),
					},
				},
			},
		}
		qu.Debug("heart query:", query)
		list_code, _ := util.MgoS.Find("spider_heart",
			query,
			map[string]interface{}{"list": 1},
			map[string]interface{}{"list": 1},
			false, -1, -1)
		codeLen := len(*list_code)
		textInfo.HeartErrCount = codeLen //个人异常心跳爬虫个数赋值
		heartCodeCount += codeLen        //总异常心跳爬虫个数++
		if codeLen > 0 {
			listTime := qu.Int64All((*list_code)[0]["list"]) //未通过任务中最迟完成时间最早的任务
			ncDay := int(math.Floor(float64(time.Now().Unix()-listTime) / float64(86400)))
			textInfo.NoCollectDataDay = ncDay //个人未通过任务最早逾期天数赋值
		}
		luaUserTextMap[userInfo.Mobile] = textInfo
	}
	//拼接content
	resultContent := fmt.Sprintf(LuaTitleContentModel, failedTaskCount, heartCodeCount)
	mobileArr := []string{}
	for mobile, t := range luaUserTextMap {
		mobileArr = append(mobileArr, "\""+mobile+"\"")
		resultContent += fmt.Sprintf(LuaUserContentModel, t.Username, t.FailedTaskCount, t.FailedTaskOverdueDay, t.HeartErrCount, t.NoCollectDataDay)
	}
	msg := fmt.Sprintf(MarkdownModel, resultContent)
	qu.Debug("msg", msg)
	toUserMsg := fmt.Sprintf(TextModel, "", strings.Join(mobileArr, ","))
	qu.Debug("toUserMsg", toUserMsg)
	resp1, err := http.Post(
		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
		"application/json",
		bytes.NewBuffer([]byte(toUserMsg)),
	)
	if err != nil {
		fmt.Println("request error:", err)
		return
	}
	defer resp1.Body.Close()
	resp2, err := http.Post(
		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
		"application/json",
		bytes.NewBuffer([]byte(msg)),
	)
	if err != nil {
		fmt.Println("request error:", err)
		return
	}
	defer resp2.Body.Close()
}

func SendPythonInfo() {
	defer qu.Catch()
	qu.Debug("python企业微信发送提示信息")
	toBeCompletedAllCount, failedAllCount := 0, 0
	toBeReviewedAllCount := util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 1, "platform": "python"})
	pythonUserTextMap := map[string]*PythonUserTextInfo{} //key:mobile
	for user, userInfo := range PythonModifyUserInfoMap {
		textInfo := &PythonUserTextInfo{}
		textInfo.Username = userInfo.Username
		//1、待完成爬虫个数
		textInfo.ToBeCompleted = util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 0, "modifyuser": user})
		toBeCompletedAllCount += textInfo.ToBeCompleted
		//2、未通过爬虫个数
		textInfo.Failed = util.MgoEB.Count("luaconfig", map[string]interface{}{"state": 2, "modifyuser": user})
		failedAllCount += textInfo.Failed
		pythonUserTextMap[userInfo.Mobile] = textInfo
	}
	//拼接content
	resultContent := fmt.Sprintf(PythonTitleContentModel, toBeCompletedAllCount, failedAllCount, toBeReviewedAllCount)
	mobileArr := []string{}
	for mobile, t := range pythonUserTextMap {
		mobileArr = append(mobileArr, "\""+mobile+"\"")
		resultContent += fmt.Sprintf(PythonUserContentModel, t.Username, t.ToBeCompleted, t.Failed)
	}
	msg := fmt.Sprintf(MarkdownModel, resultContent)
	qu.Debug("msg", msg)
	toUserMsg := fmt.Sprintf(TextModel, "", strings.Join(mobileArr, ","))
	qu.Debug("toUserMsg", toUserMsg)

	resp1, err := http.Post(
		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
		"application/json",
		bytes.NewBuffer([]byte(toUserMsg)),
	)
	if err != nil {
		fmt.Println("request error:", err)
		return
	}
	defer resp1.Body.Close()
	resp2, err := http.Post(
		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
		"application/json",
		bytes.NewBuffer([]byte(msg)),
	)
	if err != nil {
		fmt.Println("request error:", err)
		return
	}
	defer resp2.Body.Close()
}

//每日统计列表页、详情页分开采集模式节点爬虫详情页采集心跳异常信息
func SendPrivateInfo() {
	defer qu.Catch()
	var eventArr []int //节点集合
	eventNumMap := map[int]int{}
	for event, model := range util.CodeEventModel {
		if model == 1 {
			eventArr = append(eventArr, event)
		}
	}
	query := map[string]interface{}{
		"event": map[string]interface{}{
			"$in": eventArr,
		},
		"del": false,
	}
	fields := map[string]interface{}{
		"event":  1,
		"detail": 1,
	}
	logger.Info("query:", query)
	lock := &sync.Mutex{}
	wg := &sync.WaitGroup{}
	ch := make(chan bool, 2)
	sess := util.MgoS.GetMgoConn()
	defer util.MgoS.DestoryMongoConn(sess)
	it := sess.DB(util.MgoS.DbName).C("spider_heart").Find(&query).Select(&fields).Iter()
	n := 0
	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
		wg.Add(1)
		ch <- true
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			detailTime := qu.Int64All(tmp["detail"])
			if time.Now().Unix()-detailTime > 3600*3 { //统计超过3小时前爬虫个数
				event := qu.IntAll(tmp["event"])
				lock.Lock()
				eventNumMap[event] += 1
				lock.Unlock()
			}
		}(tmp)
		if n%100 == 0 {
			logger.Debug(n)
		}
		tmp = map[string]interface{}{}
	}
	wg.Wait()
	logger.Info(eventNumMap)
	content := ""
	for event, num := range eventNumMap {
		if num >= 10 {
			content += "节点：" + fmt.Sprint(event) + "详情页执行心跳异常个数：" + fmt.Sprint(num) + "；"
		}
	}
	if content != "" {
		toMyself := fmt.Sprintf(TextModel, content, "15637845493")
		logger.Info("toMyself", toMyself)
		resp, err := http.Post(
			"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
			"application/json",
			bytes.NewBuffer([]byte(toMyself)),
		)
		if err != nil {
			fmt.Println("request error:", err)
			return
		}
		defer resp.Body.Close()
	}
}

// 统计爬虫审核人员待审核爬虫
func SendInfoToWxWork_ToAuditor() {
	defer qu.Catch()
	qu.Debug("企业微信发送提示信息")
	tmpContent := "当前"
	tmpModifyList := []string{}
	qu.Debug(LuaAuditorInfoMap)
	for eu, userInfo := range LuaAuditorInfoMap {
		query := map[string]interface{}{
			"state":    1,
			"platform": "golua平台",
		}
		if eu == "niehaiyang" { //王丹婷审核的施顺才的爬虫
			query["modifyuser"] = map[string]interface{}{
				"$eq": "ssc",
			}
		} else {
			query["modifyuser"] = map[string]interface{}{
				"$ne": "ssc",
			}
		}
		count := util.MgoEB.Count("luaconfig", query)
		tmpContent += userInfo.Username + "需审核的爬虫有：" + fmt.Sprint(count) + "个；"
		tmpModifyList = append(tmpModifyList, userInfo.Mobile)
	}
	msg := fmt.Sprintf(TextModel, tmpContent+"请及时处理！", strings.Join(tmpModifyList, ","))
	qu.Debug("msg", msg)
	resp1, err := http.Post(
		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=97850772-88d0-4544-a2c3-6201aeddff9e",
		"application/json",
		bytes.NewBuffer([]byte(msg)),
	)
	if err != nil {
		fmt.Println("request error:", err)
		return
	}
	defer resp1.Body.Close()

}