邮件格式修改-vps报警-日志

data_monitoring/listen_task/src/config.json

@@ -8,7 +8,7 @@
   "qy_mgodb": {
     "addr": "192.168.3.207:27092",
     "db": "zhengkun",
-    "coll": "baidu_enterprise",
+    "coll": "baidu_enterprise_test",
     "pool": 5
   },
   "jp_mgodb": {
@@ -30,6 +30,7 @@
   "smtpMail": {
     "from": "zhengkun@topnet.net.cn",
     "to": "zhengkun@topnet.net.cn",
+    "cc": "zhengkun@topnet.net.cn",
     "smtpHost": "smtp.qq.com",
     "smtpPort": "465",
     "user":     "920032221@qq.com",

data_monitoring/listen_task/src/dataTaskJP.go

@@ -10,8 +10,8 @@ func dealWithJPData()(map[string]interface{}) {
 
 	defer qu.Catch()
 	now:=time.Now() //当前天
-	gte_time:= time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local).Unix()
-	lt_time := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
+	lt_time:= time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
+	gte_time := lt_time-86400
 
 	q := map[string]interface{}{
 		"comeintime": map[string]interface{}{
@@ -37,8 +37,10 @@ func dealWithJPData()(map[string]interface{}) {
 		tmp = make(map[string]interface{})
 	}
 
+
+
 	return map[string]interface{}{
-		"total":total,
+		"num":total,
 		"data":dict,
 		"date":qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),
 	}

data_monitoring/listen_task/src/dataTaskQY.go

@@ -13,8 +13,8 @@ func dealWithQYData()(string,string)  {
 	defer qu.Catch()
 
 	now:=time.Now() //当前天
-	gte_time:= time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local).Unix()
-	lt_time := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
+	lt_time:= time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
+	gte_time := lt_time-86400
 
 	q := map[string]interface{}{
 		"down_time": map[string]interface{}{
@@ -22,6 +22,7 @@ func dealWithQYData()(string,string)  {
 			"$lt": lt_time,
 		},
 	}
+
 	sess := qy_mgo.GetMgoConn()
 	defer qy_mgo.DestoryMongoConn(sess)
 

data_monitoring/listen_task/src/dataTaskST.go

@@ -13,28 +13,24 @@ import (
 
 func dealWithSTData()  {
 
-	log.Println("开始统计站点数据...")
+	log.Println("开始统计全量站点数据...")
 	defer qu.Catch()
 
 	start := int(time.Now().Unix())
 	now:=time.Now() //当前天
-	gte_time:= time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local).Unix()
-	lt_time := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
-
-	gte_time = 1618070400
-	lt_time = 1618156800
-
+	lt_time:= time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
+	gte_time := lt_time-86400
 
 	mgodata := toCalculateMgoData(gte_time,lt_time)
 	esdata := toCalculateEsData(fmt.Sprintf("%d",gte_time),fmt.Sprintf("%d",lt_time))
 	mgonum,esnum:= qu.IntAll(mgodata["totalnum"]),qu.IntAll(esdata["totalnum"])
 
 	comeintime:=qu.Int64All(time.Now().Unix())
-	date := qu.FormatDateByInt64(&comeintime, qu.DATEFORMAT)
+	date := qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT)
 	save_mgo.Save("monitor_site", map[string]interface{}{
 		"comeintime":comeintime,
 		"date":date,
-		"name":"es-mgo-站点",
+		"name":"全量站点",
 		"mgonum":mgonum,
 		"esnum":esnum,
 		"mgodata":mgodata["detail"],
@@ -68,8 +64,8 @@ func toCalculateEsData(gte_time string,lt_time string) map[string]interface{} {
 	dict:= make(map[string]interface{},0)
 	total :=0
 
-	//elastic.InitElasticSize("http://172.17.145.170:9800", 10,)
-	elastic.InitElasticSize("http://127.0.0.1:12003", 10,)
+	elastic.InitElasticSize("http://172.17.145.170:9800", 10,)
+	//elastic.InitElasticSize("http://127.0.0.1:12003", 10,)
 	esclient := elastic.GetEsConn()
 	defer elastic.DestoryEsConn(esclient)
 	if esclient == nil {

data_monitoring/listen_task/src/main.go

@@ -13,11 +13,12 @@ import (
 
 var (
 	sysconfig    							map[string]interface{} //配置文件
-	save_mgo,qy_mgo,jp_mgo,st_mgo        			*MongodbSim            //mongodb操作对象
+	save_mgo,qy_mgo,jp_mgo,st_mgo        	*MongodbSim            //mongodb操作对象
 	qy_c_name,jp_c_name,st_c_name			string
 	es_index_st,es_type_st					string
 	save_other_name,save_site_name			string
 	xlsx_name								string
+	infoSiteArr								[]string
 )
 
 func initMgo()  {
@@ -69,30 +70,21 @@ func initMgo()  {
 	xlsx_name = qu.ObjToString(sysconfig["xlsx_name"])
 }
 
-
 func init() {
 	//加载配置文件
 	qu.ReadConfig(&sysconfig)
 	initMgo()
 }
 
-func mainT() {
-
+func main() {
 	go taskTime()
-	time.Sleep(99999 * time.Hour)
 
-}
-
-//测试使用
-func main() {
+	time.Sleep(99999 * time.Hour)
 
-	dealWithOtherData()
 }
 
 func taskTime()  {
 
-
-
 	log.Println("部署定时任务")
 	c := cron.New()
 
@@ -111,48 +103,57 @@ func dealWithOtherData()  {
 
 	log.Println("开始统计相关数据...")
 
-	start := int(time.Now().Unix())
-	data :=dealWithJPData()//站点数据
-	qy_date,qy_total := dealWithQYData()//企业数据
+	//先获取-待统计-动态站点
+	tmp ,tmp_err := save_mgo.Find("z_site_spider", map[string]interface{}{},nil,nil)
+	if tmp_err==nil && tmp!=nil && len(tmp)>0 {
+		for _,v :=range tmp {
+			infoSiteArr = append(infoSiteArr,qu.ObjToString(v["site"]))
+		}
+		log.Println("目标站点：",len(infoSiteArr))
+	}
 
+	start := int(time.Now().Unix())
+	st_data :=dealWithJPData()//站点数据
+	qy_date,qy_num := dealWithQYData()//企业数据
 	comeintime :=qu.Int64All(time.Now().Unix())
 	save_mgo.Save(save_other_name, map[string]interface{}{
 		"name":"企业",
-		"num":qu.IntAll(qy_total),
+		"num":qu.IntAll(qy_num),
 		"comeintime":comeintime,
 		"date":qy_date,
 	})
 
 	save_mgo.Save(save_other_name, map[string]interface{}{
 		"name":"站点",
-		"num":qu.IntAll(data["total"]),
+		"num":qu.IntAll(st_data["num"]),
 		"comeintime":comeintime,
-		"date":qu.ObjToString(data["date"]),
-		"data":data["data"],
+		"date":qu.ObjToString(st_data["date"]),
+		"data":st_data["data"],
 	})
 
 
+
 	//获取前两天数据
-	qy_arr ,st_arr:= make([]map[string]string,0),make([]map[string]interface{},0)
+	qy_arr := make([]map[string]string,0)
 	now:=time.Now() //当前天
-	curtime:= time.Date(now.Year(), now.Month(), now.Day()-2, 0, 0, 0, 0, time.Local).Unix()
-	before_date :=qu.FormatDateByInt64(&curtime, qu.DATEFORMAT)
+	cur_time:= time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
+	before_time := cur_time-86400*2
+	before_date :=qu.FormatDateByInt64(&before_time, qu.DATEFORMAT)
 	//企业数据
 	before_qy_data :=save_mgo.FindOne(save_other_name, map[string]interface{}{
 		"name":"企业",
 		"date":before_date,
 	})
+	qy_arr = append(qy_arr, map[string]string{
+		"date":qy_date,
+		"num":qy_num,
+	})
 	if before_qy_data!=nil && len(before_qy_data)>2 {
 		qy_arr = append(qy_arr, map[string]string{
 			"date":qu.ObjToString(before_qy_data["date"]),
 			"num":fmt.Sprintf("%d",before_qy_data["num"]),
 		})
 	}
-	qy_arr = append(qy_arr, map[string]string{
-		"date":qy_date,
-		"num":qy_total,
-	})
-
 
 
 	//站点数据
@@ -160,63 +161,53 @@ func dealWithOtherData()  {
 		"name":"站点",
 		"date":before_date,
 	})
-	if before_st_data!=nil && len(before_st_data)>2 {
-		st_arr = append(st_arr, map[string]interface{}{
-			"date":qu.ObjToString(before_st_data["date"]),
-			"total":fmt.Sprintf("%d",before_st_data["num"]),
-			"data":before_st_data["data"],
-		})
-	}
 
-	st_arr = append(st_arr, map[string]interface{}{
-		"date":qu.ObjToString(data["date"]),
-		"total":fmt.Sprintf("%d",data["total"]),
-		"data":data["data"],
-	})
 
-	//写excle
-	os.Remove("统计.xlsx")
+	os.Remove("统计.xlsx")	//写excle
 	f :=xlsx.NewFile()
 	sheet, _ := f.AddSheet("统计")
 
-	//第一行先写标题
+	//企业数据
 	row := sheet.AddRow()
 	row.AddCell().Value = "企业分类/日期"
-	row.AddCell().Value = "总数量"
+	row.AddCell().Value = "数量"
 	for _,tmp := range qy_arr {
 		row = sheet.AddRow()
 		row.AddCell().SetString(tmp["date"])
 		row.AddCell().SetString(tmp["num"])
 	}
+
+
+	//站点相关数据
 	sheet.AddRow()
 	row = sheet.AddRow()
 	row.AddCell().Value = "站点分类/日期"
-	row.AddCell().Value = "总数量"
-	row.AddCell().Value = "站点名称"
-	row.AddCell().Value = "站点数量"
-	for _, v:= range st_arr {
-		tmp := *qu.ObjToMap(v)
+	row.AddCell().SetString(qu.ObjToString(st_data["date"]))
+	row.AddCell().SetString(qu.ObjToString(before_st_data["date"]))
+
+
+	data_1 := *qu.ObjToMap(st_data["data"])
+	data_2 := *qu.ObjToMap(before_st_data["data"])
+
+	for _,key := range infoSiteArr {
 		row = sheet.AddRow()
-		row.AddCell().SetString(qu.ObjToString(tmp["date"]))
-		row.AddCell().SetString(qu.ObjToString(tmp["total"]))
-
-		data := *qu.ObjToMap(tmp["data"])
-		if data!=nil {
-			for k,v := range data {
-				row.AddCell().SetString(k)
-				row.AddCell().SetString(fmt.Sprintf("%d",v))
-				row = sheet.AddRow()
-				row.AddCell().Value = ""
-				row.AddCell().Value = ""
-			}
-		}
+		row.AddCell().SetString(key)
+		row.AddCell().SetString(fmt.Sprintf("%d",qu.Int64All(data_1[key])))
+		row.AddCell().SetString(fmt.Sprintf("%d",qu.Int64All(data_2[key])))
 	}
+	row = sheet.AddRow()
+	row.AddCell().Value = "总计"
+	row.AddCell().SetString(fmt.Sprintf("%d",qu.Int64All(st_data["num"])))
+	row.AddCell().SetString(fmt.Sprintf("%d",qu.Int64All(before_st_data["num"])))
 
 	err := f.Save(xlsx_name)
 	if err != nil {
 		log.Println("保存xlsx失败:", err)
+	}else {
+		log.Println("保存xlsx成功:", err)
 	}
 
+
 	sendErrMailSmtp("统计-站点-企业","详情请查阅附件")
 
 	log.Println("定时处理完毕...",int(time.Now().Unix())-start,"秒")

data_monitoring/listen_task/src/mark

@@ -0,0 +1,40 @@
+{
+  "save_mgodb": {
+    "addr": "172.17.4.87:27080",
+    "db": "editor",
+    "coll": "monitor_other",
+    "pool": 5
+  },
+  "qy_mgodb": {
+    "addr": "172.17.4.87:27080",
+    "db": "spider",
+    "coll": "baidu_enterprise",
+    "pool": 5
+  },
+  "jp_mgodb": {
+    "addr": "172.17.145.179:28082",
+    "db": "spider_data",
+    "coll": "data_bak",
+    "pool": 5
+  },
+  "st_mgodb": {
+    "addr": "172.17.4.85:27080",
+    "db": "qfw",
+    "coll": "result_20210109",
+    "pool": 5
+  },
+  "es_index_st": "bidding",
+  "es_type_st": "bidding",
+  "save_other_name": "monitor_other",
+  "save_site_name" : "monitor_site",
+  "smtpMail": {
+    "from": "zhengkun@topnet.net.cn",
+    "to": "zhengkun@topnet.net.cn",
+    "cc": "zhengkun@topnet.net.cn",
+    "smtpHost": "smtp.qq.com",
+    "smtpPort": "465",
+    "user":     "920032221@qq.com",
+    "pwd":      "xomkphsjsamybdbj"
+  },
+  "xlsx_name" : "统计.xlsx"
+}

data_monitoring/listen_task/src/sendmail.go

@@ -8,7 +8,7 @@ import (
 	"qfw/util/mail"
 )
 
-var from,to, smtpHost,user,pwd string
+var from,to,cc, smtpHost,user,pwd string
 var smtpPort int
 
 
@@ -20,6 +20,7 @@ func sendErrMailSmtp(title,body string) {
 	if smtpMail != nil {
 		from, _ = smtpMail["from"].(string)
 		to, _ = smtpMail["to"].(string)
+		cc, _ = smtpMail["cc"].(string)
 		smtpHost, _ = smtpMail["smtpHost"].(string)
 		smtpPort= qu.IntAll(smtpMail["smtpPort"])
 		user, _ = smtpMail["user"].(string)
@@ -32,7 +33,7 @@ func sendErrMailSmtp(title,body string) {
 		return
 	}
 
-	ok := mail.GSendMail_B(user, from, from, from, title, body, f.Name(), b, &mail.GmailAuth{
+	ok := mail.GSendMail_B(user, from, cc, from, title, body, f.Name(), b, &mail.GmailAuth{
 		SmtpHost: smtpHost,
 		SmtpPort: smtpPort,
 		User:     user,

data_monitoring/vps_client/src/main.go

@@ -40,7 +40,7 @@ func main() {
 	//定时器
 	c := cron.New()
 	spec := fmt.Sprintf("0 */%s * * * ?",during)	//分
-	spec =fmt.Sprintf("*/%s * * * * ?",during)		//秒
+	//spec =fmt.Sprintf("*/%s * * * * ?",during)		//秒
 	c.AddFunc(spec, func() { task() })
 	c.Start()
 	time.Sleep(99999 * time.Hour)
@@ -66,8 +66,8 @@ func task()  {
 		}
 	}
 	//log.Println("当前下载器：",process)
-	u, _ := url.Parse("http://127.0.0.1:7811") //本地测试
-	//u, _ := url.Parse("http://monitor.spdata.jianyu360.com") //线上
+	//u, _ := url.Parse("http://127.0.0.1:7811") //本地测试
+	u, _ := url.Parse("http://monitor.spdata.jianyu360.com") //线上
 	q := u.Query()
 	q.Set("id", vpsID)
 	q.Set("process", process)

data_monitoring/vps_server/src/config.json

@@ -1,10 +1,16 @@
 {
   "port": "7811",
+  "save_mgodb": {
+    "addr": "192.168.3.207:27092",
+    "db": "zhengkun",
+    "coll": "monitor_other",
+    "pool": 5
+  },
   "vpsIDs" : [
     "专用-常州"
   ],
   "during": 10,
-  "isErr" : 5,
+  "isErr" : 3,
   "smtpMail": {
     "from": "zhengkun@topnet.net.cn",
     "to": "zhengkun@topnet.net.cn",

data_monitoring/vps_server/src/main.go

@@ -13,11 +13,13 @@ import (
 var (
 	sysconfig			map[string]interface{} //配置文件
 	port				string
+	save_mgo        	*MongodbSim
 	idsArr				[]string
 	dataTmp				map[string]map[string]interface{}
 	during,isErr		int64
 	test				map[string]interface{}
 	updatelock 			sync.Mutex
+	save_coll_name		string
 )
 func init()  {
 	//加载配置文件
@@ -39,6 +41,17 @@ func init()  {
 	}
 	during = qu.Int64All(sysconfig["during"])
 	isErr = qu.Int64All(sysconfig["isErr"])
+
+	saveconf := sysconfig["save_mgodb"].(map[string]interface{})
+	save_coll_name = qu.ObjToString(saveconf["coll"])
+	save_mgo = &MongodbSim{
+		MongodbAddr: saveconf["addr"].(string),
+		DbName:      saveconf["db"].(string),
+		Size:        qu.IntAllDef(saveconf["pool"], 5),
+	}
+	save_mgo.InitPool()
+
+
 	log.Println("准备完毕...")
 }
 
@@ -53,8 +66,8 @@ func main() {
 
 	//每隔1分钟执行一次：0 */1 * * * ?   每隔5秒执行一次：*/5 * * * * ?
 
-	spec :=fmt.Sprintf("0 */%d * * * ?",during)
-	spec =fmt.Sprintf("*/%d * * * * ?",during)
+	spec :=fmt.Sprintf("30 */%d * * * ?",during)
+	//spec =fmt.Sprintf("*/%d * * * * ?",during)
 	c := cron.New()
 	c.AddFunc(spec, func() { taskFinishing()})
 	c.Start()
@@ -103,10 +116,9 @@ func handler(w http.ResponseWriter, r *http.Request) {
 
 //不断监听处理
 func taskFinishing()  {
-	log.Println("...处理一次...")
 	//加锁
 	updatelock.Lock()
-
+	log.Println("...处理一次...")
 
 	isVpsMailContent,isProMailContent:= "",""
 	for _ , vpsid := range idsArr {
@@ -157,19 +169,33 @@ func taskFinishing()  {
 	}
 
 	//log.Println("处理后",isProMailContent)
+
 	if isVpsMailContent!=""{
 		log.Println("发邮件：vps异常...",isVpsMailContent)
-		sendErrMailApi("vps异常",isVpsMailContent)
+		comeintime :=qu.Int64All(time.Now().Unix())//日志记录
+		save_mgo.Save(save_coll_name, map[string]interface{}{
+			"name":"vps",
+			"comeintime":comeintime,
+			"date":qu.FormatDateByInt64(&comeintime, qu.DATEFORMAT),
+			"detail" : isVpsMailContent,
+		})
+
+		sendErrMailApi("vps",isVpsMailContent)
 	}else {
 		if isProMailContent !="" {
 			log.Println("发邮件：下载器异常...",isProMailContent)
+			comeintime :=qu.Int64All(time.Now().Unix())//日志记录
+			save_mgo.Save(save_coll_name, map[string]interface{}{
+				"name":"下载器",
+				"comeintime":comeintime,
+				"date":qu.FormatDateByInt64(&comeintime, qu.DATEFORMAT),
+				"detail" : isProMailContent,
+			})
+
 			sendErrMailApi("下载器异常",isProMailContent)
 		}
 	}
 
-
-
-
 	updatelock.Unlock()
 
 }