企业-站点数据 统计xlsx发送

data_monitoring/listen_task/src/config.json

@@ -1,5 +1,11 @@
 {
-  "data_mgodb": {
+  "save_mgodb": {
+    "addr": "192.168.3.207:27092",
+    "db": "zhengkun",
+    "coll": "monitor_other",
+    "pool": 5
+  },
+  "qy_mgodb": {
     "addr": "192.168.3.207:27092",
     "db": "zhengkun",
     "coll": "baidu_enterprise",
@@ -21,8 +27,13 @@
   "es_type_st": "bidding",
   "save_other_name": "monitor_other",
   "save_site_name" : "monitor_site",
-  "jkmail": {
+  "smtpMail": {
+    "from": "zhengkun@topnet.net.cn",
     "to": "zhengkun@topnet.net.cn",
-    "api": "http://172.17.145.179:19281/_send/_mail"
-  }
+    "smtpHost": "smtp.qq.com",
+    "smtpPort": "465",
+    "user":     "920032221@qq.com",
+    "pwd":      "xomkphsjsamybdbj"
+  },
+  "xlsx_name" : "统计.xlsx"
 }

data_monitoring/listen_task/src/dataTaskJP.go

@@ -1,18 +1,14 @@
 package main
 
 import (
-	"fmt"
-	"log"
 	qu "qfw/util"
 	"time"
 )
 
 
-func dealWithJPData()  {
+func dealWithJPData()(map[string]interface{}) {
 
-	log.Println("开始统计竞品数据...")
 	defer qu.Catch()
-	start := int(time.Now().Unix())
 	now:=time.Now() //当前天
 	gte_time:= time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local).Unix()
 	lt_time := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
@@ -29,76 +25,22 @@ func dealWithJPData()  {
 	it := sess.DB(jp_mgo.DbName).C(jp_c_name).Find(&q).Select(map[string]interface{}{
 		"site":1,
 	}).Iter()
-	total,isOK,_ :=0, 0,int(time.Now().Unix())
+	total,dict:=0,make(map[string]interface{},0)
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if qu.ObjToString(tmp["site"]) == "中国招标与采购网" {
-			isOK++
+		site := qu.ObjToString(tmp["site"])
+		if dict[site] == nil {
+			dict[site] = 1
+		}else {
+			num := qu.IntAll(dict[site])+1
+			dict[site] = num
 		}
 		tmp = make(map[string]interface{})
 	}
 
-	//是否告警条件
-	if total<1 {
-		//sendErrMailApi("竞品数据异常","数量无")
-	}
-
-	data_mgo.Save("monitor_other", map[string]interface{}{
-		"name":"竞品",
-		"num":qu.IntAll(total),
-		"comeintime":qu.Int64All(time.Now().Unix()),
+	return map[string]interface{}{
+		"total":total,
+		"data":dict,
 		"date":qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),
-	})
-
-	log.Println("竞品-定时处理完毕...",int(time.Now().Unix())-start,"秒")
+	}
 }
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-//测试分析
-func dealWithJPTest(year int,month time.Month, day int)  {
-	gte_time:= time.Date(year, month, day, 0, 0, 0, 0, time.Local).Unix()
-	lt_time := time.Date(year, month, day+1, 0, 0, 0, 0, time.Local).Unix()
-	q := map[string]interface{}{
-		"comeintime": map[string]interface{}{
-			"$gte":  gte_time,
-			"$lt": lt_time,
-		},
-	}
-	sess := jp_mgo.GetMgoConn()
-	defer jp_mgo.DestoryMongoConn(sess)
-	it := sess.DB(jp_mgo.DbName).C(jp_c_name).Find(&q).Select(map[string]interface{}{
-		"site":1,
-	}).Iter()
-	total,isOK :=0, 0
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if qu.ObjToString(tmp["site"]) == "中国招标与采购网" {
-			isOK++
-		}
-		tmp = make(map[string]interface{})
-	}
-	fmt.Println(qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),"总量：",total,"竞品：",isOK)
-	comeintime:=qu.Int64All(time.Now().Unix())
-	date := qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT)
-	data_mgo.Save("monitor_other", map[string]interface{}{
-		"name":"竞品",
-		"num":qu.IntAll(isOK),
-		"comeintime":comeintime,
-		"date":date,
-	})
-}

data_monitoring/listen_task/src/dataTaskQY.go

@@ -2,19 +2,16 @@ package main
 
 import (
 	"fmt"
-	"log"
 	qu "qfw/util"
 	"time"
 )
 
 
-func dealWithQYData()  {
+func dealWithQYData()(string,string)  {
 
-	log.Println("开始统计企业数据...")
 
 	defer qu.Catch()
 
-	start := int(time.Now().Unix())
 	now:=time.Now() //当前天
 	gte_time:= time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local).Unix()
 	lt_time := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
@@ -25,32 +22,18 @@ func dealWithQYData()  {
 			"$lt": lt_time,
 		},
 	}
-
-	sess := data_mgo.GetMgoConn()
-	defer data_mgo.DestoryMongoConn(sess)
+	sess := qy_mgo.GetMgoConn()
+	defer qy_mgo.DestoryMongoConn(sess)
 
 	//细节才需要遍历
-	it := sess.DB(data_mgo.DbName).C(qy_c_name).Find(&q).Select(map[string]interface{}{
+	it := sess.DB(qy_mgo.DbName).C(qy_c_name).Find(&q).Select(map[string]interface{}{
 		"site":1,
 	}).Iter()
-	total,_ :=0, int(time.Now().Unix())
+	total:=0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
 		tmp = make(map[string]interface{})
 	}
-
-	//是否告警条件
-	if total<1 {
-		sendErrMailApi("企业数据异常","数量无")
-	}
-
-	data_mgo.Save("monitor_other", map[string]interface{}{
-		"name":"企业变更",
-		"num":qu.IntAll(total),
-		"comeintime":qu.Int64All(time.Now().Unix()),
-		"date":qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),
-	})
-
-	log.Println("企业-定时处理完毕...",int(time.Now().Unix())-start,"秒")
+	return qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),fmt.Sprintf("%d",total)
 }
 
 
@@ -72,31 +55,3 @@ func dealWithQYData()  {
 
 
 
-
-//测试分析
-func dealWithQYTest(year int ,month time.Month, day int)  {
-	gte_time:= time.Date(year, month, day, 0, 0, 0, 0, time.Local).Unix()
-	lt_time := time.Date(year, month, day+1, 0, 0, 0, 0, time.Local).Unix()
-	q := map[string]interface{}{
-		"down_time": map[string]interface{}{
-			"$gte":  gte_time,
-			"$lt": lt_time,
-		},
-	}
-	sess := data_mgo.GetMgoConn()
-	defer data_mgo.DestoryMongoConn(sess)
-	it := sess.DB(data_mgo.DbName).C(qy_c_name).Find(&q).Select(map[string]interface{}{
-		"site":1,
-	}).Iter()
-	total :=0
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		tmp = make(map[string]interface{})
-	}
-	fmt.Println(qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),"总量：",total)
-	data_mgo.Save("monitor_other", map[string]interface{}{
-		"name":"企业变更",
-		"num":qu.IntAll(total),
-		"comeintime":qu.Int64All(time.Now().Unix()),
-		"date":qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),
-	})
-}

data_monitoring/listen_task/src/dataTaskST.go

@@ -24,15 +24,17 @@ func dealWithSTData()  {
 	gte_time = 1618070400
 	lt_time = 1618156800
 
+
 	mgodata := toCalculateMgoData(gte_time,lt_time)
 	esdata := toCalculateEsData(fmt.Sprintf("%d",gte_time),fmt.Sprintf("%d",lt_time))
 	mgonum,esnum:= qu.IntAll(mgodata["totalnum"]),qu.IntAll(esdata["totalnum"])
 
 	comeintime:=qu.Int64All(time.Now().Unix())
 	date := qu.FormatDateByInt64(&comeintime, qu.DATEFORMAT)
-	data_mgo.Save("monitor_site", map[string]interface{}{
+	save_mgo.Save("monitor_site", map[string]interface{}{
 		"comeintime":comeintime,
 		"date":date,
+		"name":"es-mgo-站点",
 		"mgonum":mgonum,
 		"esnum":esnum,
 		"mgodata":mgodata["detail"],
@@ -64,7 +66,7 @@ func toCalculateEsData(gte_time string,lt_time string) map[string]interface{} {
 	}
 
 	dict:= make(map[string]interface{},0)
-	total,isOK :=0,0
+	total :=0
 
 	//elastic.InitElasticSize("http://172.17.145.170:9800", 10,)
 	elastic.InitElasticSize("http://127.0.0.1:12003", 10,)
@@ -106,7 +108,6 @@ func toCalculateEsData(gte_time string,lt_time string) map[string]interface{} {
 			site := qu.ObjToString(tmp["site"])	//统计站点
 			if dict[site] == nil {
 				dict[site] = 1
-				isOK++
 			}else {
 				num := qu.IntAll(dict[site])+1
 				dict[site] = num
@@ -114,7 +115,6 @@ func toCalculateEsData(gte_time string,lt_time string) map[string]interface{} {
 		}
 	}
 
-	//log.Println("st is es over:",total,isOK,"有效：",len(dict),"用时：",int(time.Now().Unix())-start,"秒")
 
 	return map[string]interface{}{
 		"totalnum" : total,
@@ -150,13 +150,12 @@ func toCalculateMgoData(gte_time int64,lt_time int64) map[string]interface{} {
 	it := sess.DB(st_mgo.DbName).C(st_c_name).Find(&query).Select(map[string]interface{}{
 		"site":1,
 	}).Iter()
-	total,isOK :=0,0
+	total:=0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
 		//统计站点
 		site := qu.ObjToString(tmp["site"])
 		if dict[site] == nil {
 			dict[site] = 1
-			isOK++
 		}else {
 			num := qu.IntAll(dict[site])+1
 			dict[site] = num

data_monitoring/listen_task/src/main.go

@@ -1,39 +1,51 @@
 package main
 
 import (
+	"fmt"
 	"github.com/cron"
+	"github.com/tealeg/xlsx"
 	"log"
+	"os"
 	qu "qfw/util"
 	"time"
-
 )
 
 
 var (
 	sysconfig    							map[string]interface{} //配置文件
-	data_mgo,jp_mgo,st_mgo        			*MongodbSim            //mongodb操作对象
+	save_mgo,qy_mgo,jp_mgo,st_mgo        			*MongodbSim            //mongodb操作对象
 	qy_c_name,jp_c_name,st_c_name			string
 	es_index_st,es_type_st					string
 	save_other_name,save_site_name			string
+	xlsx_name								string
 )
 
 func initMgo()  {
 
-	dconf := sysconfig["data_mgodb"].(map[string]interface{})
-	qy_c_name = qu.ObjToString(dconf["coll"])
-	data_mgo = &MongodbSim{
-		MongodbAddr: dconf["addr"].(string),
-		DbName:      dconf["db"].(string),
-		Size:        qu.IntAllDef(dconf["pool"], 10),
+	saveconf := sysconfig["save_mgodb"].(map[string]interface{})
+	qy_c_name = qu.ObjToString(saveconf["coll"])
+	save_mgo = &MongodbSim{
+		MongodbAddr: saveconf["addr"].(string),
+		DbName:      saveconf["db"].(string),
+		Size:        qu.IntAllDef(saveconf["pool"], 5),
+	}
+	save_mgo.InitPool()
+
+	qconf := sysconfig["qy_mgodb"].(map[string]interface{})
+	qy_c_name = qu.ObjToString(qconf["coll"])
+	qy_mgo = &MongodbSim{
+		MongodbAddr: qconf["addr"].(string),
+		DbName:      qconf["db"].(string),
+		Size:        qu.IntAllDef(qconf["pool"], 5),
 	}
-	data_mgo.InitPool()
+	qy_mgo.InitPool()
 
 	jconf := sysconfig["jp_mgodb"].(map[string]interface{})
 	jp_c_name = qu.ObjToString(jconf["coll"])
 	jp_mgo = &MongodbSim{
 		MongodbAddr: jconf["addr"].(string),
 		DbName:      jconf["db"].(string),
-		Size:        qu.IntAllDef(jconf["pool"], 10),
+		Size:        qu.IntAllDef(jconf["pool"], 5),
 	}
 	jp_mgo.InitPool()
 
@@ -42,7 +54,7 @@ func initMgo()  {
 	st_mgo = &MongodbSim{
 		MongodbAddr: sconf["addr"].(string),
 		DbName:      sconf["db"].(string),
-		Size:        qu.IntAllDef(sconf["pool"], 10),
+		Size:        qu.IntAllDef(sconf["pool"], 5),
 	}
 	st_mgo.InitPool()
 
@@ -54,7 +66,7 @@ func initMgo()  {
 	save_site_name = qu.ObjToString(sysconfig["save_site_name"])
 
 
-
+	xlsx_name = qu.ObjToString(sysconfig["xlsx_name"])
 }
 
 
@@ -74,27 +86,138 @@ func mainT() {
 //测试使用
 func main() {
 
-	//dealWithJPData()
-	//dealWithQYData()
-	dealWithSTData()
-
+	dealWithOtherData()
 }
 
 func taskTime()  {
 
+
+
 	log.Println("部署定时任务")
 	c := cron.New()
-	//竞品-mongo
-	c.AddFunc("0 30 9 ? * *", func() { dealWithJPData() })
-
-	//企业变更-mongo
-	c.AddFunc("0 0 9 ? * *", func() { dealWithQYData() })
 
-	//站点相关-es-mongo
+	//站点相关全量-es-mongo
 	c.AddFunc("0 30 8 ? * *", func() { dealWithSTData() })
 
+
+	//企业变更-站点-mongo
+	c.AddFunc("0 0 9 ? * *", func() { dealWithOtherData() })
+
 	c.Start()
 
 }
 
+func dealWithOtherData()  {
+
+	log.Println("开始统计相关数据...")
+
+	start := int(time.Now().Unix())
+	data :=dealWithJPData()//站点数据
+	qy_date,qy_total := dealWithQYData()//企业数据
+
+	comeintime :=qu.Int64All(time.Now().Unix())
+	save_mgo.Save(save_other_name, map[string]interface{}{
+		"name":"企业",
+		"num":qu.IntAll(qy_total),
+		"comeintime":comeintime,
+		"date":qy_date,
+	})
+
+	save_mgo.Save(save_other_name, map[string]interface{}{
+		"name":"站点",
+		"num":qu.IntAll(data["total"]),
+		"comeintime":comeintime,
+		"date":qu.ObjToString(data["date"]),
+		"data":data["data"],
+	})
+
+
+	//获取前两天数据
+	qy_arr ,st_arr:= make([]map[string]string,0),make([]map[string]interface{},0)
+	now:=time.Now() //当前天
+	curtime:= time.Date(now.Year(), now.Month(), now.Day()-2, 0, 0, 0, 0, time.Local).Unix()
+	before_date :=qu.FormatDateByInt64(&curtime, qu.DATEFORMAT)
+	//企业数据
+	before_qy_data :=save_mgo.FindOne(save_other_name, map[string]interface{}{
+		"name":"企业",
+		"date":before_date,
+	})
+	if before_qy_data!=nil && len(before_qy_data)>2 {
+		qy_arr = append(qy_arr, map[string]string{
+			"date":qu.ObjToString(before_qy_data["date"]),
+			"num":fmt.Sprintf("%d",before_qy_data["num"]),
+		})
+	}
+	qy_arr = append(qy_arr, map[string]string{
+		"date":qy_date,
+		"num":qy_total,
+	})
+
+
+
+	//站点数据
+	before_st_data :=save_mgo.FindOne(save_other_name, map[string]interface{}{
+		"name":"站点",
+		"date":before_date,
+	})
+	if before_st_data!=nil && len(before_st_data)>2 {
+		st_arr = append(st_arr, map[string]interface{}{
+			"date":qu.ObjToString(before_st_data["date"]),
+			"total":fmt.Sprintf("%d",before_st_data["num"]),
+			"data":before_st_data["data"],
+		})
+	}
+
+	st_arr = append(st_arr, map[string]interface{}{
+		"date":qu.ObjToString(data["date"]),
+		"total":fmt.Sprintf("%d",data["total"]),
+		"data":data["data"],
+	})
+
+	//写excle
+	os.Remove("统计.xlsx")
+	f :=xlsx.NewFile()
+	sheet, _ := f.AddSheet("统计")
+
+	//第一行先写标题
+	row := sheet.AddRow()
+	row.AddCell().Value = "企业分类/日期"
+	row.AddCell().Value = "总数量"
+	for _,tmp := range qy_arr {
+		row = sheet.AddRow()
+		row.AddCell().SetString(tmp["date"])
+		row.AddCell().SetString(tmp["num"])
+	}
+	sheet.AddRow()
+	row = sheet.AddRow()
+	row.AddCell().Value = "站点分类/日期"
+	row.AddCell().Value = "总数量"
+	row.AddCell().Value = "站点名称"
+	row.AddCell().Value = "站点数量"
+	for _, v:= range st_arr {
+		tmp := *qu.ObjToMap(v)
+		row = sheet.AddRow()
+		row.AddCell().SetString(qu.ObjToString(tmp["date"]))
+		row.AddCell().SetString(qu.ObjToString(tmp["total"]))
+
+		data := *qu.ObjToMap(tmp["data"])
+		if data!=nil {
+			for k,v := range data {
+				row.AddCell().SetString(k)
+				row.AddCell().SetString(fmt.Sprintf("%d",v))
+				row = sheet.AddRow()
+				row.AddCell().Value = ""
+				row.AddCell().Value = ""
+			}
+		}
+	}
+
+	err := f.Save(xlsx_name)
+	if err != nil {
+		log.Println("保存xlsx失败:", err)
+	}
+
+	sendErrMailSmtp("统计-站点-企业","详情请查阅附件")
 
+	log.Println("定时处理完毕...",int(time.Now().Unix())-start,"秒")
+}

data_monitoring/listen_task/src/sendmail.go

@@ -3,28 +3,40 @@ package main
 import (
 	"fmt"
 	"io/ioutil"
-	"log"
-	"net/http"
+	"os"
+	qu "qfw/util"
+	"qfw/util/mail"
 )
 
+var from,to, smtpHost,user,pwd string
+var smtpPort int
 
-var tomail string
-var api string
 
-//api模式
-func sendErrMailApi(title,body string)  {
-	jkmail, _ := sysconfig["jkmail"].(map[string]interface{})
-	if jkmail != nil {
-		tomail, _ = jkmail["to"].(string)
-		api, _ = jkmail["api"].(string)
+func sendErrMailSmtp(title,body string) {
+
+
+	smtpMail, _ := sysconfig["smtpMail"].(map[string]interface{})
+
+	if smtpMail != nil {
+		from, _ = smtpMail["from"].(string)
+		to, _ = smtpMail["to"].(string)
+		smtpHost, _ = smtpMail["smtpHost"].(string)
+		smtpPort= qu.IntAll(smtpMail["smtpPort"])
+		user, _ = smtpMail["user"].(string)
+		pwd, _ = smtpMail["pwd"].(string)
 	}
-	log.Println(tomail,api)
-	res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, tomail, title, body))
-	if err == nil {
-		defer res.Body.Close()
-		read, err := ioutil.ReadAll(res.Body)
-		log.Println("邮件发送成功：", string(read), err)
-	}else {
-		log.Println("邮件发送失败：", err)
+	f, _ := os.Open(xlsx_name)
+	b, err := ioutil.ReadAll(f)
+	if err != nil {
+		fmt.Println("err:",err)
+		return
 	}
+
+	ok := mail.GSendMail_B(user, from, from, from, title, body, f.Name(), b, &mail.GmailAuth{
+		SmtpHost: smtpHost,
+		SmtpPort: smtpPort,
+		User:     user,
+		Pwd:      pwd,
+	})
+	fmt.Println(ok)
 }