备份监测结构

data_monitoring/listen_task/src/config.json

@@ -6,8 +6,8 @@
     "pool": 5
   },
   "jp_mgodb": {
-    "addr": "192.168.3.207:27092",
-    "db": "zhengkun",
+    "addr": "baibai.ink:28082",
+    "db": "spider_data",
     "coll": "data_bak",
     "pool": 5
   },
@@ -19,8 +19,8 @@
   },
   "es_index_st": "bidding",
   "es_type_st": "bidding",
-  "save_other_name": "",
-  "save_site_name" : "",
+  "save_other_name": "monitor_other",
+  "save_site_name" : "monitor_site",
   "jkmail": {
     "to": "zhengkun@topnet.net.cn",
     "api": "http://172.17.145.179:19281/_send/_mail"

data_monitoring/listen_task/src/dataTaskJP.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"log"
 	qu "qfw/util"
 	"time"
@@ -11,36 +12,27 @@ func dealWithJPData()  {
 
 	log.Println("开始统计竞品数据...")
 	defer qu.Catch()
-
 	start := int(time.Now().Unix())
 	now:=time.Now() //当前天
 	gte_time:= time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local).Unix()
 	lt_time := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
 
-	//临时测试-7号
-	gte_time = 1617724800
-	lt_time = 1617811200
-
 	q := map[string]interface{}{
 		"comeintime": map[string]interface{}{
 			"$gte":  gte_time,
 			"$lt": lt_time,
 		},
-		"site":"中国招标与采购网",
 	}
-
-
 	sess := jp_mgo.GetMgoConn()
 	defer jp_mgo.DestoryMongoConn(sess)
-
 	//细节才需要遍历
 	it := sess.DB(jp_mgo.DbName).C(jp_c_name).Find(&q).Select(map[string]interface{}{
 		"site":1,
 	}).Iter()
-	total,_ :=0, int(time.Now().Unix())
+	total,isOK,_ :=0, 0,int(time.Now().Unix())
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%10000==0 {
-			//log.Println("current index",total)
+		if qu.ObjToString(tmp["site"]) == "中国招标与采购网" {
+			isOK++
 		}
 		tmp = make(map[string]interface{})
 	}
@@ -50,14 +42,63 @@ func dealWithJPData()  {
 		//sendErrMailApi("竞品数据异常","数量无")
 	}
 
-	comeintime:=qu.Int64All(time.Now().Unix())
-	date := qu.FormatDateByInt64(&comeintime, qu.DATEFORMAT)
 	data_mgo.Save("monitor_other", map[string]interface{}{
 		"name":"竞品",
 		"num":qu.IntAll(total),
-		"comeintime":comeintime,
-		"updatedate":date,
+		"comeintime":qu.Int64All(time.Now().Unix()),
+		"date":qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),
 	})
 
 	log.Println("竞品-定时处理完毕...",int(time.Now().Unix())-start,"秒")
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+//测试分析
+func dealWithJPTest(year int,month time.Month, day int)  {
+	gte_time:= time.Date(year, month, day, 0, 0, 0, 0, time.Local).Unix()
+	lt_time := time.Date(year, month, day+1, 0, 0, 0, 0, time.Local).Unix()
+	q := map[string]interface{}{
+		"comeintime": map[string]interface{}{
+			"$gte":  gte_time,
+			"$lt": lt_time,
+		},
+	}
+	sess := jp_mgo.GetMgoConn()
+	defer jp_mgo.DestoryMongoConn(sess)
+	it := sess.DB(jp_mgo.DbName).C(jp_c_name).Find(&q).Select(map[string]interface{}{
+		"site":1,
+	}).Iter()
+	total,isOK :=0, 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if qu.ObjToString(tmp["site"]) == "中国招标与采购网" {
+			isOK++
+		}
+		tmp = make(map[string]interface{})
+	}
+	fmt.Println(qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),"总量：",total,"竞品：",isOK)
+	comeintime:=qu.Int64All(time.Now().Unix())
+	date := qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT)
+	data_mgo.Save("monitor_other", map[string]interface{}{
+		"name":"竞品",
+		"num":qu.IntAll(isOK),
+		"comeintime":comeintime,
+		"date":date,
+	})
 }

data_monitoring/listen_task/src/dataTaskQY.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"log"
 	qu "qfw/util"
 	"time"
@@ -10,6 +11,7 @@ import (
 func dealWithQYData()  {
 
 	log.Println("开始统计企业数据...")
+
 	defer qu.Catch()
 
 	start := int(time.Now().Unix())
@@ -17,10 +19,6 @@ func dealWithQYData()  {
 	gte_time:= time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local).Unix()
 	lt_time := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
 
-	//临时测试-8号
-	gte_time = 1617811200
-	lt_time = 1617897600
-
 	q := map[string]interface{}{
 		"down_time": map[string]interface{}{
 			"$gte":  gte_time,
@@ -37,29 +35,68 @@ func dealWithQYData()  {
 	}).Iter()
 	total,_ :=0, int(time.Now().Unix())
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%10000==0 {
-			//log.Println("current index",total)
-		}
-		//删选条件
-
 		tmp = make(map[string]interface{})
 	}
 
-
 	//是否告警条件
 	if total<1 {
 		sendErrMailApi("企业数据异常","数量无")
 	}
 
-
-	comeintime:=qu.Int64All(time.Now().Unix())
-	date := qu.FormatDateByInt64(&comeintime, qu.DATEFORMAT)
 	data_mgo.Save("monitor_other", map[string]interface{}{
 		"name":"企业变更",
 		"num":qu.IntAll(total),
-		"comeintime":comeintime,
-		"updatedate":date,
+		"comeintime":qu.Int64All(time.Now().Unix()),
+		"date":qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),
 	})
 
 	log.Println("企业-定时处理完毕...",int(time.Now().Unix())-start,"秒")
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+//测试分析
+func dealWithQYTest(year int ,month time.Month, day int)  {
+	gte_time:= time.Date(year, month, day, 0, 0, 0, 0, time.Local).Unix()
+	lt_time := time.Date(year, month, day+1, 0, 0, 0, 0, time.Local).Unix()
+	q := map[string]interface{}{
+		"down_time": map[string]interface{}{
+			"$gte":  gte_time,
+			"$lt": lt_time,
+		},
+	}
+	sess := data_mgo.GetMgoConn()
+	defer data_mgo.DestoryMongoConn(sess)
+	it := sess.DB(data_mgo.DbName).C(qy_c_name).Find(&q).Select(map[string]interface{}{
+		"site":1,
+	}).Iter()
+	total :=0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		tmp = make(map[string]interface{})
+	}
+	fmt.Println(qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),"总量：",total)
+	data_mgo.Save("monitor_other", map[string]interface{}{
+		"name":"企业变更",
+		"num":qu.IntAll(total),
+		"comeintime":qu.Int64All(time.Now().Unix()),
+		"date":qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),
+	})
 }

data_monitoring/listen_task/src/dataTaskST.go

@@ -21,23 +21,18 @@ func dealWithSTData()  {
 	gte_time:= time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, time.Local).Unix()
 	lt_time := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
 
-	//临时测试-8号
 	gte_time = 1618070400
 	lt_time = 1618156800
+
 	mgodata := toCalculateMgoData(gte_time,lt_time)
 	esdata := toCalculateEsData(fmt.Sprintf("%d",gte_time),fmt.Sprintf("%d",lt_time))
-
 	mgonum,esnum:= qu.IntAll(mgodata["totalnum"]),qu.IntAll(esdata["totalnum"])
 
-	//是否发警告-待定
-
-
-
 	comeintime:=qu.Int64All(time.Now().Unix())
 	date := qu.FormatDateByInt64(&comeintime, qu.DATEFORMAT)
 	data_mgo.Save("monitor_site", map[string]interface{}{
 		"comeintime":comeintime,
-		"updatedate":date,
+		"date":date,
 		"mgonum":mgonum,
 		"esnum":esnum,
 		"mgodata":mgodata["detail"],
@@ -48,6 +43,17 @@ func dealWithSTData()  {
 }
 
 
+
+
+
+
+
+
+
+
+
+
+//es数据统计
 func toCalculateEsData(gte_time string,lt_time string) map[string]interface{} {
 
 	if gte_time == "" || lt_time == "" {
@@ -110,13 +116,16 @@ func toCalculateEsData(gte_time string,lt_time string) map[string]interface{} {
 
 	//log.Println("st is es over:",total,isOK,"有效：",len(dict),"用时：",int(time.Now().Unix())-start,"秒")
 
-
 	return map[string]interface{}{
 		"totalnum" : total,
 		"detail": dict,
 	}
 }
 
+
+
+
+//mongo数据统计
 func toCalculateMgoData(gte_time int64,lt_time int64) map[string]interface{} {
 
 	if gte_time == 0 || lt_time == 0 {
@@ -143,9 +152,6 @@ func toCalculateMgoData(gte_time int64,lt_time int64) map[string]interface{} {
 	}).Iter()
 	total,isOK :=0,0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%10000==0 {
-			//log.Println("current index",total,isOK)
-		}
 		//统计站点
 		site := qu.ObjToString(tmp["site"])
 		if dict[site] == nil {

data_monitoring/listen_task/src/main.go

@@ -47,8 +47,6 @@ func initMgo()  {
 	st_mgo.InitPool()
 
 
-
-
 	//属性赋值
 	es_index_st = qu.ObjToString(sysconfig["es_index_st"])
 	es_type_st = qu.ObjToString(sysconfig["es_type_st"])
@@ -56,8 +54,6 @@ func initMgo()  {
 	save_site_name = qu.ObjToString(sysconfig["save_site_name"])
 
 
-	//log.Println(dconf,jconf,sconf)
-	//log.Println(qy_c_name,jp_c_name,st_c_name,es_index_st,es_type_st,save_other_name,save_site_name)
 
 }
 
@@ -78,8 +74,8 @@ func mainT() {
 //测试使用
 func main() {
 
-	dealWithJPData()
-	dealWithQYData()
+	//dealWithJPData()
+	//dealWithQYData()
 	dealWithSTData()
 
 }