指定站点-监听-上海通服

data_monitoring/get_assign/src/config.json

@@ -1,8 +1,17 @@
 {
-  "python_mgodb": {
-    "addr": "172.17.4.187:27082,172.17.145.163:27083",
+  "save_mgodb": {
+    "addr": "172.17.145.163:27083,172.17.4.187:27082",
     "db": "qfw",
     "coll": "bidding",
     "pool": 10
+  },
+  "smtpMail": {
+    "from": "zhengkun@topnet.net.cn",
+    "to": "zhaoxiuzhen@topnet.net.cn",
+    "cc": "zhangjinkun@topnet.net.cn",
+    "smtpHost": "smtp.exmail.qq.com",
+    "smtpPort": "465",
+    "user":     "zhengkun@topnet.net.cn",
+    "pwd":      "Zheng123456"
   }
-}
+}

data_monitoring/get_assign/src/main.go

@@ -1,9 +1,13 @@
 package main
 
 import (
-
+	"fmt"
+	"github.com/cron"
+	"github.com/tealeg/xlsx"
+	"log"
+	"os"
 	qu "qfw/util"
-
+	"time"
 )
 
 var (
@@ -11,6 +15,8 @@ var (
 	save_mgo        			*MongodbSim            //mongodb操作对象
 	save_c_name					string
 )
+var Url = "https://www.jianyu360.com/article/content/%s.html"
+
 func initMgo()  {
 	saveconf := sysconfig["save_mgodb"].(map[string]interface{})
 	save_c_name = qu.ObjToString(saveconf["coll"])
@@ -18,6 +24,8 @@ func initMgo()  {
 		MongodbAddr: saveconf["addr"].(string),
 		DbName:      saveconf["db"].(string),
 		Size:        qu.IntAllDef(saveconf["pool"], 5),
+		Password: 	 "zk@123123",
+		UserName:    "zhengkun",
 	}
 	save_mgo.InitPool()
 }
@@ -29,5 +37,240 @@ func init() {
 }
 
 func main()  {
+	//定时任务
+	c := cron.New()
+	c.AddFunc("0 30 7 ? * *", func() {
+		exportSpecSiteDataWeek() //周邮件-每周1点    8点
+		exportSpecSiteDataMonth() //月邮件-每月1号   8点
+	})
+	c.Start()
+
+	//立即处理一次
+	exportSpecSiteDataWeek()
+	exportSpecSiteDataMonth()
+
+
+	time.Sleep(99999*time.Hour)
+}
+
+//处理数据-周邮件
+func exportSpecSiteDataWeek() {
+	cur_time := time.Now().Unix()
+	today := GetOneWeekDay(TimeStampToString(cur_time))
+	if today!=1 {
+		return
+	}
+	log.Println("每周一：准备邮件数据...")
+	now:=time.Now()
+	durdays:=7*2
+	start:= time.Date(now.Year(), now.Month(), now.Day()-durdays, 0, 0, 0, 0, time.Local).Unix()
+	end := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
+
+	log.Println(start,end)
+	q := map[string]interface{}{
+		"comeintime": map[string]interface{}{
+			"$gte":  start,
+			"$lt": end,
+		},
+	}
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	log.Println("bidding 查询条件：",q)
+	//省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站
+	it_site := sess.DB(save_mgo.DbName).C(save_c_name).Find(&q).Sort("comeintime").Select(map[string]interface{}{
+		"area":1,
+		"title":1,
+		"subtype":1,
+		"publishtime":1,
+		"href":1,
+		"projectname":1,
+		"site":1,
+	}).Iter()
+	timeLayout := "2006-01-02"
+	total,isok,dataArr:= 0,0,make([]map[string]string,0)
+	for tmp := make(map[string]interface{}); it_site.Next(&tmp); total++ {
+		if total%10000 == 0 {
+			log.Println("cur index :", total,isok)
+		}
+		site:=qu.ObjToString(tmp["site"])
+		area:=qu.ObjToString(tmp["area"])
+		subtype:=qu.ObjToString(tmp["subtype"])
+		projectname:=qu.ObjToString(tmp["projectname"])
+		publishtime:=qu.Int64All(tmp["publishtime"])
+		new_publishtime := ""//转日期
+		if publishtime>0 {
+			new_publishtime = time.Unix(publishtime, 0).Format(timeLayout)
+		}
+		href:=qu.ObjToString(tmp["href"])
+		title:=qu.ObjToString(tmp["title"])
+
+		if (site=="上海政府采购网" || site=="中国政府采购网") && area=="上海" {
+			isok++
+			dataArr = append(dataArr, map[string]string{
+				"area":area,
+				"subtype":subtype,
+				"title":title,
+				"projectname":projectname,
+				"site":site,
+				"publishtime":new_publishtime,
+				"href":href,
+				"jyhref":fmt.Sprintf(Url, qu.CommonEncodeArticle("content", BsonTOStringId(tmp["_id"]))),
+			})
+		}
+
+		tmp = make(map[string]interface{})
+	}
+	log.Println("is site over :",total,isok)
+
+
+	start_str := time.Unix(start, 0).Format(timeLayout)
+	end_str := time.Unix(end-1, 0).Format(timeLayout)
+	xlsxName := "上海通服:"+start_str+"~"+end_str+".xlsx"
+	log.Println("邮件名：",xlsxName)
+	os.Remove(xlsxName)
+	f :=xlsx.NewFile()
+	sheet, _ := f.AddSheet("上海数据")
+	row := sheet.AddRow()
+	//省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站
+	row.AddCell().Value = "省份"
+	row.AddCell().Value = "公告类别"
+	row.AddCell().Value = "公告标题"
+	row.AddCell().Value = "项目名称"
+	row.AddCell().Value = "发布时间"
+	row.AddCell().Value = "来源网站"
+	row.AddCell().Value = "公告地址"
+	row.AddCell().Value = "剑鱼地址"
 
+	for _,tmp:=range dataArr {
+		row = sheet.AddRow()
+		row.AddCell().SetString(tmp["area"])
+		row.AddCell().SetString(tmp["subtype"])
+		row.AddCell().SetString(tmp["title"])
+		row.AddCell().SetString(tmp["projectname"])
+		row.AddCell().SetString(tmp["publishtime"])
+		row.AddCell().SetString(tmp["site"])
+		row.AddCell().SetString(tmp["href"])
+		row.AddCell().SetString(tmp["jyhref"])
+	}
+	err := f.Save(xlsxName)
+
+	if err != nil {
+		log.Println("保存xlsx失败:", err)
+		sendWarningSmtp("保存xlsx异常","请检查...上海通服...")
+	}else {
+		log.Println("保存xlsx成功:", err)
+		body_str := fmt.Sprintf("日期：%s~%s\t\t总计：%d条",start_str,end_str,isok)
+		sendErrMailSmtp("上海通服数据~周",body_str,xlsxName)
+	}
+}
+
+//处理数据-月邮件
+func exportSpecSiteDataMonth() {
+	cur_time := time.Now().Unix()
+	cur_day := time.Now().Day()
+	if cur_day!=1 {
+		return
+	}
+	today := GetOneWeekDay(TimeStampToString(cur_time))
+	log.Println("每月1号：准备邮件数据...")
+	now:=time.Now()
+	durdays:=int64(7)+today-int64(1)
+	start:= time.Date(now.Year(), now.Month(), now.Day()-int(durdays), 0, 0, 0, 0, time.Local).Unix()
+	end := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
+
+	log.Println(start,end)
+	q := map[string]interface{}{
+		"comeintime": map[string]interface{}{
+			"$gte":  start,
+			"$lt": end,
+		},
+	}
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	log.Println("bidding 查询条件：",q)
+	//省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站
+	it_site := sess.DB(save_mgo.DbName).C(save_c_name).Find(&q).Sort("comeintime").Select(map[string]interface{}{
+		"area":1,
+		"title":1,
+		"subtype":1,
+		"publishtime":1,
+		"href":1,
+		"projectname":1,
+		"site":1,
+	}).Iter()
+	timeLayout := "2006-01-02"
+	total,isok,dataArr:= 0,0,make([]map[string]string,0)
+	for tmp := make(map[string]interface{}); it_site.Next(&tmp); total++ {
+		if total%10000 == 0 {
+			log.Println("cur index :", total,isok)
+		}
+		site:=qu.ObjToString(tmp["site"])
+		area:=qu.ObjToString(tmp["area"])
+		subtype:=qu.ObjToString(tmp["subtype"])
+		projectname:=qu.ObjToString(tmp["projectname"])
+		publishtime:=qu.Int64All(tmp["publishtime"])
+		new_publishtime := ""//转日期
+		if publishtime>0 {
+			new_publishtime = time.Unix(publishtime, 0).Format(timeLayout)
+		}
+		href:=qu.ObjToString(tmp["href"])
+		title:=qu.ObjToString(tmp["title"])
+
+		if (site=="上海政府采购网" || site=="中国政府采购网") && area=="上海" {
+			isok++
+			dataArr = append(dataArr, map[string]string{
+				"area":area,
+				"subtype":subtype,
+				"title":title,
+				"projectname":projectname,
+				"site":site,
+				"publishtime":new_publishtime,
+				"href":href,
+				"jyhref":fmt.Sprintf(Url, qu.CommonEncodeArticle("content", BsonTOStringId(tmp["_id"]))),
+			})
+		}
+
+		tmp = make(map[string]interface{})
+	}
+	log.Println("is site over :",total,isok)
+
+
+	start_str := time.Unix(start, 0).Format(timeLayout)
+	end_str := time.Unix(end-1, 0).Format(timeLayout)
+	xlsxName := "上海通服:"+start_str+"~"+end_str+".xlsx"
+	log.Println("邮件名：",xlsxName)
+	os.Remove(xlsxName)
+	f :=xlsx.NewFile()
+	sheet, _ := f.AddSheet("上海数据")
+	row := sheet.AddRow()
+	//省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站
+	row.AddCell().Value = "省份"
+	row.AddCell().Value = "公告类别"
+	row.AddCell().Value = "公告标题"
+	row.AddCell().Value = "项目名称"
+	row.AddCell().Value = "发布时间"
+	row.AddCell().Value = "来源网站"
+	row.AddCell().Value = "公告地址"
+	row.AddCell().Value = "剑鱼地址"
+
+	for _,tmp:=range dataArr {
+		row = sheet.AddRow()
+		row.AddCell().SetString(tmp["area"])
+		row.AddCell().SetString(tmp["subtype"])
+		row.AddCell().SetString(tmp["title"])
+		row.AddCell().SetString(tmp["projectname"])
+		row.AddCell().SetString(tmp["publishtime"])
+		row.AddCell().SetString(tmp["site"])
+		row.AddCell().SetString(tmp["href"])
+		row.AddCell().SetString(tmp["jyhref"])
+	}
+	err := f.Save(xlsxName)
+	if err != nil {
+		log.Println("保存xlsx失败:", err)
+		sendWarningSmtp("保存xlsx异常","请检查...上海通服...")
+	}else {
+		log.Println("保存xlsx成功:", err)
+		body_str := fmt.Sprintf("日期：%s~%s\t\t总计：%d条",start_str,end_str,isok)
+		sendErrMailSmtp("上海通服数据~月）",body_str,xlsxName)
+	}
 }

data_monitoring/get_assign/src/mark

@@ -1,2 +1,45 @@
 获取指定数据
-上海政府采购网、中国政府采购网   上海   需求字段：省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站。
+上海政府采购网、中国政府采购网   上海
+需求字段：省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站。
+
+
+
+正式
+{
+  "save_mgodb": {
+    "addr": "172.17.145.163:27083,172.17.4.187:27082",
+    "db": "qfw",
+    "coll": "bidding",
+    "pool": 10
+  },
+  "smtpMail": {
+    "from": "zhengkun@topnet.net.cn",
+    "to": "zhaoxiuzhen@topnet.net.cn",
+    "cc": "zhangjinkun@topnet.net.cn",
+    "smtpHost": "smtp.exmail.qq.com",
+    "smtpPort": "465",
+    "user":     "zhengkun@topnet.net.cn",
+    "pwd":      "Zheng123456"
+  }
+}
+
+
+
+
+{
+  "save_mgodb": {
+    "addr": "127.0.0.1:27017",
+    "db": "zhengkun",
+    "coll": "extract_test",
+    "pool": 10
+  },
+  "smtpMail": {
+    "from": "zhengkun@topnet.net.cn",
+    "to": "zhengkun@topnet.net.cn",
+    "cc": "zhengkun@topnet.net.cn",
+    "smtpHost": "smtp.exmail.qq.com",
+    "smtpPort": "465",
+    "user":     "zhengkun@topnet.net.cn",
+    "pwd":      "Zheng123456"
+  }
+}

data_monitoring/get_assign/src/sendmain.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	qu "qfw/util"
+	"qfw/util/mail"
+)
+var tomail string
+var api string
+var from,to,cc, smtpHost,user,pwd string
+var smtpPort int
+//api模式 二选一皆可
+func sendErrMailApi(title,body string)  {
+	jkmail, _ := sysconfig["jkmail"].(map[string]interface{})
+	if jkmail != nil {
+		tomail, _ = jkmail["to"].(string)
+		api, _ = jkmail["api"].(string)
+	}
+	log.Println(tomail,api)
+	res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, tomail, title, body))
+	if err == nil {
+		defer res.Body.Close()
+		read, err := ioutil.ReadAll(res.Body)
+		log.Println("邮件发送成功：", string(read), err)
+	}else {
+		log.Println("邮件发送失败：", err)
+	}
+}
+
+func sendErrMailSmtp(title,body,xlsxName string) {
+
+	smtpMail, _ := sysconfig["smtpMail"].(map[string]interface{})
+	if smtpMail != nil {
+		from, _ = smtpMail["from"].(string)
+		to, _ = smtpMail["to"].(string)
+		cc, _ = smtpMail["cc"].(string)
+		smtpHost, _ = smtpMail["smtpHost"].(string)
+		smtpPort= qu.IntAll(smtpMail["smtpPort"])
+		user, _ = smtpMail["user"].(string)
+		pwd, _ = smtpMail["pwd"].(string)
+	}
+	f, _ := os.Open(xlsxName)
+	b, err := ioutil.ReadAll(f)
+	if err != nil {
+		fmt.Println("err:",err)
+		return
+	}
+
+
+	ok := mail.GSendMail_Bq(from, to, cc, cc, title, body, f.Name(), b, &mail.GmailAuth{
+		SmtpHost: smtpHost,
+		SmtpPort: smtpPort,
+		User:     user,
+		Pwd:      pwd,
+	})
+	fmt.Println(ok)
+
+	if !ok {//邮件发送异常
+		sendWarningSmtp("邮件附件发送异常","请检查......")
+	}
+}
+
+
+func sendWarningSmtp(title,body string) {
+
+	smtpMail, _ := sysconfig["smtpMail"].(map[string]interface{})
+	if smtpMail != nil {
+		from, _ = smtpMail["from"].(string)
+		to, _ = smtpMail["to"].(string)
+		cc, _ = smtpMail["cc"].(string)
+		smtpHost, _ = smtpMail["smtpHost"].(string)
+		smtpPort= qu.IntAll(smtpMail["smtpPort"])
+		user, _ = smtpMail["user"].(string)
+		pwd, _ = smtpMail["pwd"].(string)
+	}
+	ok := mail.GSendMail(from, from, from, from, title, body, "", "", &mail.GmailAuth{
+		SmtpHost: smtpHost,
+		SmtpPort: smtpPort,
+		User:     user,
+		Pwd:      pwd,
+	})
+	fmt.Println(ok)
+}

data_monitoring/get_assign/src/weekday.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"log"
+	"time"
+)
+
+var WeekDayMap = map[string]int64{
+	"Monday":    1,
+	"Tuesday":   2,
+	"Wednesday": 3,
+	"Thursday":  4,
+	"Friday":    5,
+	"Saturday":  6,
+	"Sunday":    7,
+}
+
+// 获取输入日期分别是星期几
+func GetOneWeekDay(startime string) (int64) {
+	startday, _ := time.Parse("2006-01-02", startime)
+	staweek_int := startday.Weekday().String()
+	return WeekDayMap[staweek_int]
+}
+
+
+// 获取输入的两个日期分别是星期几
+func GetWeekDay(startime, endtim string) (int64, int64) {
+	startday, _ := time.Parse("2006-01-02", startime)
+	endday, _ := time.Parse("2006-01-02", endtim)
+	staweek_int := startday.Weekday().String()
+	endweek_int := endday.Weekday().String()
+	return WeekDayMap[staweek_int], WeekDayMap[endweek_int]
+}
+
+
+// 字符串转时间戳
+func StringToTimeStamp(strTime string) int64 {
+	timeLayout := "2006-01-02"
+	//timeLayout := "2006-01-02 15:04:05"
+	loc, _ := time.LoadLocation("Local")
+	the_time, err := time.ParseInLocation(timeLayout, strTime, loc)
+	if err != nil {
+		log.Println("StringToTimeStamp出现异常：", err)
+	}
+	unix_time := the_time.Unix()
+	return unix_time
+}
+
+// 时间戳转 字符串
+func TimeStampToString(timeStp int64) string {
+	//转化所需模板
+	//timeLayout := "2006-01-02 15:04:05"
+	timeLayout := "2006-01-02"
+	//进行格式化
+	datetime := time.Unix(timeStp, 0).Format(timeLayout)
+	return datetime
+}
+
+
+// 时间转化为周日期列表
+func ChangeToWeek(startime, endtim string) []map[string]string {
+
+	staweek_int, endweek_int := GetWeekDay(startime, endtim)
+	// 获取时间戳
+	start_stamp := StringToTimeStamp(startime)
+	end_stamp := StringToTimeStamp(endtim)
+	log.Println("start_stamp==",start_stamp,"end_stamp==", end_stamp)
+
+	var week_list = make([]map[string]string, 0)
+	if (end_stamp-start_stamp)/604800 <= 1 && endweek_int-staweek_int >= 0 {
+		if end_stamp-start_stamp < 604800 && endweek_int-staweek_int > 0 {
+			one_map := map[string]string{}
+			mon_one := TimeStampToString(start_stamp - (staweek_int-1)*86400)
+			sun_one := TimeStampToString(start_stamp + (7-staweek_int)*86400)
+			one_map["mon"] = mon_one
+			one_map["sun"] = sun_one
+			week_list = append(week_list, one_map)
+			return week_list
+		}
+		one_map := map[string]string{}
+		mon_one := TimeStampToString(start_stamp - (staweek_int-1)*86400)
+		sun_one := TimeStampToString(start_stamp + (7-staweek_int)*86400)
+		one_map["mon"] = mon_one
+		one_map["sun"] = sun_one
+		week_list = append(week_list, one_map)
+		tow_map := map[string]string{}
+		mon_tow := TimeStampToString(end_stamp - (endweek_int-1)*86400)
+		sun_tow := TimeStampToString(end_stamp + (7-endweek_int)*86400)
+		tow_map["mon"] = mon_tow
+		tow_map["sun"] = sun_tow
+		week_list = append(week_list, tow_map)
+		return week_list
+	}
+	week_n := (end_stamp - start_stamp) / 604800
+	one_map := map[string]string{}
+	mon_one := TimeStampToString(start_stamp - (staweek_int-1)*86400)
+	sun_one := TimeStampToString(start_stamp + (7-staweek_int)*86400)
+	one_map["mon"] = mon_one
+	one_map["sun"] = sun_one
+	week_list = append(week_list, one_map)
+	for i := 1; i <= int(week_n); i++ {
+		week_map := map[string]string{}
+		mon_day := TimeStampToString(start_stamp - (staweek_int-1)*86400 + int64(i)*604800)
+		sun_day := TimeStampToString(start_stamp + (7-staweek_int)*86400 + int64(i)*604800)
+		week_map["mon"] = mon_day
+		week_map["sun"] = sun_day
+		week_list = append(week_list, week_map)
+	}
+	if endweek_int-staweek_int >= 0 {
+		return week_list
+	}
+	tow_map := map[string]string{}
+	mon_tow := TimeStampToString(end_stamp - (endweek_int-1)*86400)
+	sun_tow := TimeStampToString(end_stamp + (7-endweek_int)*86400)
+	tow_map["mon"] = mon_tow
+	tow_map["sun"] = sun_tow
+	week_list = append(week_list, tow_map)
+	return week_list
+
+}

udp_datacheck/src/main.go

@@ -160,8 +160,8 @@ func mainT()  {
 
 //临时校验
 func main()  {
-	sid := "618aef9745a326c6c3eb7e50"
-	eid := "618b3d6745a326c6c3ec6d4e"
+	sid := "618dc3b045a326c6c3f2f230"
+	eid := "618e137545a326c6c3f44195"
 	startCheckData(sid,eid)
 	time.Sleep(99999 * time.Hour)
 }

udps/main.go

@@ -22,13 +22,10 @@ func main() {
 	flag.IntVar(&p, "p", 6601, "端口")
 	flag.IntVar(&tmptime, "tmptime", 0, "时间查询")
 	flag.StringVar(&tmpkey, "tmpkey", "", "时间字段")
-<<<<<<< HEAD
-	flag.StringVar(&id1, "gtid", "", "gtid")
-	flag.StringVar(&id2, "lteid", "", "lteid")
-=======
+
 	flag.StringVar(&id1, "gtid", "114168ea1a75b8f44678a39b", "gtid")
 	flag.StringVar(&id2, "lteid", "9142e5741a75b8f4467b3276", "lteid")
->>>>>>> f5506419ade445c9ae7863e3d184d86b2b0ca84f
+
 	flag.StringVar(&ids, "ids", "", "id1,id2")
 	flag.StringVar(&stype, "stype", "biddingall", "stype,传递类型")
 	flag.StringVar(&bkey, "bkey", "", "bkey,加上此参数表示不生关键词和摘要")