jy-data-extract/data_monitoring/get_assign/src/main.go

package main

import (
	"fmt"
	"github.com/cron"
	"github.com/tealeg/xlsx"
	"log"
	"os"
	qu "qfw/util"
	"time"
)

var (
	sysconfig    				map[string]interface{} //配置文件
	save_mgo        			*MongodbSim            //mongodb操作对象
	save_c_name					string
)
var Url = "https://www.jianyu360.com/article/content/%s.html"

func initMgo()  {
	saveconf := sysconfig["save_mgodb"].(map[string]interface{})
	save_c_name = qu.ObjToString(saveconf["coll"])
	save_mgo = &MongodbSim{
		MongodbAddr: saveconf["addr"].(string),
		DbName:      saveconf["db"].(string),
		Size:        qu.IntAllDef(saveconf["pool"], 5),
		Password: 	 "zk@123123",
		UserName:    "zhengkun",
	}
	save_mgo.InitPool()
}

func init() {
	//加载配置文件
	qu.ReadConfig(&sysconfig)
	initMgo()
}

func main()  {
	//定时任务
	c := cron.New()
	c.AddFunc("0 30 7 ? * *", func() {
		exportSpecSiteDataWeek() //周邮件-每周1点    8点
		exportSpecSiteDataMonth() //月邮件-每月1号   8点
	})
	c.Start()

	//立即处理一次
	exportSpecSiteDataWeek()
	exportSpecSiteDataMonth()


	time.Sleep(99999*time.Hour)
}

//处理数据-周邮件
func exportSpecSiteDataWeek() {
	cur_time := time.Now().Unix()
	today := GetOneWeekDay(TimeStampToString(cur_time))
	if today!=1 {
		return
	}
	log.Println("每周一：准备邮件数据...")
	now:=time.Now()
	durdays:=7*2
	start:= time.Date(now.Year(), now.Month(), now.Day()-durdays, 0, 0, 0, 0, time.Local).Unix()
	end := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()

	log.Println(start,end)
	q := map[string]interface{}{
		"comeintime": map[string]interface{}{
			"$gte":  start,
			"$lt": end,
		},
	}
	sess := save_mgo.GetMgoConn()
	defer save_mgo.DestoryMongoConn(sess)
	log.Println("bidding 查询条件：",q)
	//省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站
	it_site := sess.DB(save_mgo.DbName).C(save_c_name).Find(&q).Sort("comeintime").Select(map[string]interface{}{
		"area":1,
		"title":1,
		"subtype":1,
		"publishtime":1,
		"href":1,
		"projectname":1,
		"site":1,
	}).Iter()
	timeLayout := "2006-01-02"
	total,isok,dataArr:= 0,0,make([]map[string]string,0)
	for tmp := make(map[string]interface{}); it_site.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Println("cur index :", total,isok)
		}
		site:=qu.ObjToString(tmp["site"])
		area:=qu.ObjToString(tmp["area"])
		subtype:=qu.ObjToString(tmp["subtype"])
		projectname:=qu.ObjToString(tmp["projectname"])
		publishtime:=qu.Int64All(tmp["publishtime"])
		new_publishtime := ""//转日期
		if publishtime>0 {
			new_publishtime = time.Unix(publishtime, 0).Format(timeLayout)
		}
		href:=qu.ObjToString(tmp["href"])
		title:=qu.ObjToString(tmp["title"])

		if (site=="上海政府采购网" || site=="中国政府采购网") && area=="上海" {
			isok++
			dataArr = append(dataArr, map[string]string{
				"area":area,
				"subtype":subtype,
				"title":title,
				"projectname":projectname,
				"site":site,
				"publishtime":new_publishtime,
				"href":href,
				"jyhref":fmt.Sprintf(Url, qu.CommonEncodeArticle("content", BsonTOStringId(tmp["_id"]))),
			})
		}

		tmp = make(map[string]interface{})
	}
	log.Println("is site over :",total,isok)


	start_str := time.Unix(start, 0).Format(timeLayout)
	end_str := time.Unix(end-1, 0).Format(timeLayout)
	xlsxName := "上海通服:"+start_str+"~"+end_str+".xlsx"
	log.Println("邮件名：",xlsxName)
	os.Remove(xlsxName)
	f :=xlsx.NewFile()
	sheet, _ := f.AddSheet("上海数据")
	row := sheet.AddRow()
	//省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站
	row.AddCell().Value = "省份"
	row.AddCell().Value = "公告类别"
	row.AddCell().Value = "公告标题"
	row.AddCell().Value = "项目名称"
	row.AddCell().Value = "发布时间"
	row.AddCell().Value = "来源网站"
	row.AddCell().Value = "公告地址"
	row.AddCell().Value = "剑鱼地址"

	for _,tmp:=range dataArr {
		row = sheet.AddRow()
		row.AddCell().SetString(tmp["area"])
		row.AddCell().SetString(tmp["subtype"])
		row.AddCell().SetString(tmp["title"])
		row.AddCell().SetString(tmp["projectname"])
		row.AddCell().SetString(tmp["publishtime"])
		row.AddCell().SetString(tmp["site"])
		row.AddCell().SetString(tmp["href"])
		row.AddCell().SetString(tmp["jyhref"])
	}
	err := f.Save(xlsxName)

	if err != nil {
		log.Println("保存xlsx失败:", err)
		sendWarningSmtp("保存xlsx异常","请检查...上海通服...")
	}else {
		log.Println("保存xlsx成功:", err)
		body_str := fmt.Sprintf("日期：%s~%s\t\t总计：%d条",start_str,end_str,isok)
		sendErrMailSmtp("上海通服数据~周",body_str,xlsxName)
	}
}

//处理数据-月邮件
func exportSpecSiteDataMonth() {
	cur_time := time.Now().Unix()
	cur_day := time.Now().Day()
	if cur_day!=1 {
		return
	}
	today := GetOneWeekDay(TimeStampToString(cur_time))
	log.Println("每月1号：准备邮件数据...")
	now:=time.Now()
	durdays:=int64(7)+today-int64(1)
	start:= time.Date(now.Year(), now.Month(), now.Day()-int(durdays), 0, 0, 0, 0, time.Local).Unix()
	end := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()

	log.Println(start,end)
	q := map[string]interface{}{
		"comeintime": map[string]interface{}{
			"$gte":  start,
			"$lt": end,
		},
	}
	sess := save_mgo.GetMgoConn()
	defer save_mgo.DestoryMongoConn(sess)
	log.Println("bidding 查询条件：",q)
	//省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站
	it_site := sess.DB(save_mgo.DbName).C(save_c_name).Find(&q).Sort("comeintime").Select(map[string]interface{}{
		"area":1,
		"title":1,
		"subtype":1,
		"publishtime":1,
		"href":1,
		"projectname":1,
		"site":1,
	}).Iter()
	timeLayout := "2006-01-02"
	total,isok,dataArr:= 0,0,make([]map[string]string,0)
	for tmp := make(map[string]interface{}); it_site.Next(&tmp); total++ {
		if total%10000 == 0 {
			log.Println("cur index :", total,isok)
		}
		site:=qu.ObjToString(tmp["site"])
		area:=qu.ObjToString(tmp["area"])
		subtype:=qu.ObjToString(tmp["subtype"])
		projectname:=qu.ObjToString(tmp["projectname"])
		publishtime:=qu.Int64All(tmp["publishtime"])
		new_publishtime := ""//转日期
		if publishtime>0 {
			new_publishtime = time.Unix(publishtime, 0).Format(timeLayout)
		}
		href:=qu.ObjToString(tmp["href"])
		title:=qu.ObjToString(tmp["title"])

		if (site=="上海政府采购网" || site=="中国政府采购网") && area=="上海" {
			isok++
			dataArr = append(dataArr, map[string]string{
				"area":area,
				"subtype":subtype,
				"title":title,
				"projectname":projectname,
				"site":site,
				"publishtime":new_publishtime,
				"href":href,
				"jyhref":fmt.Sprintf(Url, qu.CommonEncodeArticle("content", BsonTOStringId(tmp["_id"]))),
			})
		}

		tmp = make(map[string]interface{})
	}
	log.Println("is site over :",total,isok)


	start_str := time.Unix(start, 0).Format(timeLayout)
	end_str := time.Unix(end-1, 0).Format(timeLayout)
	xlsxName := "上海通服："+start_str+"~"+end_str+".xlsx"
	log.Println("邮件名：",xlsxName)
	os.Remove(xlsxName)
	f :=xlsx.NewFile()
	sheet, _ := f.AddSheet("上海数据")
	row := sheet.AddRow()
	//省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站
	row.AddCell().Value = "省份"
	row.AddCell().Value = "公告类别"
	row.AddCell().Value = "公告标题"
	row.AddCell().Value = "项目名称"
	row.AddCell().Value = "发布时间"
	row.AddCell().Value = "来源网站"
	row.AddCell().Value = "公告地址"
	row.AddCell().Value = "剑鱼地址"

	for _,tmp:=range dataArr {
		row = sheet.AddRow()
		row.AddCell().SetString(tmp["area"])
		row.AddCell().SetString(tmp["subtype"])
		row.AddCell().SetString(tmp["title"])
		row.AddCell().SetString(tmp["projectname"])
		row.AddCell().SetString(tmp["publishtime"])
		row.AddCell().SetString(tmp["site"])
		row.AddCell().SetString(tmp["href"])
		row.AddCell().SetString(tmp["jyhref"])
	}
	err := f.Save(xlsxName)
	if err != nil {
		log.Println("保存xlsx失败:", err)
		sendWarningSmtp("保存xlsx异常","请检查...上海通服...")
	}else {
		log.Println("保存xlsx成功:", err)
		body_str := fmt.Sprintf("日期：%s~%s\t\t总计：%d条",start_str,end_str,isok)
		sendErrMailSmtp("上海通服数据~月）",body_str,xlsxName)
	}
}