package main import ( "fmt" "github.com/cron" "github.com/tealeg/xlsx" "log" "os" qu "qfw/util" "time" ) var ( sysconfig map[string]interface{} //配置文件 save_mgo *MongodbSim //mongodb操作对象 save_c_name string ) var Url = "https://www.jianyu360.com/article/content/%s.html" func initMgo() { saveconf := sysconfig["save_mgodb"].(map[string]interface{}) save_c_name = qu.ObjToString(saveconf["coll"]) save_mgo = &MongodbSim{ MongodbAddr: saveconf["addr"].(string), DbName: saveconf["db"].(string), Size: qu.IntAllDef(saveconf["pool"], 5), Password: "zk@123123", UserName: "zhengkun", } save_mgo.InitPool() } func init() { //加载配置文件 qu.ReadConfig(&sysconfig) initMgo() } func main() { //定时任务 c := cron.New() c.AddFunc("0 30 7 ? * *", func() { exportSpecSiteDataWeek() //周邮件-每周1点 8点 exportSpecSiteDataMonth() //月邮件-每月1号 8点 }) c.Start() //立即处理一次 exportSpecSiteDataWeek() exportSpecSiteDataMonth() time.Sleep(99999*time.Hour) } //处理数据-周邮件 func exportSpecSiteDataWeek() { cur_time := time.Now().Unix() today := GetOneWeekDay(TimeStampToString(cur_time)) if today!=1 { return } log.Println("每周一:准备邮件数据...") now:=time.Now() durdays:=7*2 start:= time.Date(now.Year(), now.Month(), now.Day()-durdays, 0, 0, 0, 0, time.Local).Unix() end := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix() log.Println(start,end) q := map[string]interface{}{ "comeintime": map[string]interface{}{ "$gte": start, "$lt": end, }, } sess := save_mgo.GetMgoConn() defer save_mgo.DestoryMongoConn(sess) log.Println("bidding 查询条件:",q) //省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站 it_site := sess.DB(save_mgo.DbName).C(save_c_name).Find(&q).Sort("comeintime").Select(map[string]interface{}{ "area":1, "title":1, "subtype":1, "publishtime":1, "href":1, "projectname":1, "site":1, }).Iter() timeLayout := "2006-01-02" total,isok,dataArr:= 0,0,make([]map[string]string,0) for tmp := make(map[string]interface{}); it_site.Next(&tmp); total++ { if total%10000 == 0 { log.Println("cur index :", total,isok) } site:=qu.ObjToString(tmp["site"]) area:=qu.ObjToString(tmp["area"]) subtype:=qu.ObjToString(tmp["subtype"]) projectname:=qu.ObjToString(tmp["projectname"]) publishtime:=qu.Int64All(tmp["publishtime"]) new_publishtime := ""//转日期 if publishtime>0 { new_publishtime = time.Unix(publishtime, 0).Format(timeLayout) } href:=qu.ObjToString(tmp["href"]) title:=qu.ObjToString(tmp["title"]) if (site=="上海政府采购网" || site=="中国政府采购网") && area=="上海" { isok++ dataArr = append(dataArr, map[string]string{ "area":area, "subtype":subtype, "title":title, "projectname":projectname, "site":site, "publishtime":new_publishtime, "href":href, "jyhref":fmt.Sprintf(Url, qu.CommonEncodeArticle("content", BsonTOStringId(tmp["_id"]))), }) } tmp = make(map[string]interface{}) } log.Println("is site over :",total,isok) start_str := time.Unix(start, 0).Format(timeLayout) end_str := time.Unix(end-1, 0).Format(timeLayout) xlsxName := "上海通服:"+start_str+"~"+end_str+".xlsx" log.Println("邮件名:",xlsxName) os.Remove(xlsxName) f :=xlsx.NewFile() sheet, _ := f.AddSheet("上海数据") row := sheet.AddRow() //省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站 row.AddCell().Value = "省份" row.AddCell().Value = "公告类别" row.AddCell().Value = "公告标题" row.AddCell().Value = "项目名称" row.AddCell().Value = "发布时间" row.AddCell().Value = "来源网站" row.AddCell().Value = "公告地址" row.AddCell().Value = "剑鱼地址" for _,tmp:=range dataArr { row = sheet.AddRow() row.AddCell().SetString(tmp["area"]) row.AddCell().SetString(tmp["subtype"]) row.AddCell().SetString(tmp["title"]) row.AddCell().SetString(tmp["projectname"]) row.AddCell().SetString(tmp["publishtime"]) row.AddCell().SetString(tmp["site"]) row.AddCell().SetString(tmp["href"]) row.AddCell().SetString(tmp["jyhref"]) } err := f.Save(xlsxName) if err != nil { log.Println("保存xlsx失败:", err) sendWarningSmtp("保存xlsx异常","请检查...上海通服...") }else { log.Println("保存xlsx成功:", err) body_str := fmt.Sprintf("日期:%s~%s\t\t总计:%d条",start_str,end_str,isok) sendErrMailSmtp("上海通服数据~周",body_str,xlsxName) } } //处理数据-月邮件 func exportSpecSiteDataMonth() { cur_time := time.Now().Unix() cur_day := time.Now().Day() if cur_day!=1 { return } today := GetOneWeekDay(TimeStampToString(cur_time)) log.Println("每月1号:准备邮件数据...") now:=time.Now() durdays:=int64(7)+today-int64(1) start:= time.Date(now.Year(), now.Month(), now.Day()-int(durdays), 0, 0, 0, 0, time.Local).Unix() end := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix() log.Println(start,end) q := map[string]interface{}{ "comeintime": map[string]interface{}{ "$gte": start, "$lt": end, }, } sess := save_mgo.GetMgoConn() defer save_mgo.DestoryMongoConn(sess) log.Println("bidding 查询条件:",q) //省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站 it_site := sess.DB(save_mgo.DbName).C(save_c_name).Find(&q).Sort("comeintime").Select(map[string]interface{}{ "area":1, "title":1, "subtype":1, "publishtime":1, "href":1, "projectname":1, "site":1, }).Iter() timeLayout := "2006-01-02" total,isok,dataArr:= 0,0,make([]map[string]string,0) for tmp := make(map[string]interface{}); it_site.Next(&tmp); total++ { if total%10000 == 0 { log.Println("cur index :", total,isok) } site:=qu.ObjToString(tmp["site"]) area:=qu.ObjToString(tmp["area"]) subtype:=qu.ObjToString(tmp["subtype"]) projectname:=qu.ObjToString(tmp["projectname"]) publishtime:=qu.Int64All(tmp["publishtime"]) new_publishtime := ""//转日期 if publishtime>0 { new_publishtime = time.Unix(publishtime, 0).Format(timeLayout) } href:=qu.ObjToString(tmp["href"]) title:=qu.ObjToString(tmp["title"]) if (site=="上海政府采购网" || site=="中国政府采购网") && area=="上海" { isok++ dataArr = append(dataArr, map[string]string{ "area":area, "subtype":subtype, "title":title, "projectname":projectname, "site":site, "publishtime":new_publishtime, "href":href, "jyhref":fmt.Sprintf(Url, qu.CommonEncodeArticle("content", BsonTOStringId(tmp["_id"]))), }) } tmp = make(map[string]interface{}) } log.Println("is site over :",total,isok) start_str := time.Unix(start, 0).Format(timeLayout) end_str := time.Unix(end-1, 0).Format(timeLayout) xlsxName := "上海通服:"+start_str+"~"+end_str+".xlsx" log.Println("邮件名:",xlsxName) os.Remove(xlsxName) f :=xlsx.NewFile() sheet, _ := f.AddSheet("上海数据") row := sheet.AddRow() //省份、公告标题、公告类别、发布时间、公告地址、项目名称、来源网站 row.AddCell().Value = "省份" row.AddCell().Value = "公告类别" row.AddCell().Value = "公告标题" row.AddCell().Value = "项目名称" row.AddCell().Value = "发布时间" row.AddCell().Value = "来源网站" row.AddCell().Value = "公告地址" row.AddCell().Value = "剑鱼地址" for _,tmp:=range dataArr { row = sheet.AddRow() row.AddCell().SetString(tmp["area"]) row.AddCell().SetString(tmp["subtype"]) row.AddCell().SetString(tmp["title"]) row.AddCell().SetString(tmp["projectname"]) row.AddCell().SetString(tmp["publishtime"]) row.AddCell().SetString(tmp["site"]) row.AddCell().SetString(tmp["href"]) row.AddCell().SetString(tmp["jyhref"]) } err := f.Save(xlsxName) if err != nil { log.Println("保存xlsx失败:", err) sendWarningSmtp("保存xlsx异常","请检查...上海通服...") }else { log.Println("保存xlsx成功:", err) body_str := fmt.Sprintf("日期:%s~%s\t\t总计:%d条",start_str,end_str,isok) sendErrMailSmtp("上海通服数据~月)",body_str,xlsxName) } }