|
|
@@ -0,0 +1,318 @@
|
|
|
+package timedTask
|
|
|
+
|
|
|
+import (
|
|
|
+ "app.yhyue.com/moapp/jybase/common"
|
|
|
+ "encoding/json"
|
|
|
+ "fmt"
|
|
|
+ "github.com/tealeg/xlsx"
|
|
|
+ . "hostkword/config"
|
|
|
+ "io"
|
|
|
+ "log"
|
|
|
+ "os"
|
|
|
+ "sync"
|
|
|
+ "time"
|
|
|
+)
|
|
|
+
|
|
|
+const (
|
|
|
+ filePathDoc = "./hotWordFile/jyDoc.xlsx"
|
|
|
+)
|
|
|
+
|
|
|
+type userDoc struct {
|
|
|
+ name string
|
|
|
+ url string ///
|
|
|
+ docTags string
|
|
|
+ number int
|
|
|
+ create string
|
|
|
+}
|
|
|
+
|
|
|
+func DocStatistics() {
|
|
|
+ //MgoLog.Del("jy_hot_word", map[string]interface{}{
|
|
|
+ // "$or": []map[string]interface{}{
|
|
|
+ // {"statistical_type": 8},
|
|
|
+ // {"statistical_type": 7},
|
|
|
+ // },
|
|
|
+ //})
|
|
|
+ tm := time.Now()
|
|
|
+ startTime := tm.AddDate(0, 0, -7)
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "date": map[string]interface{}{
|
|
|
+ "$gt": startTime.Unix(),
|
|
|
+ "$lt": tm.Unix(), //统计7天的数据
|
|
|
+ },
|
|
|
+ "url": map[string]interface{}{"$regex": "jydocs/search"},
|
|
|
+ }
|
|
|
+ dataMap := make(map[string]int)
|
|
|
+ docData, _ := MgoLog.Find("jy_docs_logs", query, "", `{"mdescribe":1}`, false, -1, -1)
|
|
|
+ if docData != nil {
|
|
|
+ for _, m := range *docData {
|
|
|
+ var data map[string]interface{}
|
|
|
+ err := json.Unmarshal([]byte(common.InterfaceToStr(m["mdescribe"])), &data)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("JSON parsing error:", err)
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ keyWord, _ := data["keyWord"].([]interface{})
|
|
|
+ for _, s := range common.ObjArrToStringArr(keyWord) {
|
|
|
+ dataMap[s]++
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ var recordData []map[string]interface{}
|
|
|
+ for key, num := range dataMap { //分词插入
|
|
|
+ recordData = append(recordData, map[string]interface{}{
|
|
|
+ "keyWord": key,
|
|
|
+ "number": num,
|
|
|
+ "statistical_type": 7,
|
|
|
+ "year": time.Now().Year(),
|
|
|
+ "month": int(time.Now().Month()),
|
|
|
+ "day": time.Now().Day(),
|
|
|
+ "cycle": fmt.Sprintf("%s-%s", startTime.Format("01.02"), tm.Format("01.02")),
|
|
|
+ })
|
|
|
+ if len(recordData) > 100 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ recordData = []map[string]interface{}{}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ dataSta := make(map[string]userDoc)
|
|
|
+ //统计下载
|
|
|
+ myData := Mysql.SelectBySql(`SELECT a.*,b.docTags FROM jydocs.user_doc a LEFT JOIN jydocs.doc b on a.docId = b.id
|
|
|
+WHERE a.create_at >? ORDER BY a.create_at`, startTime.Format("2006-01-02 15:04:05"))
|
|
|
+ if myData != nil {
|
|
|
+ for _, m := range *myData {
|
|
|
+ docId := common.InterfaceToStr(m["docId"])
|
|
|
+ dataSta[docId] = userDoc{
|
|
|
+ name: common.InterfaceToStr(m["docName"]),
|
|
|
+ url: fmt.Sprintf("https://www.jianyu360.cn/swordfish/docs/content/%s", docId),
|
|
|
+ docTags: common.InterfaceToStr(m["docTags"]),
|
|
|
+ number: dataSta[docId].number + 1,
|
|
|
+ create: common.InterfaceToStr(m["create_at"]),
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for _, num := range dataSta { //文档下载
|
|
|
+ recordData = append(recordData, map[string]interface{}{
|
|
|
+ "name": num.name,
|
|
|
+ "url": num.url,
|
|
|
+ "create": num.create,
|
|
|
+ "number": num.number,
|
|
|
+ "docTags": num.docTags,
|
|
|
+ "statistical_type": 8,
|
|
|
+ "year": time.Now().Year(),
|
|
|
+ "month": int(time.Now().Month()),
|
|
|
+ "day": time.Now().Day(),
|
|
|
+ "cycle": fmt.Sprintf("%s-%s", startTime.Format("01.02"), tm.Format("01.02")),
|
|
|
+ })
|
|
|
+ if len(recordData) > 100 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ recordData = []map[string]interface{}{}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if len(recordData) > 0 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ }
|
|
|
+ //统计本月数据 发送邮件
|
|
|
+ DocXlsx()
|
|
|
+}
|
|
|
+
|
|
|
+func DocSendMail() {
|
|
|
+ f, err := os.Open(filePathDoc)
|
|
|
+ if err != nil {
|
|
|
+ panic(err)
|
|
|
+ }
|
|
|
+ bb, err := io.ReadAll(f)
|
|
|
+ if err != nil {
|
|
|
+ panic(err)
|
|
|
+ }
|
|
|
+ SendRetryMail(3, "yanglu@topnet.net.cn", "文档统计", fmt.Sprintf("%d年%d月%d日文档统计", time.Now().Year(), int(time.Now().Month()), time.Now().Day()), filePathDoc, bb, GmailAuth)
|
|
|
+}
|
|
|
+
|
|
|
+func DocXlsx() {
|
|
|
+ // 创建一个新的 Excel 文件
|
|
|
+ var (
|
|
|
+ sheet1, sheet2 *xlsx.Sheet
|
|
|
+ file *xlsx.File
|
|
|
+ sheetName1, sheetName2 = "文档词频统计", "文档下载统计"
|
|
|
+ )
|
|
|
+ _, err := os.Stat(filePathDoc)
|
|
|
+ if err == nil {
|
|
|
+ log.Printf("文件 %s 存在\n", filePathDoc)
|
|
|
+ // 打开 Excel 文件
|
|
|
+ file, err = xlsx.OpenFile(filePathDoc)
|
|
|
+ if err != nil {
|
|
|
+ log.Printf("打开文件时发生错误: %s\n", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ sheet1 = file.Sheet[sheetName1]
|
|
|
+ if sheet1 == nil {
|
|
|
+ sheet1, err = file.AddSheet(sheetName1)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // 清空 Sheet 的数据
|
|
|
+ sheet1.Rows = []*xlsx.Row{}
|
|
|
+ }
|
|
|
+ sheet2 = file.Sheet[sheetName2]
|
|
|
+ if sheet2 == nil {
|
|
|
+ sheet2, err = file.AddSheet(sheetName2)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ sheet2.Rows = []*xlsx.Row{}
|
|
|
+ }
|
|
|
+
|
|
|
+ } else if os.IsNotExist(err) {
|
|
|
+ log.Printf("文件 %s 不存在\n", filePath)
|
|
|
+ file = xlsx.NewFile()
|
|
|
+ // 创建一个工作表
|
|
|
+ sheet1, err = file.AddSheet(sheetName1)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ sheet2, err = file.AddSheet(sheetName2)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ log.Println("文件校验失败:", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ wait := &sync.WaitGroup{}
|
|
|
+ wait.Add(2)
|
|
|
+ go func() { //词条数据
|
|
|
+ defer wait.Done()
|
|
|
+ keyArr1, data1, staDay1 := DocKeyWod()
|
|
|
+ DocTableData(sheet1, keyArr1, data1, staDay1)
|
|
|
+ }()
|
|
|
+ go func() { //分词数据
|
|
|
+ defer wait.Done()
|
|
|
+ allData := DocDownload()
|
|
|
+ DocDownloadData(sheet2, allData)
|
|
|
+ }()
|
|
|
+
|
|
|
+ wait.Wait()
|
|
|
+ // 保存 Excel 文件
|
|
|
+ err = file.Save(filePathDoc)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("保存 Excel 文件出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ log.Println("Excel 文件生成成功")
|
|
|
+ //发送邮件
|
|
|
+ DocSendMail()
|
|
|
+}
|
|
|
+
|
|
|
+func DocKeyWod() (strArr []string, statisticsMap []map[string]int, staDay []string) {
|
|
|
+ query := make(map[string]interface{})
|
|
|
+ query["statistical_type"] = 7
|
|
|
+ query["year"] = time.Now().Year()
|
|
|
+ query["month"] = int(time.Now().Month())
|
|
|
+ sess := MgoLog.GetMgoConn()
|
|
|
+ defer MgoLog.DestoryMongoConn(sess)
|
|
|
+ var allData []map[string]interface{}
|
|
|
+ sess.DB(MgoLog.DbName).C("jy_hot_word").Find(query).Sort("cycle", "-number").All(&allData)
|
|
|
+ if len(allData) > 0 {
|
|
|
+ cMap := make(map[string]bool)
|
|
|
+ dataMap := make(map[string]bool)
|
|
|
+
|
|
|
+ for _, datum := range allData {
|
|
|
+ cycle := common.InterfaceToStr(datum["cycle"])
|
|
|
+ if !cMap[cycle] {
|
|
|
+ staDay = append(staDay, cycle)
|
|
|
+ cMap[cycle] = true
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for key := range cMap {
|
|
|
+ dMap := make(map[string]int)
|
|
|
+ for _, datum := range allData {
|
|
|
+ if key == common.InterfaceToStr(datum["cycle"]) {
|
|
|
+ keyWord := common.InterfaceToStr(datum["keyWord"])
|
|
|
+ dMap[keyWord] = common.IntAll(datum["number"])
|
|
|
+ if !dataMap[keyWord] {
|
|
|
+ dataMap[keyWord] = true
|
|
|
+ strArr = append(strArr, keyWord)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ statisticsMap = append(statisticsMap, dMap)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+func DocDownload() []map[string]interface{} {
|
|
|
+ query := make(map[string]interface{})
|
|
|
+ query["statistical_type"] = 8
|
|
|
+ query["year"] = time.Now().Year()
|
|
|
+ query["month"] = int(time.Now().Month())
|
|
|
+ sess := MgoLog.GetMgoConn()
|
|
|
+ defer MgoLog.DestoryMongoConn(sess)
|
|
|
+ var allData []map[string]interface{}
|
|
|
+ sess.DB(MgoLog.DbName).C("jy_hot_word").Find(query).Sort("cycle", "-number").All(&allData)
|
|
|
+ return allData
|
|
|
+}
|
|
|
+
|
|
|
+func DocTableData(sheet *xlsx.Sheet, keyArr []string, data []map[string]int, staDay []string) {
|
|
|
+ // 设置表头
|
|
|
+ row := sheet.AddRow()
|
|
|
+ cell := row.AddCell()
|
|
|
+ cell.Value = "检索词"
|
|
|
+ for _, v := range staDay {
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = v
|
|
|
+ }
|
|
|
+ // 写入检索词和数据
|
|
|
+ for _, key := range keyArr {
|
|
|
+ row = sheet.AddRow()
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = key
|
|
|
+ for _, datum := range data {
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.SetInt(datum[key])
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func DocDownloadData(sheet *xlsx.Sheet, allData []map[string]interface{}) {
|
|
|
+ // 设置表头
|
|
|
+ row := sheet.AddRow()
|
|
|
+ cell := row.AddCell()
|
|
|
+ cell.Value = "周期"
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = "名称"
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = "行业分类"
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = "链接"
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = "下载次数"
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = "日期"
|
|
|
+ // 写入检索词和数据
|
|
|
+ for _, data := range allData {
|
|
|
+ row = sheet.AddRow()
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.SetString(common.InterfaceToStr(data["cycle"]))
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.SetString(common.InterfaceToStr(data["name"]))
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.SetString(common.InterfaceToStr(data["docTags"]))
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.SetString(common.InterfaceToStr(data["url"]))
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.SetInt(common.IntAll(data["number"]))
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.SetString(common.InterfaceToStr(data["create"]))
|
|
|
+ }
|
|
|
+}
|
