|
|
@@ -0,0 +1,910 @@
|
|
|
+package timedTask
|
|
|
+
|
|
|
+import (
|
|
|
+ qutil "app.yhyue.com/moapp/jybase/common"
|
|
|
+ "app.yhyue.com/moapp/jybase/mail"
|
|
|
+ mg "app.yhyue.com/moapp/jybase/mongodb"
|
|
|
+ "encoding/json"
|
|
|
+ "fmt"
|
|
|
+ "github.com/tealeg/xlsx"
|
|
|
+ "hostkword/util"
|
|
|
+ "io"
|
|
|
+ "log"
|
|
|
+ "os"
|
|
|
+ "regexp"
|
|
|
+ "strings"
|
|
|
+ "sync"
|
|
|
+ "time"
|
|
|
+ "unicode/utf8"
|
|
|
+)
|
|
|
+
|
|
|
+type config struct {
|
|
|
+ Mail []struct {
|
|
|
+ Addr string `json:"addr"`
|
|
|
+ Port int `json:"port"`
|
|
|
+ Pwd string `json:"pwd"`
|
|
|
+ User string `json:"user"`
|
|
|
+ } `json:"mail"`
|
|
|
+
|
|
|
+ UserTimeLimit int `json:"userTimeLimit"`
|
|
|
+ EverydayWordLimit int `json:"everydayWordLimit"`
|
|
|
+ Mongodb struct {
|
|
|
+ Main struct {
|
|
|
+ Address string `json:"address"`
|
|
|
+ Size int `json:"size"`
|
|
|
+ DbName string `json:"dbName"`
|
|
|
+ } `json:"main"`
|
|
|
+ Log struct {
|
|
|
+ Address string `json:"address"`
|
|
|
+ Size int `json:"size"`
|
|
|
+ DbName string `json:"dbName"`
|
|
|
+ UserName string `json:"userName"`
|
|
|
+ Password string `json:"password"`
|
|
|
+ } `json:"log"`
|
|
|
+ } `json:"mongodb"`
|
|
|
+ Redis struct {
|
|
|
+ Main struct {
|
|
|
+ Address string `json:"address"`
|
|
|
+ } `json:"main"`
|
|
|
+ } `json:"redis"`
|
|
|
+ SubWordLimit int `json:"subWordLimit"`
|
|
|
+ WordDuration int `json:"wordDuration"`
|
|
|
+
|
|
|
+ UserMail string `json:"userMail"`
|
|
|
+}
|
|
|
+
|
|
|
+type keyAllData struct {
|
|
|
+ AItems []struct {
|
|
|
+ AKey []struct {
|
|
|
+ Key []string `json:"key"`
|
|
|
+ Appendkey []string `json:"appendkey"`
|
|
|
+ Notkey []interface{} `json:"notkey"`
|
|
|
+ } `json:"a_key"`
|
|
|
+ } `json:"a_items"`
|
|
|
+}
|
|
|
+
|
|
|
+var (
|
|
|
+ Mgo *mg.MongodbSim
|
|
|
+ MgoLog *mg.MongodbSim
|
|
|
+
|
|
|
+ Config *config
|
|
|
+ entWordNumber map[string]int
|
|
|
+ chineseRegex = regexp.MustCompile(`\p{Han}`)
|
|
|
+
|
|
|
+ GmailAuth []*mail.GmailAuth
|
|
|
+)
|
|
|
+
|
|
|
+func init() {
|
|
|
+ entWordNumber = make(map[string]int)
|
|
|
+ qutil.ReadConfig(&Config)
|
|
|
+ Mgo = &mg.MongodbSim{
|
|
|
+ MongodbAddr: Config.Mongodb.Main.Address,
|
|
|
+ Size: Config.Mongodb.Main.Size,
|
|
|
+ DbName: Config.Mongodb.Main.DbName,
|
|
|
+ UserName: "",
|
|
|
+ Password: "",
|
|
|
+ ReplSet: "",
|
|
|
+ }
|
|
|
+ MgoLog = &mg.MongodbSim{
|
|
|
+ MongodbAddr: Config.Mongodb.Log.Address,
|
|
|
+ Size: Config.Mongodb.Log.Size,
|
|
|
+ DbName: Config.Mongodb.Log.DbName,
|
|
|
+ UserName: Config.Mongodb.Log.UserName,
|
|
|
+ Password: Config.Mongodb.Log.Password,
|
|
|
+ ReplSet: "",
|
|
|
+ }
|
|
|
+ MgoLog.InitPool()
|
|
|
+ Mgo.InitPool()
|
|
|
+ //每月用户订阅词
|
|
|
+ UserWord()
|
|
|
+ //每月搜索统计搜索词
|
|
|
+ SearchWordStatistics()
|
|
|
+ for _, v := range Config.Mail {
|
|
|
+ GmailAuth = append(GmailAuth, &mail.GmailAuth{
|
|
|
+ SmtpHost: v.Addr,
|
|
|
+ SmtpPort: v.Port,
|
|
|
+ User: v.User,
|
|
|
+ Pwd: v.Pwd,
|
|
|
+ })
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func timeFmt() (int64, int64) {
|
|
|
+ now := time.Now()
|
|
|
+ // 计算起始时间
|
|
|
+ yesterdayStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local)
|
|
|
+ // 计算结束时间
|
|
|
+ yesterdayEnd := time.Date(now.Year(), now.Month(), now.Day(), 23, 0, 0, 0, time.Local).Add(-time.Second)
|
|
|
+ // 转换为时间戳
|
|
|
+ return yesterdayStart.Unix(), yesterdayEnd.Unix()
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+// 每月1号统计新搜索词频
|
|
|
+func SearchWordStatistics() {
|
|
|
+ tm := time.Now()
|
|
|
+ startTime := tm.AddDate(0, -Config.WordDuration, 0)
|
|
|
+ currentTime := startTime
|
|
|
+ participleMap := make(map[string]int)
|
|
|
+ glossaryMap := make(map[string]int)
|
|
|
+ if MgoLog.Count("jy_hot_word", map[string]interface{}{
|
|
|
+ "statistical_type": 1,
|
|
|
+ "year": time.Now().Year(),
|
|
|
+ "month": int(time.Now().Month()),
|
|
|
+ }) > 0 { //已统计过当月数据
|
|
|
+ return
|
|
|
+ }
|
|
|
+ pool := make(chan bool, 10)
|
|
|
+ wait := &sync.WaitGroup{}
|
|
|
+ var lock sync.Mutex
|
|
|
+ for currentTime.Before(tm) {
|
|
|
+ pool <- true
|
|
|
+ wait.Add(1)
|
|
|
+ go func(currentTime1 int64) {
|
|
|
+ defer func() {
|
|
|
+ wait.Done()
|
|
|
+ <-pool
|
|
|
+ }()
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "createtime": map[string]interface{}{
|
|
|
+ "$gt": currentTime1,
|
|
|
+ "$lt": currentTime1 + 3600*24, //统计一天的数据
|
|
|
+ },
|
|
|
+ "search_word": map[string]interface{}{
|
|
|
+ "$nin": []interface{}{nil, ""},
|
|
|
+ },
|
|
|
+ }
|
|
|
+ participle, glossary := searchPartWord(query)
|
|
|
+ lock.Lock()
|
|
|
+ defer lock.Unlock()
|
|
|
+ for k, v := range participle {
|
|
|
+ participleMap[k] += v
|
|
|
+ }
|
|
|
+ for k, v := range glossary {
|
|
|
+ glossaryMap[k] += v
|
|
|
+ }
|
|
|
+ }(currentTime.Unix())
|
|
|
+ wait.Wait()
|
|
|
+ log.Println("执行完成日期", currentTime)
|
|
|
+ currentTime = currentTime.AddDate(0, 0, 1)
|
|
|
+ }
|
|
|
+ var recordData []map[string]interface{}
|
|
|
+ for key, num := range participleMap { //分词插入
|
|
|
+ if num > Config.SubWordLimit {
|
|
|
+ recordData = append(recordData, map[string]interface{}{
|
|
|
+ "keyWord": key,
|
|
|
+ "number": num,
|
|
|
+ "statistical_type": 1,
|
|
|
+ "year": time.Now().Year(),
|
|
|
+ "month": int(time.Now().Month()),
|
|
|
+ "day": time.Now().Day(),
|
|
|
+ })
|
|
|
+ }
|
|
|
+ if len(recordData) > 100 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ recordData = []map[string]interface{}{}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for key, num := range glossaryMap { //词条插入
|
|
|
+ if num > Config.SubWordLimit {
|
|
|
+ recordData = append(recordData, map[string]interface{}{
|
|
|
+ "keyWord": key,
|
|
|
+ "number": num,
|
|
|
+ "statistical_type": 2,
|
|
|
+ "year": time.Now().Year(),
|
|
|
+ "month": int(time.Now().Month()),
|
|
|
+ "day": time.Now().Day(),
|
|
|
+ })
|
|
|
+ }
|
|
|
+ if len(recordData) > 100 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ recordData = []map[string]interface{}{}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(recordData) > 0 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ }
|
|
|
+ log.Println("搜索词频初始化完成")
|
|
|
+}
|
|
|
+
|
|
|
+// 每日搜索词统计
|
|
|
+func DailySearchWordStatistics() {
|
|
|
+ st, et := timeFmt()
|
|
|
+ if MgoLog.Count("jy_hot_word", map[string]interface{}{
|
|
|
+ "statistical_type": 4, //词条
|
|
|
+ "year": time.Now().Year(),
|
|
|
+ "month": int(time.Now().Month()),
|
|
|
+ "day": time.Now().Day(),
|
|
|
+ }) > 0 { //已统计过当日数据
|
|
|
+ return
|
|
|
+ }
|
|
|
+ var recordData []map[string]interface{}
|
|
|
+ //每日词条搜索统计
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "createtime": map[string]interface{}{
|
|
|
+ "$gt": st,
|
|
|
+ "$lt": et,
|
|
|
+ },
|
|
|
+ "search_word": map[string]interface{}{
|
|
|
+ "$nin": []interface{}{nil, ""},
|
|
|
+ },
|
|
|
+ }
|
|
|
+
|
|
|
+ participleMap, glossaryMap := searchPartWord(query)
|
|
|
+ for key, num := range participleMap {
|
|
|
+ if num > Config.EverydayWordLimit {
|
|
|
+ recordData = append(recordData, map[string]interface{}{
|
|
|
+ "keyWord": key,
|
|
|
+ "number": num,
|
|
|
+ "statistical_type": 3, //分词
|
|
|
+ "year": time.Now().Year(),
|
|
|
+ "month": int(time.Now().Month()),
|
|
|
+ "day": time.Now().Day(),
|
|
|
+ })
|
|
|
+ if len(recordData) > 100 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ recordData = []map[string]interface{}{}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for key, num := range glossaryMap {
|
|
|
+ if num > Config.EverydayWordLimit {
|
|
|
+ recordData = append(recordData, map[string]interface{}{
|
|
|
+ "keyWord": key,
|
|
|
+ "number": num,
|
|
|
+ "statistical_type": 4, //词条
|
|
|
+ "year": time.Now().Year(),
|
|
|
+ "month": int(time.Now().Month()),
|
|
|
+ "day": time.Now().Day(),
|
|
|
+ })
|
|
|
+ if len(recordData) > 100 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ recordData = []map[string]interface{}{}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(recordData) > 0 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func keyFmt(aItems interface{}) {
|
|
|
+ var keys keyAllData
|
|
|
+ // 将map转换为JSON字节切片
|
|
|
+ jsonData, err := json.Marshal(aItems)
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ err = json.Unmarshal(jsonData, &keys)
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ for _, k1 := range keys.AItems {
|
|
|
+ for _, k2 := range k1.AKey {
|
|
|
+ for _, k3 := range k2.Key {
|
|
|
+ if HasChineseAndValidLength(k3) {
|
|
|
+ entWordNumber[k3]++
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ for _, k3 := range k2.Appendkey {
|
|
|
+ if HasChineseAndValidLength(k3) {
|
|
|
+ entWordNumber[k3]++
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// 企业订阅
|
|
|
+func EntWord() {
|
|
|
+ _, lt := IsOneDay()
|
|
|
+ tms := lt.AddDate(0, 0, -1)
|
|
|
+ if MgoLog.Count("jy_hot_word", map[string]interface{}{
|
|
|
+ "statistical_type": 6,
|
|
|
+ "year": tms.Year(),
|
|
|
+ "month": int(tms.Month()),
|
|
|
+ }) > 0 { //已统计过当月数据
|
|
|
+ return
|
|
|
+ }
|
|
|
+ //企业关键词查找
|
|
|
+ res_, _ := Mgo.Find("entniche_rule", map[string]interface{}{
|
|
|
+ "i_userid": map[string]interface{}{
|
|
|
+ "$exists": false,
|
|
|
+ },
|
|
|
+ }, "", `{"o_entniche":1}`, false, -1, -1)
|
|
|
+ o_entniche := map[string]interface{}{}
|
|
|
+ if res_ != nil && len(*res_) > 0 {
|
|
|
+ for _, ruleV := range *res_ {
|
|
|
+ o_entniche = qutil.StructToMapMore(ruleV["o_entniche"])
|
|
|
+ if o_entniche["a_items"] != nil {
|
|
|
+ keyFmt(o_entniche)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ var recordData []map[string]interface{}
|
|
|
+ for key, number := range entWordNumber {
|
|
|
+ if number > Config.UserTimeLimit && HasChineseAndValidLength(key) {
|
|
|
+ recordData = append(recordData, map[string]interface{}{
|
|
|
+ "keyWord": key,
|
|
|
+ "number": number,
|
|
|
+ "statistical_type": 6,
|
|
|
+ "year": tms.Year(),
|
|
|
+ "month": int(tms.Month()),
|
|
|
+ "day": tms.Day(),
|
|
|
+ })
|
|
|
+ }
|
|
|
+ if len(recordData) > 100 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ recordData = []map[string]interface{}{}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(recordData) > 0 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func IsOneDay() (time.Time, time.Time) {
|
|
|
+ now := time.Now()
|
|
|
+ firstDayOfMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
|
|
|
+ st := firstDayOfMonth.AddDate(0, 0, -1)
|
|
|
+ stm := time.Date(st.Year(), st.Month(), 1, 0, 0, 0, 0, now.Location())
|
|
|
+ return stm, firstDayOfMonth
|
|
|
+}
|
|
|
+
|
|
|
+func UserMonthWord() {
|
|
|
+ // 个人订阅词
|
|
|
+ //总词频统计
|
|
|
+ gt, lt := IsOneDay()
|
|
|
+ tms := lt.AddDate(0, 0, -1)
|
|
|
+ if MgoLog.Count("jy_hot_word", map[string]interface{}{
|
|
|
+ "statistical_type": 5,
|
|
|
+ "year": tms.Year(),
|
|
|
+ "month": int(tms.Month()),
|
|
|
+ }) > 0 { //已统计过当月数据
|
|
|
+ return
|
|
|
+ }
|
|
|
+ sess := Mgo.GetMgoConn()
|
|
|
+ defer Mgo.DestoryMongoConn(sess)
|
|
|
+ pipeline := []map[string]interface{}{
|
|
|
+ {
|
|
|
+ "$match": map[string]interface{}{
|
|
|
+ "l_registedate": map[string]interface{}{"$gte": gt.Unix(), "$lte": lt.Unix()},
|
|
|
+ "$or": []map[string]interface{}{
|
|
|
+ {"o_jy.a_key.key.0": map[string]interface{}{"$exists": 1}},
|
|
|
+ {"o_vipjy.a_items.a_key.key.0": map[string]interface{}{"$exists": 1}},
|
|
|
+ {"o_member_jy.a_items.a_key.key.1": map[string]interface{}{"$exists": 1}},
|
|
|
+ },
|
|
|
+ },
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "$project": map[string]interface{}{
|
|
|
+ "key": "$o_jy.a_key.key",
|
|
|
+ "key1": "$o_vipjy.a_items.a_key.key",
|
|
|
+ "key2": "$o_member_jy.a_items.a_key.key",
|
|
|
+ "key3": "$o_jy.a_key.appendkey",
|
|
|
+ "key4": "$o_vipjy.a_items.a_key.appendkey",
|
|
|
+ "key5": "$o_member_jy.a_items.a_key.appendkey",
|
|
|
+ },
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "$project": map[string]interface{}{
|
|
|
+ "keya": []string{"$key", "$key3", "$key4", "$key1", "$key5", "$key2"},
|
|
|
+ },
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "$unwind": "$keya",
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "$unwind": "$keya",
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "$unwind": "$keya",
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "$unwind": "$keya",
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "$group": map[string]interface{}{
|
|
|
+ "_id": "$keya",
|
|
|
+ "sum": map[string]interface{}{"$sum": 1},
|
|
|
+ },
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "$match": map[string]interface{}{
|
|
|
+ "sum": map[string]interface{}{"$gt": 1},
|
|
|
+ },
|
|
|
+ },
|
|
|
+ }
|
|
|
+ var data []map[string]interface{}
|
|
|
+ sess.DB(Mgo.DbName).C("user").Pipe(pipeline).All(&data)
|
|
|
+ if data != nil && len(data) > 0 {
|
|
|
+ var recordData []map[string]interface{}
|
|
|
+ for _, v := range data {
|
|
|
+ keyWord := qutil.InterfaceToStr(v["_id"])
|
|
|
+ if HasChineseAndValidLength(keyWord) {
|
|
|
+ recordData = append(recordData, map[string]interface{}{
|
|
|
+ "keyWord": keyWord,
|
|
|
+ "number": v["sum"],
|
|
|
+ "statistical_type": 5,
|
|
|
+ "year": tms.Year(),
|
|
|
+ "month": int(tms.Month()),
|
|
|
+ "day": tms.Day(),
|
|
|
+ })
|
|
|
+ }
|
|
|
+ if len(recordData) > 100 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ recordData = []map[string]interface{}{}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(recordData) > 0 {
|
|
|
+ // 插入数据库
|
|
|
+ MgoLog.SaveBulk("jy_hot_word", recordData...)
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+type UserKeyWord struct {
|
|
|
+ OMemberJy struct {
|
|
|
+ AItems []struct {
|
|
|
+ AKey []struct {
|
|
|
+ Key []string `json:"key"`
|
|
|
+ Appendkey []string `json:"appendkey"`
|
|
|
+ } `json:"a_key"`
|
|
|
+ } `json:"a_items"`
|
|
|
+ } `json:"o_member_jy"`
|
|
|
+ OVipjy struct {
|
|
|
+ AItems []struct {
|
|
|
+ AKey []struct {
|
|
|
+ Appendkey []string `json:"appendkey"`
|
|
|
+ Key []string `json:"key"`
|
|
|
+ } `json:"a_key"`
|
|
|
+ } `json:"a_items"`
|
|
|
+ } `json:"o_vipjy"`
|
|
|
+ OJy struct {
|
|
|
+ AKey []struct {
|
|
|
+ Appendkey []string `json:"appendkey"`
|
|
|
+ Key []string `json:"key"`
|
|
|
+ } `json:"a_key"`
|
|
|
+ } `json:"o_jy"`
|
|
|
+}
|
|
|
+
|
|
|
+func UserWordAll() []map[string]interface{} {
|
|
|
+ _, lt := IsOneDay()
|
|
|
+ sess := Mgo.GetMgoConn()
|
|
|
+ defer Mgo.DestoryMongoConn(sess)
|
|
|
+ keyWordMap := make(map[string]int)
|
|
|
+ iter := sess.DB(Mgo.DbName).C("user").Find(map[string]interface{}{
|
|
|
+ "l_registedate": map[string]interface{}{
|
|
|
+ "$lt": lt.Unix(),
|
|
|
+ },
|
|
|
+ "$or": []map[string]interface{}{
|
|
|
+ {"o_jy.a_key": map[string]interface{}{"$exists": 1}},
|
|
|
+ {"o_vipjy.a_items": map[string]interface{}{"$exists": 1}},
|
|
|
+ {"o_member_jy.a_items": map[string]interface{}{"$exists": 1}},
|
|
|
+ },
|
|
|
+ }).Select(map[string]interface{}{"o_jy": 1, "o_vipjy": 1, "o_member_jy": 1}).Iter()
|
|
|
+ for m := make(map[string]interface{}); iter.Next(&m); {
|
|
|
+ var data UserKeyWord
|
|
|
+ ms, _ := json.Marshal(m)
|
|
|
+ json.Unmarshal(ms, &data)
|
|
|
+ for _, keys := range data.OJy.AKey {
|
|
|
+ for _, key := range keys.Key {
|
|
|
+ keyWordMap[key]++
|
|
|
+ }
|
|
|
+ for _, appendKey := range keys.Appendkey {
|
|
|
+ keyWordMap[appendKey]++
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for _, keys := range data.OVipjy.AItems {
|
|
|
+ for _, key := range keys.AKey {
|
|
|
+ for _, word := range key.Key {
|
|
|
+ keyWordMap[word]++
|
|
|
+ }
|
|
|
+ for _, appendKey := range key.Appendkey {
|
|
|
+ keyWordMap[appendKey]++
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for _, keys := range data.OMemberJy.AItems {
|
|
|
+ for _, key := range keys.AKey {
|
|
|
+ for _, word := range key.Key {
|
|
|
+ keyWordMap[word]++
|
|
|
+ }
|
|
|
+ for _, appendKey := range key.Appendkey {
|
|
|
+ keyWordMap[appendKey]++
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ m = make(map[string]interface{})
|
|
|
+ }
|
|
|
+ var dataArr []map[string]interface{}
|
|
|
+ for key, number := range keyWordMap {
|
|
|
+ if number > Config.SubWordLimit && HasChineseAndValidLength(key) {
|
|
|
+ dataArr = append(dataArr, map[string]interface{}{
|
|
|
+ "keyWord": key,
|
|
|
+ "number": number,
|
|
|
+ })
|
|
|
+ }
|
|
|
+ }
|
|
|
+ util.SortData(&dataArr, "number", true)
|
|
|
+ return dataArr
|
|
|
+}
|
|
|
+
|
|
|
+// 订阅词
|
|
|
+func UserWord() {
|
|
|
+ var (
|
|
|
+ userArr, entArr []string
|
|
|
+ userWordArr, entWordArr []map[string]int
|
|
|
+ )
|
|
|
+
|
|
|
+ dataMap := make(map[string]bool)
|
|
|
+ dataEntMap := make(map[string]bool)
|
|
|
+ UserMonthWord() //个人月词
|
|
|
+ EntWord() //企业月词
|
|
|
+ wordAll := UserWordAll() //个人总词
|
|
|
+
|
|
|
+ dMap := make(map[string]int)
|
|
|
+ for _, v := range wordAll { //个人订阅词频
|
|
|
+ keyWord := qutil.InterfaceToStr(v["keyWord"])
|
|
|
+ dMap[keyWord] = qutil.IntAll(v["number"])
|
|
|
+ if !dataMap[keyWord] {
|
|
|
+ dataMap[keyWord] = true
|
|
|
+ userArr = append(userArr, keyWord)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ userWordArr = append(userWordArr, dMap)
|
|
|
+
|
|
|
+ tm := time.Now()
|
|
|
+ var userStaMon, entStaMon int
|
|
|
+ for i := 1; i < int(tm.Month()); i++ {
|
|
|
+ data, ok := MgoLog.Find("jy_hot_word", map[string]interface{}{
|
|
|
+ "year": tm.Year(),
|
|
|
+ "month": i,
|
|
|
+ "statistical_type": 5,
|
|
|
+ }, `{"number":-1}`, "", false, -1, -1)
|
|
|
+ if ok && len(*data) > 0 {
|
|
|
+ dMonMap := make(map[string]int)
|
|
|
+ if userStaMon == 0 {
|
|
|
+ userStaMon = i
|
|
|
+ }
|
|
|
+ for _, v := range *data {
|
|
|
+ keyWord := qutil.InterfaceToStr(v["keyWord"])
|
|
|
+ dMonMap[keyWord] = qutil.IntAll(v["number"])
|
|
|
+ if !dataMap[keyWord] {
|
|
|
+ dataMap[keyWord] = true
|
|
|
+ userArr = append(userArr, keyWord)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ userWordArr = append(userWordArr, dMonMap)
|
|
|
+ }
|
|
|
+ data, ok = MgoLog.Find("jy_hot_word", map[string]interface{}{
|
|
|
+ "year": tm.Year(),
|
|
|
+ "month": i,
|
|
|
+ "statistical_type": 6,
|
|
|
+ }, `{"number":-1}`, "", false, -1, -1)
|
|
|
+ if ok && len(*data) > 0 {
|
|
|
+ if entStaMon == 0 {
|
|
|
+ entStaMon = i
|
|
|
+ }
|
|
|
+ dEntMap := make(map[string]int)
|
|
|
+ for _, v := range *data {
|
|
|
+ keyWord := qutil.InterfaceToStr(v["keyWord"])
|
|
|
+ dEntMap[keyWord] = qutil.IntAll(v["number"])
|
|
|
+ if !dataEntMap[keyWord] {
|
|
|
+ dataEntMap[keyWord] = true
|
|
|
+ entArr = append(entArr, keyWord)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ entWordArr = append(entWordArr, dEntMap)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ var (
|
|
|
+ sheet1, sheet2 *xlsx.Sheet
|
|
|
+ file *xlsx.File
|
|
|
+ )
|
|
|
+ //生成xlex
|
|
|
+ filePath := fmt.Sprintf("./hotWordFile/jyHotWord_%d%d.xlsx", time.Now().Year(), int(time.Now().Month()))
|
|
|
+ _, err := os.Stat(filePath)
|
|
|
+ sheetName1, sheetName2 := "个人订阅词", "企业订阅词"
|
|
|
+ if err == nil {
|
|
|
+ log.Printf("文件 %s 存在\n", filePath)
|
|
|
+ // 打开 Excel 文件
|
|
|
+ file, err = xlsx.OpenFile(filePath)
|
|
|
+ if err != nil {
|
|
|
+ log.Printf("打开文件时发生错误: %s\n", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ sheet1 = file.Sheet[sheetName1]
|
|
|
+ if sheet1 == nil {
|
|
|
+ sheet1, err = file.AddSheet(sheetName1)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // 清空 Sheet 的数据
|
|
|
+ sheet1.Rows = []*xlsx.Row{}
|
|
|
+ }
|
|
|
+
|
|
|
+ sheet2 = file.Sheet[sheetName2]
|
|
|
+ if sheet2 == nil {
|
|
|
+ sheet2, err = file.AddSheet(sheetName2)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // 清空 Sheet 的数据
|
|
|
+ sheet2.Rows = []*xlsx.Row{}
|
|
|
+ }
|
|
|
+
|
|
|
+ } else if os.IsNotExist(err) {
|
|
|
+ log.Printf("文件 %s 不存在\n", filePath)
|
|
|
+ file = xlsx.NewFile()
|
|
|
+ // 创建一个工作表
|
|
|
+ sheet1, err = file.AddSheet(sheetName1)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ sheet2, err = file.AddSheet(sheetName2)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ log.Println("文件校验失败:", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ TableDataMonth(sheet1, userArr, userWordArr, false, userStaMon) //个人订阅词
|
|
|
+ TableDataMonth(sheet2, entArr, entWordArr, true, entStaMon) //企业订阅词
|
|
|
+ // 保存 Excel 文件
|
|
|
+ err = file.Save(filePath)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("保存 Excel 文件出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ log.Println("Excel 文件生成成功")
|
|
|
+}
|
|
|
+
|
|
|
+// 搜索
|
|
|
+func searchPartWord(query map[string]interface{}) (map[string]int, map[string]int) {
|
|
|
+ sess := MgoLog.GetMgoConn()
|
|
|
+ defer MgoLog.DestoryMongoConn(sess)
|
|
|
+
|
|
|
+ iter := sess.DB(MgoLog.DbName).C("jy_search_log").Find(query).Select(map[string]interface{}{"search_word": 1}).Iter()
|
|
|
+ glossaryMap := make(map[string]int)
|
|
|
+ participleMap := make(map[string]int)
|
|
|
+ for m := make(map[string]interface{}); iter.Next(&m); {
|
|
|
+ searchWord := qutil.InterfaceToStr(m["search_word"])
|
|
|
+ if HasChineseAndValidLength(searchWord) {
|
|
|
+ glossaryMap[searchWord]++
|
|
|
+ }
|
|
|
+ searchWords := strings.Split(searchWord, " ")
|
|
|
+ if len(searchWords) > 1 {
|
|
|
+ for _, str := range searchWords {
|
|
|
+ if HasChineseAndValidLength(str) {
|
|
|
+ participleMap[str]++
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ m = make(map[string]interface{})
|
|
|
+ }
|
|
|
+ log.Println("本次统计数", len(participleMap), len(glossaryMap))
|
|
|
+ return participleMap, glossaryMap
|
|
|
+}
|
|
|
+
|
|
|
+// 每天生成搜索词表格
|
|
|
+func WordXlsx() {
|
|
|
+ // 创建一个新的 Excel 文件
|
|
|
+ var (
|
|
|
+ sheet1, sheet2 *xlsx.Sheet
|
|
|
+ file *xlsx.File
|
|
|
+ sheetName1, sheetName2 = "搜查词条", "搜索分词"
|
|
|
+ )
|
|
|
+
|
|
|
+ filePath := fmt.Sprintf("./hotWordFile/jyHotWord_%d%d.xlsx", time.Now().Year(), int(time.Now().Month()))
|
|
|
+ _, err := os.Stat(filePath)
|
|
|
+ if err == nil {
|
|
|
+ log.Printf("文件 %s 存在\n", filePath)
|
|
|
+ // 打开 Excel 文件
|
|
|
+ file, err = xlsx.OpenFile(filePath)
|
|
|
+ if err != nil {
|
|
|
+ log.Printf("打开文件时发生错误: %s\n", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ sheet1 = file.Sheet[sheetName1]
|
|
|
+ if sheet1 == nil {
|
|
|
+ sheet1, err = file.AddSheet(sheetName1)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // 清空 Sheet 的数据
|
|
|
+ sheet1.Rows = []*xlsx.Row{}
|
|
|
+ }
|
|
|
+ sheet2 = file.Sheet[sheetName2]
|
|
|
+ if sheet2 == nil {
|
|
|
+ sheet2, err = file.AddSheet(sheetName2)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ sheet2.Rows = []*xlsx.Row{}
|
|
|
+ }
|
|
|
+
|
|
|
+ } else if os.IsNotExist(err) {
|
|
|
+ log.Printf("文件 %s 不存在\n", filePath)
|
|
|
+ file = xlsx.NewFile()
|
|
|
+ // 创建一个工作表
|
|
|
+ sheet1, err = file.AddSheet(sheetName1)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ sheet2, err = file.AddSheet(sheetName2)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("创建工作表出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ log.Println("文件校验失败:", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ wait := &sync.WaitGroup{}
|
|
|
+ wait.Add(2)
|
|
|
+ go func() { //词条数据
|
|
|
+ defer wait.Done()
|
|
|
+ keyArr1, data1, staDay := wordXlsxData(false)
|
|
|
+ TableData(sheet1, keyArr1, data1, staDay)
|
|
|
+ }()
|
|
|
+ go func() { //分词数据
|
|
|
+ defer wait.Done()
|
|
|
+ keyArr2, data2, staDay := wordXlsxData(true)
|
|
|
+ TableData(sheet2, keyArr2, data2, staDay)
|
|
|
+ }()
|
|
|
+
|
|
|
+ wait.Wait()
|
|
|
+ // 保存 Excel 文件
|
|
|
+ err = file.Save(filePath)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("保存 Excel 文件出错:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ log.Println("Excel 文件生成成功")
|
|
|
+}
|
|
|
+
|
|
|
+func TableData(sheet *xlsx.Sheet, keyArr []string, data []map[string]int, staDay int) {
|
|
|
+ // 设置表头
|
|
|
+ row := sheet.AddRow()
|
|
|
+ cell := row.AddCell()
|
|
|
+ cell.Value = "检索词"
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = "词频"
|
|
|
+ for i := staDay; i <= time.Now().Day(); i++ {
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = fmt.Sprintf("%d号", i)
|
|
|
+ }
|
|
|
+ // 写入检索词和数据
|
|
|
+ for _, key := range keyArr {
|
|
|
+ row = sheet.AddRow()
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = key
|
|
|
+ for _, datum := range data {
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.SetInt(datum[key])
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func TableDataMonth(sheet *xlsx.Sheet, keyArr []string, data []map[string]int, isEnt bool, sta int) {
|
|
|
+ // 设置表头
|
|
|
+ row := sheet.AddRow()
|
|
|
+ cell := row.AddCell()
|
|
|
+ cell.Value = "检索词"
|
|
|
+ if !isEnt {
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = "词频"
|
|
|
+ }
|
|
|
+ for i := sta; i < int(time.Now().Month()); i++ {
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = fmt.Sprintf("%d月", i)
|
|
|
+ }
|
|
|
+ // 写入检索词和数据
|
|
|
+ for _, key := range keyArr {
|
|
|
+ row = sheet.AddRow()
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.Value = key
|
|
|
+ for _, datum := range data {
|
|
|
+ cell = row.AddCell()
|
|
|
+ cell.SetInt(datum[key])
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func wordXlsxData(isParticiple bool) (strArr []string, statisticsMap []map[string]int, staDay int) {
|
|
|
+ dataMap := make(map[string]bool)
|
|
|
+ tm := time.Now()
|
|
|
+ for i := 0; i <= tm.Day(); i++ {
|
|
|
+ query := make(map[string]interface{})
|
|
|
+ if i == 0 { //首次统计词频
|
|
|
+ query["statistical_type"] = qutil.If(isParticiple, 1, 2)
|
|
|
+ query["year"] = tm.Year()
|
|
|
+ query["month"] = int(tm.Month())
|
|
|
+ } else {
|
|
|
+ query["statistical_type"] = qutil.If(isParticiple, 3, 4)
|
|
|
+ query["year"] = tm.Year()
|
|
|
+ query["month"] = int(tm.Month())
|
|
|
+ query["day"] = i
|
|
|
+ }
|
|
|
+ data, ok := MgoLog.Find("jy_hot_word", query, `{"number":-1}`, "", false, -1, -1)
|
|
|
+ if ok && len(*data) > 0 {
|
|
|
+ dMap := make(map[string]int)
|
|
|
+ if staDay == 0 && i > 0 {
|
|
|
+ staDay = i
|
|
|
+ }
|
|
|
+ for _, v := range *data {
|
|
|
+ keyWord := qutil.InterfaceToStr(v["keyWord"])
|
|
|
+ dMap[keyWord] = qutil.IntAll(v["number"])
|
|
|
+ if !dataMap[keyWord] {
|
|
|
+ dataMap[keyWord] = true
|
|
|
+ strArr = append(strArr, keyWord)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ statisticsMap = append(statisticsMap, dMap)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// 过滤不符合条件的词
|
|
|
+func HasChineseAndValidLength(str string) bool {
|
|
|
+ length := utf8.RuneCountInString(str)
|
|
|
+ return chineseRegex.MatchString(str) && length >= 2 && length < 20
|
|
|
+}
|
|
|
+
|
|
|
+func SendMail() {
|
|
|
+ filePath := fmt.Sprintf("./hotWordFile/jyHotWord_%d%d.xlsx", time.Now().Year(), int(time.Now().Month()))
|
|
|
+ f, err := os.Open(filePath)
|
|
|
+ if err != nil {
|
|
|
+ panic(err)
|
|
|
+ }
|
|
|
+ bb, err := io.ReadAll(f)
|
|
|
+ if err != nil {
|
|
|
+ panic(err)
|
|
|
+ }
|
|
|
+ SendRetryMail(3, Config.UserMail, "站内每日热词统计", fmt.Sprintf("%d年%d月%d日热词统计", time.Now().Year(), int(time.Now().Month()), time.Now().Day()), filePath, bb, GmailAuth)
|
|
|
+}
|
|
|
+
|
|
|
+func SendRetryMail(retry int, userMail, subject, content, fName string, bt []byte, auth []*mail.GmailAuth) bool {
|
|
|
+ for i := 1; i <= retry; i++ {
|
|
|
+ for _, v := range auth { //使用多个邮箱尝试发送
|
|
|
+ if mail.GSendMail_B("剑鱼标讯", userMail, "", "", subject, content, fName, bt, v) {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ t := time.Duration(i) * 30 * time.Second
|
|
|
+ log.Println(userMail, fmt.Sprintf("第%d轮,使用%s发送邮件失败！%v后重试", i, v.User, t))
|
|
|
+ time.Sleep(t)
|
|
|
+ }
|
|
|
+ if i == retry {
|
|
|
+ log.Println(userMail, fmt.Sprintf("发送邮件失败"))
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return false
|
|
|
+}
|
