|
|
@@ -3,10 +3,10 @@ package timedTask
|
|
|
import (
|
|
|
qutil "app.yhyue.com/moapp/jybase/common"
|
|
|
"app.yhyue.com/moapp/jybase/mail"
|
|
|
- mg "app.yhyue.com/moapp/jybase/mongodb"
|
|
|
"encoding/json"
|
|
|
"fmt"
|
|
|
"github.com/tealeg/xlsx"
|
|
|
+ . "hostkword/config"
|
|
|
"hostkword/util"
|
|
|
"io"
|
|
|
"log"
|
|
|
@@ -18,41 +18,6 @@ import (
|
|
|
"unicode/utf8"
|
|
|
)
|
|
|
|
|
|
-type config struct {
|
|
|
- Mail []struct {
|
|
|
- Addr string `json:"addr"`
|
|
|
- Port int `json:"port"`
|
|
|
- Pwd string `json:"pwd"`
|
|
|
- User string `json:"user"`
|
|
|
- } `json:"mail"`
|
|
|
-
|
|
|
- UserTimeLimit int `json:"userTimeLimit"`
|
|
|
- EverydayWordLimit int `json:"everydayWordLimit"`
|
|
|
- Mongodb struct {
|
|
|
- Main struct {
|
|
|
- Address string `json:"address"`
|
|
|
- Size int `json:"size"`
|
|
|
- DbName string `json:"dbName"`
|
|
|
- } `json:"main"`
|
|
|
- Log struct {
|
|
|
- Address string `json:"address"`
|
|
|
- Size int `json:"size"`
|
|
|
- DbName string `json:"dbName"`
|
|
|
- UserName string `json:"userName"`
|
|
|
- Password string `json:"password"`
|
|
|
- } `json:"log"`
|
|
|
- } `json:"mongodb"`
|
|
|
- Redis struct {
|
|
|
- Main struct {
|
|
|
- Address string `json:"address"`
|
|
|
- } `json:"main"`
|
|
|
- } `json:"redis"`
|
|
|
- SubWordLimit int `json:"subWordLimit"`
|
|
|
- WordDuration int `json:"wordDuration"`
|
|
|
-
|
|
|
- UserMail string `json:"userMail"`
|
|
|
-}
|
|
|
-
|
|
|
type keyAllData struct {
|
|
|
AItems []struct {
|
|
|
AKey []struct {
|
|
|
@@ -63,50 +28,47 @@ type keyAllData struct {
|
|
|
} `json:"a_items"`
|
|
|
}
|
|
|
|
|
|
-var (
|
|
|
- Mgo *mg.MongodbSim
|
|
|
- MgoLog *mg.MongodbSim
|
|
|
+type UserKeyWord struct {
|
|
|
+ OMemberJy struct {
|
|
|
+ AItems []struct {
|
|
|
+ AKey []struct {
|
|
|
+ Key []string `json:"key"`
|
|
|
+ Appendkey []string `json:"appendkey"`
|
|
|
+ } `json:"a_key"`
|
|
|
+ } `json:"a_items"`
|
|
|
+ } `json:"o_member_jy"`
|
|
|
+ OVipjy struct {
|
|
|
+ AItems []struct {
|
|
|
+ AKey []struct {
|
|
|
+ Appendkey []string `json:"appendkey"`
|
|
|
+ Key []string `json:"key"`
|
|
|
+ } `json:"a_key"`
|
|
|
+ } `json:"a_items"`
|
|
|
+ } `json:"o_vipjy"`
|
|
|
+ OJy struct {
|
|
|
+ AKey []struct {
|
|
|
+ Appendkey []string `json:"appendkey"`
|
|
|
+ Key []string `json:"key"`
|
|
|
+ } `json:"a_key"`
|
|
|
+ } `json:"o_jy"`
|
|
|
+}
|
|
|
|
|
|
- Config *config
|
|
|
+var (
|
|
|
entWordNumber map[string]int
|
|
|
chineseRegex = regexp.MustCompile(`\p{Han}`)
|
|
|
+)
|
|
|
|
|
|
- GmailAuth []*mail.GmailAuth
|
|
|
+const (
|
|
|
+ filePath = "./hotWordFile/jyHotWord.xlsx"
|
|
|
)
|
|
|
|
|
|
func init() {
|
|
|
entWordNumber = make(map[string]int)
|
|
|
- qutil.ReadConfig(&Config)
|
|
|
- Mgo = &mg.MongodbSim{
|
|
|
- MongodbAddr: Config.Mongodb.Main.Address,
|
|
|
- Size: Config.Mongodb.Main.Size,
|
|
|
- DbName: Config.Mongodb.Main.DbName,
|
|
|
- UserName: "",
|
|
|
- Password: "",
|
|
|
- ReplSet: "",
|
|
|
- }
|
|
|
- MgoLog = &mg.MongodbSim{
|
|
|
- MongodbAddr: Config.Mongodb.Log.Address,
|
|
|
- Size: Config.Mongodb.Log.Size,
|
|
|
- DbName: Config.Mongodb.Log.DbName,
|
|
|
- UserName: Config.Mongodb.Log.UserName,
|
|
|
- Password: Config.Mongodb.Log.Password,
|
|
|
- ReplSet: "",
|
|
|
- }
|
|
|
- MgoLog.InitPool()
|
|
|
- Mgo.InitPool()
|
|
|
//每月用户订阅词
|
|
|
UserWord()
|
|
|
//每月搜索统计搜索词
|
|
|
SearchWordStatistics()
|
|
|
- for _, v := range Config.Mail {
|
|
|
- GmailAuth = append(GmailAuth, &mail.GmailAuth{
|
|
|
- SmtpHost: v.Addr,
|
|
|
- SmtpPort: v.Port,
|
|
|
- User: v.User,
|
|
|
- Pwd: v.Pwd,
|
|
|
- })
|
|
|
- }
|
|
|
+ //注册过滤词
|
|
|
}
|
|
|
|
|
|
func timeFmt() (int64, int64) {
|
|
|
@@ -123,17 +85,17 @@ func timeFmt() (int64, int64) {
|
|
|
// 每月1号统计新搜索词频
|
|
|
func SearchWordStatistics() {
|
|
|
tm := time.Now()
|
|
|
- startTime := tm.AddDate(0, -Config.WordDuration, 0)
|
|
|
- currentTime := startTime
|
|
|
- participleMap := make(map[string]int)
|
|
|
- glossaryMap := make(map[string]int)
|
|
|
if MgoLog.Count("jy_hot_word", map[string]interface{}{
|
|
|
"statistical_type": 1,
|
|
|
- "year": time.Now().Year(),
|
|
|
- "month": int(time.Now().Month()),
|
|
|
+ "year": tm.Year(),
|
|
|
+ "month": int(tm.Month()),
|
|
|
}) > 0 { //已统计过当月数据
|
|
|
return
|
|
|
- }
|
|
|
+ } // 避免本月重复统计
|
|
|
+ startTime := tm.AddDate(0, -Config.WordDuration, 0)
|
|
|
+ currentTime := startTime
|
|
|
+ participleMap := make(map[string]int)
|
|
|
+ glossaryMap := make(map[string]int)
|
|
|
pool := make(chan bool, 10)
|
|
|
wait := &sync.WaitGroup{}
|
|
|
var lock sync.Mutex
|
|
|
@@ -453,36 +415,15 @@ func UserMonthWord() {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-type UserKeyWord struct {
|
|
|
- OMemberJy struct {
|
|
|
- AItems []struct {
|
|
|
- AKey []struct {
|
|
|
- Key []string `json:"key"`
|
|
|
- Appendkey []string `json:"appendkey"`
|
|
|
- } `json:"a_key"`
|
|
|
- } `json:"a_items"`
|
|
|
- } `json:"o_member_jy"`
|
|
|
- OVipjy struct {
|
|
|
- AItems []struct {
|
|
|
- AKey []struct {
|
|
|
- Appendkey []string `json:"appendkey"`
|
|
|
- Key []string `json:"key"`
|
|
|
- } `json:"a_key"`
|
|
|
- } `json:"a_items"`
|
|
|
- } `json:"o_vipjy"`
|
|
|
- OJy struct {
|
|
|
- AKey []struct {
|
|
|
- Appendkey []string `json:"appendkey"`
|
|
|
- Key []string `json:"key"`
|
|
|
- } `json:"a_key"`
|
|
|
- } `json:"o_jy"`
|
|
|
-}
|
|
|
-
|
|
|
func UserWordAll() []map[string]interface{} {
|
|
|
+ _, lt := IsOneDay()
|
|
|
sess := Mgo.GetMgoConn()
|
|
|
defer Mgo.DestoryMongoConn(sess)
|
|
|
keyWordMap := make(map[string]int)
|
|
|
iter := sess.DB(Mgo.DbName).C("user").Find(map[string]interface{}{
|
|
|
+ "l_registedate": map[string]interface{}{
|
|
|
+ "$lt": lt.Unix(),
|
|
|
+ },
|
|
|
"$or": []map[string]interface{}{
|
|
|
{"o_jy.a_key": map[string]interface{}{"$exists": 1}},
|
|
|
{"o_vipjy.a_items": map[string]interface{}{"$exists": 1}},
|
|
|
@@ -542,13 +483,15 @@ func UserWord() {
|
|
|
userArr, entArr []string
|
|
|
userWordArr, entWordArr []map[string]int
|
|
|
)
|
|
|
-
|
|
|
dataMap := make(map[string]bool)
|
|
|
dataEntMap := make(map[string]bool)
|
|
|
UserMonthWord() //个人月词
|
|
|
EntWord() //企业月词
|
|
|
wordAll := UserWordAll() //个人总词
|
|
|
-
|
|
|
+ var userName, entName []string
|
|
|
+ if len(wordAll) > 0 {
|
|
|
+ userName = append(userName, "词频")
|
|
|
+ }
|
|
|
dMap := make(map[string]int)
|
|
|
for _, v := range wordAll { //个人订阅词频
|
|
|
keyWord := qutil.InterfaceToStr(v["keyWord"])
|
|
|
@@ -559,20 +502,25 @@ func UserWord() {
|
|
|
}
|
|
|
}
|
|
|
userWordArr = append(userWordArr, dMap)
|
|
|
+ var year, month int
|
|
|
+ switch time.Now().Month() {
|
|
|
+ case time.January: //一月统计往年全部数据
|
|
|
+ year = time.Now().Year() - 1
|
|
|
+ month = 13
|
|
|
+ default:
|
|
|
+ year = time.Now().Year()
|
|
|
+ month = int(time.Now().Month())
|
|
|
+ }
|
|
|
|
|
|
- tm := time.Now()
|
|
|
- var userStaMon, entStaMon int
|
|
|
- for i := 1; i < int(tm.Month()); i++ {
|
|
|
+ for i := 1; i < month; i++ {
|
|
|
data, ok := MgoLog.Find("jy_hot_word", map[string]interface{}{
|
|
|
- "year": tm.Year(),
|
|
|
+ "year": year,
|
|
|
"month": i,
|
|
|
"statistical_type": 5,
|
|
|
}, `{"number":-1}`, "", false, -1, -1)
|
|
|
if ok && len(*data) > 0 {
|
|
|
dMonMap := make(map[string]int)
|
|
|
- if userStaMon == 0 {
|
|
|
- userStaMon = i
|
|
|
- }
|
|
|
+ userName = append(userName, fmt.Sprintf("%d月", i))
|
|
|
for _, v := range *data {
|
|
|
keyWord := qutil.InterfaceToStr(v["keyWord"])
|
|
|
dMonMap[keyWord] = qutil.IntAll(v["number"])
|
|
|
@@ -584,14 +532,12 @@ func UserWord() {
|
|
|
userWordArr = append(userWordArr, dMonMap)
|
|
|
}
|
|
|
data, ok = MgoLog.Find("jy_hot_word", map[string]interface{}{
|
|
|
- "year": tm.Year(),
|
|
|
+ "year": year,
|
|
|
"month": i,
|
|
|
"statistical_type": 6,
|
|
|
}, `{"number":-1}`, "", false, -1, -1)
|
|
|
if ok && len(*data) > 0 {
|
|
|
- if entStaMon == 0 {
|
|
|
- entStaMon = i
|
|
|
- }
|
|
|
+ entName = append(entName, fmt.Sprintf("%d月", i))
|
|
|
dEntMap := make(map[string]int)
|
|
|
for _, v := range *data {
|
|
|
keyWord := qutil.InterfaceToStr(v["keyWord"])
|
|
|
@@ -609,7 +555,6 @@ func UserWord() {
|
|
|
file *xlsx.File
|
|
|
)
|
|
|
//生成xlex
|
|
|
- filePath := fmt.Sprintf("./hotWordFile/jyHotWord_%d%d.xlsx", time.Now().Year(), int(time.Now().Month()))
|
|
|
_, err := os.Stat(filePath)
|
|
|
sheetName1, sheetName2 := "个人订阅词", "企业订阅词"
|
|
|
if err == nil {
|
|
|
@@ -661,8 +606,8 @@ func UserWord() {
|
|
|
log.Println("文件校验失败:", err.Error())
|
|
|
return
|
|
|
}
|
|
|
- TableDataMonth(sheet1, userArr, userWordArr, false, userStaMon) //个人订阅词
|
|
|
- TableDataMonth(sheet2, entArr, entWordArr, true, entStaMon) //企业订阅词
|
|
|
+ TableDataMonth(sheet1, userArr, userWordArr, userName) //个人订阅词
|
|
|
+ TableDataMonth(sheet2, entArr, entWordArr, entName) //企业订阅词
|
|
|
// 保存 Excel 文件
|
|
|
err = file.Save(filePath)
|
|
|
if err != nil {
|
|
|
@@ -709,7 +654,6 @@ func WordXlsx() {
|
|
|
sheetName1, sheetName2 = "搜查词条", "搜索分词"
|
|
|
)
|
|
|
|
|
|
- filePath := fmt.Sprintf("./hotWordFile/jyHotWord_%d%d.xlsx", time.Now().Year(), int(time.Now().Month()))
|
|
|
_, err := os.Stat(filePath)
|
|
|
if err == nil {
|
|
|
log.Printf("文件 %s 存在\n", filePath)
|
|
|
@@ -784,16 +728,16 @@ func WordXlsx() {
|
|
|
log.Println("Excel 文件生成成功")
|
|
|
}
|
|
|
|
|
|
-func TableData(sheet *xlsx.Sheet, keyArr []string, data []map[string]int, staDay int) {
|
|
|
+func TableData(sheet *xlsx.Sheet, keyArr []string, data []map[string]int, staDay []string) {
|
|
|
// 设置表头
|
|
|
row := sheet.AddRow()
|
|
|
cell := row.AddCell()
|
|
|
cell.Value = "检索词"
|
|
|
cell = row.AddCell()
|
|
|
cell.Value = "词频"
|
|
|
- for i := staDay; i <= time.Now().Day(); i++ {
|
|
|
+ for _, v := range staDay {
|
|
|
cell = row.AddCell()
|
|
|
- cell.Value = fmt.Sprintf("%d号", i)
|
|
|
+ cell.Value = v
|
|
|
}
|
|
|
// 写入检索词和数据
|
|
|
for _, key := range keyArr {
|
|
|
@@ -807,18 +751,14 @@ func TableData(sheet *xlsx.Sheet, keyArr []string, data []map[string]int, staDay
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-func TableDataMonth(sheet *xlsx.Sheet, keyArr []string, data []map[string]int, isEnt bool, sta int) {
|
|
|
+func TableDataMonth(sheet *xlsx.Sheet, keyArr []string, data []map[string]int, name []string) {
|
|
|
// 设置表头
|
|
|
row := sheet.AddRow()
|
|
|
cell := row.AddCell()
|
|
|
cell.Value = "检索词"
|
|
|
- if !isEnt {
|
|
|
- cell = row.AddCell()
|
|
|
- cell.Value = "词频"
|
|
|
- }
|
|
|
- for i := sta; i < int(time.Now().Month()); i++ {
|
|
|
+ for _, v := range name {
|
|
|
cell = row.AddCell()
|
|
|
- cell.Value = fmt.Sprintf("%d月", i)
|
|
|
+ cell.Value = v
|
|
|
}
|
|
|
// 写入检索词和数据
|
|
|
for _, key := range keyArr {
|
|
|
@@ -832,17 +772,20 @@ func TableDataMonth(sheet *xlsx.Sheet, keyArr []string, data []map[string]int, i
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-func wordXlsxData(isParticiple bool) (strArr []string, statisticsMap []map[string]int, staDay int) {
|
|
|
+func wordXlsxData(isParticiple bool) (strArr []string, statisticsMap []map[string]int, staDay []string) {
|
|
|
dataMap := make(map[string]bool)
|
|
|
tm := time.Now()
|
|
|
- for i := 0; i <= tm.Day(); i++ {
|
|
|
+ statistical1 := qutil.If(isParticiple, 1, 2)
|
|
|
+ statistical2 := qutil.If(isParticiple, 3, 4)
|
|
|
+
|
|
|
+ for i := 0; i < tm.Day(); i++ {
|
|
|
query := make(map[string]interface{})
|
|
|
if i == 0 { //首次统计词频
|
|
|
- query["statistical_type"] = qutil.If(isParticiple, 1, 2)
|
|
|
+ query["statistical_type"] = statistical1
|
|
|
query["year"] = tm.Year()
|
|
|
query["month"] = int(tm.Month())
|
|
|
} else {
|
|
|
- query["statistical_type"] = qutil.If(isParticiple, 3, 4)
|
|
|
+ query["statistical_type"] = statistical2
|
|
|
query["year"] = tm.Year()
|
|
|
query["month"] = int(tm.Month())
|
|
|
query["day"] = i
|
|
|
@@ -850,8 +793,8 @@ func wordXlsxData(isParticiple bool) (strArr []string, statisticsMap []map[strin
|
|
|
data, ok := MgoLog.Find("jy_hot_word", query, `{"number":-1}`, "", false, -1, -1)
|
|
|
if ok && len(*data) > 0 {
|
|
|
dMap := make(map[string]int)
|
|
|
- if staDay == 0 && i > 0 {
|
|
|
- staDay = i
|
|
|
+ if i > 0 {
|
|
|
+ staDay = append(staDay, fmt.Sprintf("%d号", i))
|
|
|
}
|
|
|
for _, v := range *data {
|
|
|
keyWord := qutil.InterfaceToStr(v["keyWord"])
|
|
|
@@ -870,11 +813,10 @@ func wordXlsxData(isParticiple bool) (strArr []string, statisticsMap []map[strin
|
|
|
// 过滤不符合条件的词
|
|
|
func HasChineseAndValidLength(str string) bool {
|
|
|
length := utf8.RuneCountInString(str)
|
|
|
- return chineseRegex.MatchString(str) && length >= 2 && length < 20
|
|
|
+ return chineseRegex.MatchString(str) && length >= 2 && length < 20 && !util.FilterKey.Search(str)
|
|
|
}
|
|
|
|
|
|
func SendMail() {
|
|
|
- filePath := fmt.Sprintf("./hotWordFile/jyHotWord_%d%d.xlsx", time.Now().Year(), int(time.Now().Month()))
|
|
|
f, err := os.Open(filePath)
|
|
|
if err != nil {
|
|
|
panic(err)
|
