|
@@ -2,6 +2,7 @@ package util
|
|
|
|
|
|
import (
|
|
import (
|
|
"bytes"
|
|
"bytes"
|
|
|
|
+ "context"
|
|
"encoding/json"
|
|
"encoding/json"
|
|
"fmt"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"io/ioutil"
|
|
@@ -51,17 +52,27 @@ func LoadAllBehavior() map[string]*Behavior {
|
|
q += ` where userid in ('` + strings.Join(testIds, "','") + `')`
|
|
q += ` where userid in ('` + strings.Join(testIds, "','") + `')`
|
|
}
|
|
}
|
|
logger.Info("开始加载用户行为数据。。", q)
|
|
logger.Info("开始加载用户行为数据。。", q)
|
|
-
|
|
|
|
index := 0
|
|
index := 0
|
|
allBehavior := map[string]*Behavior{}
|
|
allBehavior := map[string]*Behavior{}
|
|
- Clickhouse.SelectByBath(1, func(l *[]map[string]interface{}) bool {
|
|
|
|
|
|
+ rows, err := Clickhouse.Query(context.Background(), q)
|
|
|
|
+ if err != nil {
|
|
|
|
+ logger.Error(err)
|
|
|
|
+ return allBehavior
|
|
|
|
+ }
|
|
|
|
+ for rows.Next() {
|
|
index++
|
|
index++
|
|
if index%500 == 0 {
|
|
if index%500 == 0 {
|
|
logger.Info("加载用户行为数据", index)
|
|
logger.Info("加载用户行为数据", index)
|
|
}
|
|
}
|
|
- searchfor, _ := (*l)[0]["searchfor"].(string)
|
|
|
|
- browse, _ := (*l)[0]["browse"].(string)
|
|
|
|
- userId, _ := (*l)[0]["userid"].(string)
|
|
|
|
|
|
+ var (
|
|
|
|
+ userId string
|
|
|
|
+ browse string
|
|
|
|
+ searchfor string
|
|
|
|
+ )
|
|
|
|
+ if err := rows.Scan(&userId, &browse, &searchfor); err != nil {
|
|
|
|
+ logger.Error(err)
|
|
|
|
+ continue
|
|
|
|
+ }
|
|
searchfors := []*Searchfor{}
|
|
searchfors := []*Searchfor{}
|
|
if searchfor != "" {
|
|
if searchfor != "" {
|
|
searchforsTemp := []*Searchfor{}
|
|
searchforsTemp := []*Searchfor{}
|
|
@@ -101,7 +112,7 @@ func LoadAllBehavior() map[string]*Behavior {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if len(searchfors) == 0 && len(browses) == 0 {
|
|
if len(searchfors) == 0 && len(browses) == 0 {
|
|
- return true
|
|
|
|
|
|
+ continue
|
|
}
|
|
}
|
|
area := ""
|
|
area := ""
|
|
if browses != nil && len(browses) > 0 {
|
|
if browses != nil && len(browses) > 0 {
|
|
@@ -112,40 +123,59 @@ func LoadAllBehavior() map[string]*Behavior {
|
|
Browses: browses,
|
|
Browses: browses,
|
|
Area: area,
|
|
Area: area,
|
|
}
|
|
}
|
|
- return true
|
|
|
|
- }, q)
|
|
|
|
|
|
+ }
|
|
|
|
+ rows.Close()
|
|
|
|
+ if err := rows.Err(); err != nil {
|
|
|
|
+ logger.Error(err)
|
|
|
|
+ }
|
|
logger.Info("加载用户行为数据结束。。", index)
|
|
logger.Info("加载用户行为数据结束。。", index)
|
|
return allBehavior
|
|
return allBehavior
|
|
}
|
|
}
|
|
|
|
|
|
//
|
|
//
|
|
-func LoadBehavior(userId string) ([]*Searchfor, []*Browse, string) {
|
|
|
|
- rules := Clickhouse.SelectBySql(`select browse,searchfor from jianyu.sub_recommend_rule where userid=? order by update_time desc limit 1`, userId)
|
|
|
|
- if rules == nil || len(*rules) == 0 {
|
|
|
|
- return nil, nil, ""
|
|
|
|
- }
|
|
|
|
- searchfor, _ := (*rules)[0]["searchfor"].(string)
|
|
|
|
- browse, _ := (*rules)[0]["browse"].(string)
|
|
|
|
- if searchfor == "" && browse == "" {
|
|
|
|
- return nil, nil, ""
|
|
|
|
|
|
+func LoadAllRecommend() map[string][]uint64 {
|
|
|
|
+ allRecommend := map[string][]uint64{}
|
|
|
|
+ q := `select userid,bitmapToArray(infoids) as infoids from jianyu.sub_recommend_list`
|
|
|
|
+ testIds := []string{}
|
|
|
|
+ if len(Config.TestIds) > 0 {
|
|
|
|
+ testIds = append(testIds, Config.TestIds...)
|
|
}
|
|
}
|
|
- var searchfors []*Searchfor
|
|
|
|
- if searchfor != "" {
|
|
|
|
- if err := json.Unmarshal([]byte(searchfor), &searchfors); err != nil {
|
|
|
|
- logger.Error(userId, err)
|
|
|
|
|
|
+ if len(Config.TestUserIds) > 0 {
|
|
|
|
+ for _, v := range Config.TestUserIds {
|
|
|
|
+ testIds = append(testIds, fmt.Sprint(v))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
- var browses []*Browse
|
|
|
|
- if browse != "" {
|
|
|
|
- if err := json.Unmarshal([]byte(browse), &browses); err != nil {
|
|
|
|
- logger.Error(userId, err)
|
|
|
|
|
|
+ if len(testIds) > 0 {
|
|
|
|
+ q += ` where userid in ('` + strings.Join(testIds, "','") + `')`
|
|
|
|
+ }
|
|
|
|
+ logger.Info("开始加载用户推荐数据。。", q)
|
|
|
|
+ index := 0
|
|
|
|
+ rows, err := Clickhouse.Query(context.Background(), q)
|
|
|
|
+ if err != nil {
|
|
|
|
+ logger.Error(err)
|
|
|
|
+ return nil
|
|
|
|
+ }
|
|
|
|
+ for rows.Next() {
|
|
|
|
+ index++
|
|
|
|
+ if index%500 == 0 {
|
|
|
|
+ logger.Info("加载用户推荐数据", index)
|
|
}
|
|
}
|
|
|
|
+ var (
|
|
|
|
+ userId string
|
|
|
|
+ infoids []uint64
|
|
|
|
+ )
|
|
|
|
+ if err := rows.Scan(&userId, &infoids); err != nil {
|
|
|
|
+ logger.Error(err)
|
|
|
|
+ continue
|
|
|
|
+ }
|
|
|
|
+ allRecommend[userId] = infoids
|
|
}
|
|
}
|
|
- area := ""
|
|
|
|
- if browses != nil && len(browses) > 0 {
|
|
|
|
- area = browses[0].Area
|
|
|
|
|
|
+ rows.Close()
|
|
|
|
+ if err := rows.Err(); err != nil {
|
|
|
|
+ logger.Error(err)
|
|
}
|
|
}
|
|
- return searchfors, browses, area
|
|
|
|
|
|
+ logger.Info("加载用户推荐数据结束。。", index)
|
|
|
|
+ return allRecommend
|
|
}
|
|
}
|
|
|
|
|
|
//tp 0:企业-商机管理 2:企业-超级订阅/大会员 3:企业-免费 -1:个人-免费 -2:个人-超级订阅 -3:个人-大会员
|
|
//tp 0:企业-商机管理 2:企业-超级订阅/大会员 3:企业-免费 -1:个人-免费 -2:个人-超级订阅 -3:个人-大会员
|
|
@@ -265,7 +295,6 @@ func proccessKey(a_key []interface{}) []map[string]interface{} {
|
|
|
|
|
|
//获取相似词
|
|
//获取相似词
|
|
func GetLikeWord(keys []string) []string {
|
|
func GetLikeWord(keys []string) []string {
|
|
- return keys
|
|
|
|
redisKey := fmt.Sprintf("subLikeKey_%s", strings.Join(keys, "+"))
|
|
redisKey := fmt.Sprintf("subLikeKey_%s", strings.Join(keys, "+"))
|
|
b, err := redis.GetNewBytes("pushcache_2_d", redisKey)
|
|
b, err := redis.GetNewBytes("pushcache_2_d", redisKey)
|
|
if err != nil {
|
|
if err != nil {
|