|
|
@@ -4,6 +4,7 @@ import (
|
|
|
"encoding/json"
|
|
|
"fmt"
|
|
|
"sort"
|
|
|
+ "strings"
|
|
|
. "subrecommend/config"
|
|
|
. "subrecommend/db"
|
|
|
"sync"
|
|
|
@@ -16,9 +17,6 @@ import (
|
|
|
. "bp.jydev.jianyu360.cn/BaseService/pushpkg/p"
|
|
|
)
|
|
|
|
|
|
-type OtherMatcher interface {
|
|
|
- Start()
|
|
|
-}
|
|
|
type Searchfor struct {
|
|
|
WinnerTel int //0：不限 1：有中标单位联系方式
|
|
|
SelectType string //搜索范围 detail,title,filetext,purchasing,projectname.pname,buyer.mbuyer
|
|
|
@@ -44,100 +42,89 @@ type Browse struct {
|
|
|
}
|
|
|
|
|
|
type MatchOther struct {
|
|
|
- IsEnt bool
|
|
|
- UserMap *map[*UserInfo]*SortList
|
|
|
- AllUsers map[*UserInfo]bool
|
|
|
- Datas *[]map[string]interface{}
|
|
|
+ UserMap *map[*UserInfo]*SortList
|
|
|
+ AllUsers map[*UserInfo]bool
|
|
|
+ Searchfors map[string][]*Searchfor
|
|
|
+ Browses map[string][]*Browse
|
|
|
+ Datas *[]map[string]interface{}
|
|
|
}
|
|
|
|
|
|
func (m *MatchOther) Match() {
|
|
|
m.MatchSearchfor()
|
|
|
+ m.MatchBrowse()
|
|
|
+}
|
|
|
+
|
|
|
+//
|
|
|
+func LoadBehavior(userId string) ([]*Searchfor, []*Browse, string) {
|
|
|
+ rules := Clickhouse.SelectBySql(`select browse,searchfor from jianyu.sub_recommend_rule where userid=? order by update_time desc limit 1`, userId)
|
|
|
+ if rules == nil || len(*rules) == 0 {
|
|
|
+ return nil, nil, ""
|
|
|
+ }
|
|
|
+ searchfor, _ := (*rules)[0]["searchfor"].(string)
|
|
|
+ browse, _ := (*rules)[0]["browse"].(string)
|
|
|
+ if searchfor == "" && browse == "" {
|
|
|
+ return nil, nil, ""
|
|
|
+ }
|
|
|
+ var searchfors []*Searchfor
|
|
|
+ if searchfor != "" {
|
|
|
+ if err := json.Unmarshal([]byte(searchfor), &searchfors); err != nil {
|
|
|
+ logger.Error(userId, err)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ var browses []*Browse
|
|
|
+ if browse != "" {
|
|
|
+ if err := json.Unmarshal([]byte(browse), &browses); err != nil {
|
|
|
+ logger.Error(userId, err)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ area := ""
|
|
|
+ if browses != nil && len(browses) > 0 {
|
|
|
+ area = browses[0].Area
|
|
|
+ }
|
|
|
+ return searchfors, browses, area
|
|
|
}
|
|
|
|
|
|
//
|
|
|
func (m *MatchOther) MatchSearchfor() {
|
|
|
onceUserMap := []*UserInfo{}
|
|
|
- browseCache := map[string][]*Browse{}
|
|
|
- pool := make(chan bool, Config.LoadUserPoolSize)
|
|
|
- wait := &sync.WaitGroup{}
|
|
|
- lock := &sync.Mutex{}
|
|
|
- for k, _ := range m.AllUsers {
|
|
|
- pool <- true
|
|
|
- wait.Add(1)
|
|
|
- go func(ui *UserInfo) {
|
|
|
- defer Catch()
|
|
|
- defer func() {
|
|
|
- <-pool
|
|
|
- wait.Done()
|
|
|
- }()
|
|
|
- if (*m.UserMap)[k] != nil && len(*(*m.UserMap)[k]) >= getMaxSize(ui) {
|
|
|
- return
|
|
|
- }
|
|
|
- rules := Clickhouse.SelectBySql(`select browse,searchfor from jianyu.sub_recommend_rule where userid=? order by update_time desc limit 1`, ui.Id)
|
|
|
- if rules == nil || len(*rules) == 0 {
|
|
|
- return
|
|
|
- }
|
|
|
- searchfor, _ := (*rules)[0]["searchfor"].(string)
|
|
|
- browse, _ := (*rules)[0]["browse"].(string)
|
|
|
- if searchfor == "" && browse == "" {
|
|
|
- return
|
|
|
- }
|
|
|
- var searchfors []*Searchfor
|
|
|
- if searchfor != "" {
|
|
|
- if err := json.Unmarshal([]byte(searchfor), &searchfors); err != nil {
|
|
|
- logger.Info(ui.Id, err)
|
|
|
- }
|
|
|
- }
|
|
|
- var browses []*Browse
|
|
|
- if browse != "" {
|
|
|
- if err := json.Unmarshal([]byte(browse), &browses); err != nil {
|
|
|
- logger.Info(ui.Id, err)
|
|
|
- }
|
|
|
- }
|
|
|
- if (searchfors == nil || len(searchfors) == 0) && (browses == nil || len(browses) == 0) {
|
|
|
- return
|
|
|
- }
|
|
|
- if browses != nil && len(browses) > 0 {
|
|
|
- browseCache[ui.Id] = browses
|
|
|
- }
|
|
|
- keys := []string{}
|
|
|
- for _, vv := range searchfors {
|
|
|
- datetime, err := time.ParseInLocation(Date_Full_Layout, vv.Datetime, time.Local)
|
|
|
- if err != nil {
|
|
|
- logger.Info(ui.Id, err)
|
|
|
- continue
|
|
|
- }
|
|
|
- if datetime.Before(time.Now().AddDate(0, 0, -Config.SearchLimitDay)) {
|
|
|
- continue
|
|
|
- }
|
|
|
- keys = append(keys, vv.Keys...)
|
|
|
+ for ui, _ := range m.AllUsers {
|
|
|
+ if (*m.UserMap)[ui] != nil && len(*(*m.UserMap)[ui]) >= getMaxSize(ui) {
|
|
|
+ continue
|
|
|
+ } else if m.Searchfors[ui.Id] == nil {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ keys := []string{}
|
|
|
+ for _, vv := range m.Searchfors[ui.Id] {
|
|
|
+ datetime, err := time.ParseInLocation(Date_Full_Layout, vv.Datetime, time.Local)
|
|
|
+ if err != nil {
|
|
|
+ logger.Info(ui.Id, err)
|
|
|
+ continue
|
|
|
}
|
|
|
- lock.Lock()
|
|
|
- defer lock.Unlock()
|
|
|
- if cnss := m.CreateNewUserInfo(ui, "", keys); cnss != nil {
|
|
|
- onceUserMap = append(onceUserMap, cnss)
|
|
|
+ if datetime.Before(time.Now().AddDate(0, 0, -Config.SearchLimitDay)) {
|
|
|
+ continue
|
|
|
}
|
|
|
- }(k)
|
|
|
+ keys = append(keys, vv.Keys...)
|
|
|
+ }
|
|
|
+ if cnss := m.CreateNewUserInfo(ui, keys); cnss != nil {
|
|
|
+ onceUserMap = append(onceUserMap, cnss)
|
|
|
+ }
|
|
|
}
|
|
|
- wait.Wait()
|
|
|
if len(onceUserMap) > 0 {
|
|
|
m.OnceBatchMatch(onceUserMap, "searchfor")
|
|
|
}
|
|
|
- m.MatchBrowse(browseCache)
|
|
|
}
|
|
|
|
|
|
//
|
|
|
-func (m *MatchOther) MatchBrowse(browseCache map[string][]*Browse) {
|
|
|
+func (m *MatchOther) MatchBrowse() {
|
|
|
onceUserMap := []*UserInfo{}
|
|
|
for ui, _ := range m.AllUsers {
|
|
|
if (*m.UserMap)[ui] != nil && len(*(*m.UserMap)[ui]) >= getMaxSize(ui) {
|
|
|
continue
|
|
|
- } else if browseCache[ui.Id] == nil {
|
|
|
+ } else if m.Browses[ui.Id] == nil {
|
|
|
continue
|
|
|
}
|
|
|
keys := []string{}
|
|
|
- area := ""
|
|
|
- for _, vv := range browseCache[ui.Id] {
|
|
|
+ for _, vv := range m.Browses[ui.Id] {
|
|
|
datetime, err := time.ParseInLocation(Date_Full_Layout, vv.Datetime, time.Local)
|
|
|
if err != nil {
|
|
|
logger.Info(ui.Id, err)
|
|
|
@@ -146,12 +133,9 @@ func (m *MatchOther) MatchBrowse(browseCache map[string][]*Browse) {
|
|
|
if datetime.Before(time.Now().AddDate(0, 0, -Config.BrowseLimitDay)) {
|
|
|
continue
|
|
|
}
|
|
|
- if len(keys) == 0 {
|
|
|
- area = vv.Area
|
|
|
- }
|
|
|
keys = append(keys, vv.Classify)
|
|
|
}
|
|
|
- if cnss := m.CreateNewUserInfo(ui, area, keys); cnss != nil {
|
|
|
+ if cnss := m.CreateNewUserInfo(ui, keys); cnss != nil {
|
|
|
onceUserMap = append(onceUserMap, cnss)
|
|
|
}
|
|
|
}
|
|
|
@@ -161,7 +145,7 @@ func (m *MatchOther) MatchBrowse(browseCache map[string][]*Browse) {
|
|
|
}
|
|
|
|
|
|
//
|
|
|
-func (m *MatchOther) CreateNewUserInfo(ui *UserInfo, area string, keys []string) *UserInfo {
|
|
|
+func (m *MatchOther) CreateNewUserInfo(ui *UserInfo, keys []string) *UserInfo {
|
|
|
if len(keys) == 0 {
|
|
|
return nil
|
|
|
}
|
|
|
@@ -172,20 +156,14 @@ func (m *MatchOther) CreateNewUserInfo(ui *UserInfo, area string, keys []string)
|
|
|
Keys: keys,
|
|
|
OriginalKeys: keys,
|
|
|
},
|
|
|
- Extend: &UserInfoExtend{
|
|
|
- Object: map[string]interface{}{
|
|
|
- "area": area,
|
|
|
- },
|
|
|
- },
|
|
|
+ Extend: ui.Extend,
|
|
|
+ Entniche: ui.Entniche,
|
|
|
}
|
|
|
if ui.SubSet != nil {
|
|
|
userInfo.SubSet.Area = ui.SubSet.Area
|
|
|
userInfo.SubSet.Areas = ui.SubSet.Areas
|
|
|
userInfo.SubSet.District = ui.SubSet.District
|
|
|
}
|
|
|
- if ui.Entniche != nil {
|
|
|
- userInfo.Entniche = ui.Entniche
|
|
|
- }
|
|
|
return userInfo
|
|
|
}
|
|
|
|
|
|
@@ -252,65 +230,93 @@ func (m *MatchOther) MergeInfos(userMap *map[*UserInfo]*SortList) {
|
|
|
}
|
|
|
|
|
|
//
|
|
|
-func Save(ui *UserInfo, infos *SortList) {
|
|
|
- // for _, vv := range *infos {
|
|
|
- // logger.Info(userId, "-----", vv.Info["title"], vv.Keys)
|
|
|
- // }
|
|
|
- area := ""
|
|
|
- if ui.Extend != nil && ui.Extend.Object != nil {
|
|
|
- area, _ = ui.Extend.Object["area"].(string)
|
|
|
- }
|
|
|
- maxSize := getMaxSize(ui)
|
|
|
- subCount, searchforCount, browseCount := 0, 0, 0
|
|
|
- sort.Sort(infos)
|
|
|
- matchLen := len(*infos)
|
|
|
- if len(*infos) > maxSize {
|
|
|
- *infos = (*infos)[:maxSize]
|
|
|
- }
|
|
|
- ids := []int{}
|
|
|
- for _, v := range *infos {
|
|
|
- ids = append(ids, IntAll(v.Info["autoid"]))
|
|
|
- if len(v.MatchWays) == 0 {
|
|
|
- } else if len(v.MatchWays) > 0 && v.MatchWays[0] == "searchfor" {
|
|
|
- searchforCount++
|
|
|
- } else if len(v.MatchWays) > 0 && v.MatchWays[0] == "browse" {
|
|
|
- browseCount++
|
|
|
- } else {
|
|
|
- subCount++
|
|
|
- }
|
|
|
- }
|
|
|
- logger.Info(ui.Id, "匹配上", matchLen, "条", "保存", len(*infos), "条", "订阅", subCount, "搜索", searchforCount, "浏览", browseCount)
|
|
|
- if len(ids) == 0 {
|
|
|
- return
|
|
|
- }
|
|
|
- nowFormat := NowFormat(Date_Full_Layout)
|
|
|
- datas := Clickhouse.SelectBySql(`select bitmapToArray(infoids) as infoids from jianyu.sub_recommend_list where userid=? order by update_time desc limit 1`, ui.Id)
|
|
|
- if datas == nil {
|
|
|
- logger.Error(ui.Id, "查询clickhouse失败")
|
|
|
- return
|
|
|
- }
|
|
|
- if len(*datas) == 0 {
|
|
|
- Clickhouse.InsertBySql(`INSERT INTO jianyu.sub_recommend_list (userid, infoids, area, update_time) values ('` + ui.Id + `',bitmapBuild([` + toUInt64(ids) + `]),'` + area + `','` + nowFormat + `')`)
|
|
|
- } else {
|
|
|
- infoids, _ := (*datas)[0]["infoids"].([]uint64)
|
|
|
- intIds := []int{}
|
|
|
- allInfoIds := map[int]bool{}
|
|
|
- for _, v := range infoids {
|
|
|
- intIds = append(intIds, int(v))
|
|
|
- allInfoIds[int(v)] = true
|
|
|
- }
|
|
|
- for _, v := range ids {
|
|
|
- if allInfoIds[v] {
|
|
|
- continue
|
|
|
+func Save(batchIndex int, userMap *map[*UserInfo]*SortList) {
|
|
|
+ logger.Info("第", batchIndex, "批开始保存。。。")
|
|
|
+ index := 0
|
|
|
+ //
|
|
|
+ savePool := make(chan bool, Config.SavePoolSize)
|
|
|
+ saveWaitGroup := &sync.WaitGroup{}
|
|
|
+ for k, v := range *userMap {
|
|
|
+ savePool <- true
|
|
|
+ saveWaitGroup.Add(1)
|
|
|
+ go func(ui *UserInfo, infos *SortList) {
|
|
|
+ defer Catch()
|
|
|
+ defer func() {
|
|
|
+ <-savePool
|
|
|
+ saveWaitGroup.Done()
|
|
|
+ }()
|
|
|
+ // for _, vv := range *infos {
|
|
|
+ // logger.Info(userId, "-----", vv.Info["title"], vv.Keys)
|
|
|
+ // }
|
|
|
+ area := ""
|
|
|
+ if ui.SubSet != nil {
|
|
|
+ area = strings.Join(ui.SubSet.Areas, " ")
|
|
|
+ if len([]rune(area)) > 20 {
|
|
|
+ area = string([]rune(area)[:20]) + "..."
|
|
|
+ }
|
|
|
}
|
|
|
- intIds = append(intIds, v)
|
|
|
- }
|
|
|
- if len(intIds) > maxSize {
|
|
|
- intIds = intIds[len(intIds)-maxSize:]
|
|
|
+ if area == "" {
|
|
|
+ area = "全国"
|
|
|
+ }
|
|
|
+ maxSize := getMaxSize(ui)
|
|
|
+ subCount, searchforCount, browseCount := 0, 0, 0
|
|
|
+ sort.Sort(infos)
|
|
|
+ matchLen := len(*infos)
|
|
|
+ if len(*infos) > maxSize {
|
|
|
+ *infos = (*infos)[:maxSize]
|
|
|
+ }
|
|
|
+ ids := []int{}
|
|
|
+ for _, v := range *infos {
|
|
|
+ ids = append(ids, IntAll(v.Info["autoid"]))
|
|
|
+ if len(v.MatchWays) == 0 {
|
|
|
+ } else if len(v.MatchWays) > 0 && v.MatchWays[0] == "searchfor" {
|
|
|
+ searchforCount++
|
|
|
+ } else if len(v.MatchWays) > 0 && v.MatchWays[0] == "browse" {
|
|
|
+ browseCount++
|
|
|
+ } else {
|
|
|
+ subCount++
|
|
|
+ }
|
|
|
+ }
|
|
|
+ logger.Info(ui.Id, "匹配上", matchLen, "条", "保存", len(*infos), "条", "订阅", subCount, "搜索", searchforCount, "浏览", browseCount)
|
|
|
+ if len(ids) == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ nowFormat := NowFormat(Date_Full_Layout)
|
|
|
+ datas := Clickhouse.SelectBySql(`select bitmapToArray(infoids) as infoids from jianyu.sub_recommend_list where userid=? order by update_time desc limit 1`, ui.Id)
|
|
|
+ if datas == nil {
|
|
|
+ logger.Error(ui.Id, "查询clickhouse失败")
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(*datas) == 0 {
|
|
|
+ Clickhouse.InsertBySql(`INSERT INTO jianyu.sub_recommend_list (userid, infoids, area, update_time) values ('` + ui.Id + `',bitmapBuild([` + toUInt64(ids) + `]),'` + area + `','` + nowFormat + `')`)
|
|
|
+ } else {
|
|
|
+ infoids, _ := (*datas)[0]["infoids"].([]uint64)
|
|
|
+ intIds := []int{}
|
|
|
+ allInfoIds := map[int]bool{}
|
|
|
+ for _, v := range infoids {
|
|
|
+ intIds = append(intIds, int(v))
|
|
|
+ allInfoIds[int(v)] = true
|
|
|
+ }
|
|
|
+ for _, v := range ids {
|
|
|
+ if allInfoIds[v] {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ intIds = append(intIds, v)
|
|
|
+ }
|
|
|
+ if len(intIds) > maxSize {
|
|
|
+ intIds = intIds[len(intIds)-maxSize:]
|
|
|
+ }
|
|
|
+ text := `ALTER TABLE jianyu.sub_recommend_list update area='` + area + `',infoids=bitmapBuild([` + toUInt64(intIds) + `]),update_time='` + nowFormat + `' where userid='` + ui.Id + `'`
|
|
|
+ Clickhouse.UpdateOrDeleteBySql(text)
|
|
|
+ }
|
|
|
+ }(k, v)
|
|
|
+ index++
|
|
|
+ if index%500 == 0 {
|
|
|
+ logger.Info("保存：", index)
|
|
|
}
|
|
|
- text := `ALTER TABLE jianyu.sub_recommend_list update area='` + area + `',infoids=bitmapBuild([` + toUInt64(intIds) + `]),update_time='` + nowFormat + `' where userid='` + ui.Id + `'`
|
|
|
- Clickhouse.UpdateOrDeleteBySql(text)
|
|
|
}
|
|
|
+ saveWaitGroup.Wait()
|
|
|
+ logger.Info("第", batchIndex, "批保存。。。", index)
|
|
|
}
|
|
|
|
|
|
//
|
