|
|
@@ -42,7 +42,7 @@ type Bidding struct {
|
|
|
Toptype string
|
|
|
}
|
|
|
|
|
|
-//2. 已匹配上的关键词，同步到crm.topnet_key表，每次全量加载crm.topnet_key表数据到内存，进行去重即可
|
|
|
+// 2. 已匹配上的关键词，同步到crm.topnet_key表，每次全量加载crm.topnet_key表数据到内存，进行去重即可
|
|
|
func InitTopKey() map[string]bool {
|
|
|
m := map[string]bool{}
|
|
|
Tidb.SelectByBath(200, func(l *[]map[string]interface{}) bool {
|
|
|
@@ -229,8 +229,163 @@ func GetUserMailData() []*Bidding {
|
|
|
log.Println("加载usermail结束。。", ct)
|
|
|
return biddings
|
|
|
}
|
|
|
+func GetUserMailEmployData() []*Bidding {
|
|
|
+ biddings := []*Bidding{}
|
|
|
+ log.Println("开始加载usermail数据Employ")
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "appid": gconv.String(Config.AppUser["appId"]),
|
|
|
+ }
|
|
|
+ layout := "2006-01-02 15:04"
|
|
|
+ nowt := time.Now()
|
|
|
+ start, _ := time.ParseInLocation(layout, nowt.Add(-time.Duration(Config.RuleTime)*time.Minute).Format(layout), time.Local)
|
|
|
+ end, _ := time.ParseInLocation(layout, nowt.Format(layout), time.Local)
|
|
|
+ log.Println(start, end)
|
|
|
+ sid := bson.NewObjectIdWithTime(start)
|
|
|
+ eid := bson.NewObjectIdWithTime(end)
|
|
|
+ s_id := mongodb.StringTOBsonId(util.SpitObjectId(sid.String()))
|
|
|
+ e_id := mongodb.StringTOBsonId(util.SpitObjectId(eid.String()))
|
|
|
+ log.Println(s_id, e_id)
|
|
|
+ query["_id"] = map[string]interface{}{
|
|
|
+ "$gte": s_id,
|
|
|
+ "$lt": e_id,
|
|
|
+ }
|
|
|
+ //历史
|
|
|
+ /*query = map[string]interface{}{
|
|
|
+ "appid": gconv.String(Config.AppUser["appId"]),
|
|
|
+ }*/
|
|
|
+ log.Println("query:", query)
|
|
|
+ session := Mgo_Datatag.GetMgoConn()
|
|
|
+ defer Mgo_Datatag.DestoryMongoConn(session)
|
|
|
+ iter := session.DB(Config.Mongodb.Qyfw.DbName).C("usermail").Find(query).Iter()
|
|
|
+ ct := 0
|
|
|
+ for thisData := map[string]interface{}{}; iter.Next(&thisData); {
|
|
|
+ ct++
|
|
|
+ if ct%2000 == 0 {
|
|
|
+ log.Println("已加载Employ", ct)
|
|
|
+ }
|
|
|
+ _id := mongodb.BsonIdToSId(thisData["_id"])
|
|
|
+ bId := gconv.String(thisData["id"])
|
|
|
+ title := gconv.String(thisData["title"])
|
|
|
+ area := gconv.String(thisData["area"])
|
|
|
+ city := gconv.String(thisData["city"])
|
|
|
+ subtype := gconv.String(thisData["subtype"])
|
|
|
+ toptype := gconv.String(thisData["toptype"])
|
|
|
+ buyerclass := gconv.String(thisData["buyerclass"])
|
|
|
+ budget := gconv.Float64(thisData["budget"])
|
|
|
+ bidamount := gconv.Float64(thisData["bidamount"])
|
|
|
+ rulename := gconv.String(thisData["rulename"])
|
|
|
+ matchkey := gconv.String(thisData["matchkey"])
|
|
|
+ departname := gconv.String(thisData["departname"])
|
|
|
+ procurementlist := gconv.Maps(thisData["procurementlist"])
|
|
|
+ smallestTime := int64(0)
|
|
|
+ for _, v := range procurementlist {
|
|
|
+ expurasingtime := gconv.Int64(v["expurasingtime"])
|
|
|
+ if smallestTime < expurasingtime {
|
|
|
+ smallestTime = expurasingtime
|
|
|
+ }
|
|
|
+ }
|
|
|
+ expurasingtime := ""
|
|
|
+ if smallestTime > 0 {
|
|
|
+
|
|
|
+ // expurasingtime = time.Unix(smallestTime, 0).Format(date.Date_yyyyMMdd)
|
|
|
+ expurasingtime = util.FormatDate(smallestTime)
|
|
|
+ }
|
|
|
+ winner := gconv.String(thisData["winner"])
|
|
|
+ buyer := gconv.String(thisData["buyer"])
|
|
|
+
|
|
|
+ annex := 0
|
|
|
+ projectinfo := gconv.Map(thisData["projectinfo"])
|
|
|
+ attachments := gconv.Map(projectinfo["attachments"])
|
|
|
+ if len(attachments) > 0 {
|
|
|
+ annex = 1
|
|
|
+ }
|
|
|
+ publishtime := time.Unix(gconv.Int64(thisData["publishtime"]), 0).Format(date.Date_Full_Layout)
|
|
|
+ show_time := time.Unix(gconv.Int64(thisData["publishtime"]), 0).Format(date.Date_Full_Layout)
|
|
|
+ //有城市存城市的代码，没有城市存省份的代码
|
|
|
+ areacode := ""
|
|
|
+ if area != "" {
|
|
|
+ areacode = AreaCodeMap[area]
|
|
|
+ }
|
|
|
+ if city != "" {
|
|
|
+ areacode = AreaCodeMap[city]
|
|
|
+ }
|
|
|
+ biddings = append(biddings, &Bidding{
|
|
|
+ BId: bId,
|
|
|
+ Title: title,
|
|
|
+ Area: area,
|
|
|
+ City: city,
|
|
|
+ Subtype: subtype,
|
|
|
+ Buyerclass: buyerclass,
|
|
|
+ Budget: budget,
|
|
|
+ Bidamount: bidamount,
|
|
|
+ Expurasingtime: expurasingtime,
|
|
|
+ Winner: winner,
|
|
|
+ Buyer: buyer,
|
|
|
+ Annex: annex,
|
|
|
+ Publishtime: publishtime,
|
|
|
+ Show_time: show_time,
|
|
|
+ RuleName: rulename,
|
|
|
+ MatchKey: matchkey,
|
|
|
+ Departname: departname,
|
|
|
+ AreaCode: areacode,
|
|
|
+ Id: _id,
|
|
|
+ Toptype: toptype,
|
|
|
+ })
|
|
|
+ }
|
|
|
+ log.Println("加载usermail结束Employ。。", ct)
|
|
|
+ return biddings
|
|
|
+}
|
|
|
+
|
|
|
+// 某些用户的usermail进employ_info
|
|
|
+func UsermailInsertEmploy() {
|
|
|
+ //加载usermail数据
|
|
|
+ biddings := GetUserMailEmployData()
|
|
|
+ log.Println("Employ加载数量：", len(biddings))
|
|
|
+ if len(biddings) == 0 {
|
|
|
+ log.Println("Employ本轮无数据")
|
|
|
+ return
|
|
|
+ }
|
|
|
+ fieids := []string{"position_id", "ent_id", "ent_dept_id", "ent_user_id", "type", "source", "source_id", "employ_way", "status", "title", "area", "city", "subtype", "buyerclass", "budget", "bidamount", "expurasing_time", "jybx_url", "winner", "buyer", "annex", "publishtime", "show_time", "create_time", "data_type", "khglxt_industry", "khglxt_matchkey", "khglxt_rulename", "area_code", "toptype"}
|
|
|
+ args := []interface{}{}
|
|
|
+ count := 0
|
|
|
+ for _, vv := range biddings {
|
|
|
+ args = append(args, gconv.Int64(Config.AppUser["positionId"]), gconv.Int64(Config.AppUser["entId"]), gconv.Int64(Config.AppUser["deptId"]), gconv.Int64(Config.AppUser["entUserId"]),
|
|
|
+ 1, 1, vv.BId, 3, 1, vv.Title, qu.If(vv.Area == "", nil, vv.Area), qu.If(vv.City == "", nil, vv.City),
|
|
|
+ qu.If(vv.Subtype == "", nil, vv.Subtype),
|
|
|
+ qu.If(vv.Buyerclass == "", nil, vv.Buyerclass),
|
|
|
+ qu.If(vv.Budget <= 0, nil, vv.Budget),
|
|
|
+ qu.If(vv.Bidamount <= 0, nil, vv.Bidamount),
|
|
|
+ qu.If(vv.Expurasingtime == "", nil, vv.Expurasingtime),
|
|
|
+ fmt.Sprintf("/article/content/%s.html", encrypt.CommonEncodeArticle("content", vv.BId)),
|
|
|
+ qu.If(vv.Winner == "", nil, vv.Winner),
|
|
|
+ qu.If(vv.Buyer == "", nil, vv.Buyer),
|
|
|
+ vv.Annex,
|
|
|
+ vv.Publishtime,
|
|
|
+ vv.Show_time,
|
|
|
+ date.NowFormat(date.Date_Full_Layout),
|
|
|
+ 0,
|
|
|
+ vv.Departname,
|
|
|
+ vv.MatchKey,
|
|
|
+ vv.RuleName,
|
|
|
+ vv.AreaCode,
|
|
|
+ qu.If(vv.Toptype == "", nil, vv.Toptype),
|
|
|
+ )
|
|
|
+ count++
|
|
|
+ if count%500 == 0 {
|
|
|
+ if len(args) > 0 {
|
|
|
+ Tidb.InsertIgnoreBatch("employ_info", fieids, args)
|
|
|
+ args = []interface{}{}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(args) > 0 {
|
|
|
+ Tidb.InsertIgnoreBatch("employ_info", fieids, args)
|
|
|
+ args = []interface{}{}
|
|
|
+ }
|
|
|
+ log.Println("本轮userMailEmploy已经处理完毕")
|
|
|
+}
|
|
|
|
|
|
-//usermail迁移tidb
|
|
|
+// usermail迁移tidb
|
|
|
func UsermailETL() {
|
|
|
//加载规则
|
|
|
AccountRuleMap, deptUser, _ := ReadAccountExcel()
|
|
|
@@ -510,9 +665,9 @@ type AccountRule struct {
|
|
|
Platform string //平台
|
|
|
}
|
|
|
|
|
|
-//ReadAccountExcel 获取拓普账号规则
|
|
|
-//省份、城市、业务领域关键词，先支持这三个吧
|
|
|
-//return 用户规则及分发规则，部门用户，部门规则
|
|
|
+// ReadAccountExcel 获取拓普账号规则
|
|
|
+// 省份、城市、业务领域关键词，先支持这三个吧
|
|
|
+// return 用户规则及分发规则，部门用户，部门规则
|
|
|
func ReadAccountExcel() (map[string]*AccountRule, map[string][]string, map[string]map[string]bool) {
|
|
|
//获取规则
|
|
|
AccountRuleMap := map[string]*AccountRule{}
|
|
|
@@ -622,7 +777,7 @@ func ReadAccountExcel() (map[string]*AccountRule, map[string][]string, map[strin
|
|
|
return AccountRuleMap, deptUser, deptRule
|
|
|
}
|
|
|
|
|
|
-//历史
|
|
|
+// 历史
|
|
|
func GetUserMailDataHistroy() []*Bidding {
|
|
|
keyMap := InitTopKey()
|
|
|
|
WH01243