feat: 行为采集

telemarketingEtl/config.yaml

@@ -6,6 +6,14 @@ tidb:
     password: '=PDT49#80Z!RVv52_z'
     maxOpenConns: 80
     maxIdleConns: 5
+mysql:
+  main:
+    dbName: jianyu
+    address: 192.168.3.149:3306
+    userName: root
+    password: 'Topnet123'
+    maxOpenConns: 80
+    maxIdleConns: 5
 mongo:
   main:
     dbName: qfw
@@ -47,15 +55,15 @@ regex:
   - "^/jyapp/big/page/unit_portrayal:appBuyPortrait" #app采购单位画像
   - "^/jyapp/big/page/ent_portrait:appEntPortrait" #app企业画像
   - "^(/swordfish/page_big_pc/free/project_progress)|(/page_big_pc/pro_follow_detail):pcProjectMonitor" #pc项目进度监控
-  - "^(/swordfish/page_big_pc/free/ent_follow)|(/swordfish/page_big_pc/ent_portrait):pcEntIntelligentceMonitor" #pc企业情报监控/jyapp/big/page/ent_portrait
-  - "^/swordfish/page_big_pc/my_client:pcClient"#pc-客户监控
-  - "^/jyapp/big/page/client_list:appClient"#app-客户监控
-  - "^/swordfish/page_big_pc/forecast_list:pcForecast"#pc-潜在项目预测
-  - "^/jyapp/big/page/forecast_list:appForecast"#app-潜在项目预测列表页
+  - "^/swordfish/page_big_pc/ent_portrait:pcEntIntelligentceMonitor" #pc企业情报监控/jyapp/big/page/ent_portrait
+  - "^/swordfish/page_big_pc/unit_portrayal:pcClient"#pc-客户监控
+  - "^/jyapp/big/page/client_portrayal:appClient"#app-客户监控
+  - "^/swordfish/page_big_pc/analysis_filter:pcForecast"#pc-潜在项目预测
+  - "^/jyapp/big/page/forecast_detail:appForecast"#app-潜在项目预测列表页
   - "^/swordfish/frontPage/collection/sess/index:pcCollection"#pc-标讯收藏
   - "^/jyapp/frontPage/collection/sess/index:appCollection"#app-标讯收藏
   - "^/weixin/frontPage/collection/sess/index:wxCollection"#wx-标讯收藏
-  - "^/product/index?serviceType=0:productIndex"#pc-会员介绍页面
+  - "^/product/index:productIndex"#pc-会员介绍页面
 classOneHighSeas_A: 3  #一级公海A 3天内点击过剑鱼付费产品介绍页...
 classOneHighSeas_B: 30 #一级公海B 购买了超级订阅产品剩余使用时长不足30天的客户；
 classTwoHighSeaslastDay: 30 #二级公海 最近30天内活跃天数
@@ -105,4 +113,10 @@ intentionCustomer: 30
 #潜在客户
 latentCustomer: 60
 #沉睡客户
-sleepCustomer: 90
+sleepCustomer: 90
+#商机线索
+businessLeads: 2
+#回收站A.5个自然日内被销售人员手动退回公海的客户；
+handReturn: 5
+#回收站B.3个自然日内有过“已接听”的通话记录且仍处于“商机线索”状态下的客户。
+recycleB: 3

telemarketingEtl/config/config.go

@@ -1,16 +1,18 @@
 package config
 
 import (
+	"log"
+
 	mg "app.yhyue.com/moapp/jybase/mongodb"
 	"app.yhyue.com/moapp/jybase/mysql"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/os/gcfg"
 	"github.com/gogf/gf/v2/os/gctx"
-	"log"
 )
 
 var (
 	JianyuSubjectdb *mysql.Mysql
+	JianyuMaindb    *mysql.Mysql
 	Mgo             mg.MongodbSim
 	MgoLog          mg.MongodbSim
 	MgoBid          mg.MongodbSim
@@ -33,6 +35,18 @@ func init() {
 		}
 		JianyuSubjectdb.Init()
 	}
+	if g.Cfg().MustGet(ctx, "mysql.main.dbName").String() != "" {
+		log.Println("初始化tidb")
+		JianyuMaindb = &mysql.Mysql{
+			Address:      g.Cfg().MustGet(ctx, "mysql.main.address").String(),
+			UserName:     g.Cfg().MustGet(ctx, "mysql.main.userName").String(),
+			PassWord:     g.Cfg().MustGet(ctx, "mysql.main.password").String(),
+			DBName:       g.Cfg().MustGet(ctx, "mysql.main.dbName").String(),
+			MaxOpenConns: g.Cfg().MustGet(ctx, "mysql.main.maxOpenConns").Int(),
+			MaxIdleConns: g.Cfg().MustGet(ctx, "mysql.main.maxIdleConns").Int(),
+		}
+		JianyuMaindb.Init()
+	}
 	//
 	if g.Cfg().MustGet(ctx, "mongo.main.dbName").String() != "" {
 		log.Println("初始化 mongodb main")

telemarketingEtl/entity/dwd_f_crm_open_sea.go

@@ -5,7 +5,6 @@ import (
 	"log"
 	"strings"
 	"telemarketingEtl/config"
-	"telemarketingEtl/util"
 	"time"
 
 	"app.yhyue.com/moapp/jybase/date"
@@ -130,18 +129,17 @@ func GetOpenSea() {
 		if userid == "" {
 			continue
 		}
-		//根据userid获取线索id
-		if oneClassA[userid] {
+		clubId := GetClueIdByUserId(userid)
+		if clubId == "" {
 			continue
 		}
-		uuid := GetClueIdByUserId(userid)
-		if uuid == "" {
+		//根据userid获取线索id
+		if oneClassA[clubId] {
 			continue
 		}
-		oneClassA[uuid] = true
-
+		oneClassA[clubId] = true
 		count++
-		if count%10 == 0 {
+		if count%100 == 0 {
 			log.Printf("已完成%d条数据\n", count)
 		}
 		thisData = map[string]interface{}{}
@@ -186,14 +184,14 @@ func GetOpenSea() {
 		if userid == "" {
 			continue
 		}
-		//根据userid获取线索id
-		if oneClassA[userid] {
-			continue
-		}
 		clubId := GetClueIdByUserId(userid)
 		if clubId == "" {
 			continue
 		}
+		//根据userid获取线索id
+		if oneClassA[clubId] {
+			continue
+		}
 		oneClassA[clubId] = true
 
 		count++
@@ -204,12 +202,12 @@ func GetOpenSea() {
 	}
 	//更新公海
 	AddOpenSea(oneClassA, 1, "A")
-	log.Println("一级公海 更新结束。")
 	//
 	oneClassB := map[string]bool{}
+	vipendtime := t.AddDate(0, 0, -g.Cfg().MustGet(ctx, "classOneHighSeas_B").Int())
 	classB, ok := config.Mgo.Find("user", map[string]interface{}{
 		"l_vip_endtime": map[string]interface{}{
-			"$gte": t.AddDate(0, 0, -g.Cfg().MustGet(ctx, "classOneHighSeas_B").Int()),
+			"$lt": vipendtime.Unix(),
 		},
 		"i_vip_status": map[string]interface{}{
 			"$gt": 0,
@@ -219,16 +217,17 @@ func GetOpenSea() {
 	if classB != nil && ok && len(*classB) > 0 {
 		for _, v := range *classB {
 			userid := mongodb.BsonIdToSId(v["_id"])
-			if oneClassA[userid] {
+			clubId := GetClueIdByUserId(userid)
+			if oneClassA[clubId] {
 				continue
 			}
-			clubId := GetClueIdByUserId(userid)
 			oneClassB[clubId] = true
 		}
 	}
 	AddOpenSea(oneClassB, 1, "B")
-	oneClassC := GetOneSeaC()
+	oneClassC := GetOneSeaC(oneClassA, oneClassB)
 	AddOpenSea(oneClassC, 1, "C")
+	log.Println("一级公海 更新结束。")
 	//
 	twoA, twoB, twoC, twoD := TwoOpenSea(oneClassA, oneClassB, oneClassC)
 	AddOpenSea(twoA, 2, "A")
@@ -237,7 +236,9 @@ func GetOpenSea() {
 	AddOpenSea(twoD, 2, "D")
 	//三级公海
 	ThreeOpenSea(oneClassA, oneClassB, oneClassC, twoA, twoB, twoC, twoD)
+	// AddOpenSea(three, 3, "D")
 	//
+	Recycle()
 	log.Println("end")
 }
 
@@ -298,6 +299,7 @@ func TwoOpenSea(oneClassA, oneClassB, oneClassC map[string]bool) (aMap, bMap, cM
 		if ct < 5 && ct >= 1 {
 			dMap[clubId] = true
 		}
+
 	}
 	// Check for errors during iteration
 	err = rows.Err()
@@ -309,38 +311,37 @@ func TwoOpenSea(oneClassA, oneClassB, oneClassC map[string]bool) (aMap, bMap, cM
 
 // 三级公海：
 // 最近30天内活跃天数=0天的客户。
-func ThreeOpenSea(oneA, oneB, oneC, twoA, twoB, twoC, twoD map[string]bool) {
+func ThreeOpenSea(oneA, oneB, oneC, twoA, twoB, twoC, twoD map[string]bool) map[string]bool {
+	m := map[string]bool{}
 	ctx := gctx.New()
 	t := time.Now()
 	t = time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local)
 	day := g.Cfg().MustGet(ctx, "classThreeHighSeaslastDay").Int()
 	t = t.AddDate(0, 0, -day)
-	start := util.GetObjectId(t.Unix())
-	log.Println(t.Format(date.Date_Full_Layout), "~~")
 
 	config.JianyuSubjectdb.SelectByBath(500, func(l *[]map[string]interface{}) bool {
+		ids := []interface{}{}
+
 		for _, v := range *l {
+			clubId := gconv.String(v["clue_id"])
 			userid := gconv.String(v["userid"])
-			query := map[string]interface{}{
-				"userid": userid,
-				"_id": map[string]interface{}{
-					"$gte": start,
-				},
-			}
-			id := gconv.Int64(v["id"])
-			clubId := gconv.String(v["id"])
-			if r, ok := config.MgoLog.Find("subscribepay_logs", query, nil, `{"_id":1}`, false, 0, 1); ok && r != nil && len(*r) == 0 {
-				if oneA[clubId] || oneB[clubId] || oneC[clubId] || twoA[clubId] || twoB[clubId] || twoC[clubId] || twoD[clubId] {
-					continue
-				}
-				config.JianyuSubjectdb.Update("dwd_f_crm_open_sea", map[string]interface{}{"clue_id": id}, map[string]interface{}{
-					"LEVEL":      3,
-					"clue_level": "D",
-				})
+			q := fmt.Sprintf("SELECT count(1)  FROM dwd_f_userbase_visit_info WHERE  createtime > '%s' AND contentnum =0 and userid = '%s'", t.Format(date.Date_Full_Layout), userid)
+			config.JianyuSubjectdb.SelectBySql(q)
+			if oneA[clubId] || oneB[clubId] || oneC[clubId] || twoA[clubId] || twoB[clubId] || twoC[clubId] || twoD[clubId] {
+				continue
 			}
+
+			ids = append(ids, clubId)
+		}
+		whs := []string{}
+		for i := 0; i < len(ids); i++ {
+			whs = append(whs, "?")
 		}
+		wh := strings.Join(whs, ",")
+		config.JianyuSubjectdb.UpdateOrDeleteBySql(`UPDATE dwd_f_crm_open_sea SET level = 3,clue_level='D' WHERE clue_id in (`+wh+`)`, ids...)
 		return true
-	}, `select userid,uid,id from dwd_f_crm_clue_info where is_assign !=1`)
+	}, `SELECT a.clue_id,b.userid FROM dwd_f_crm_open_sea a LEFT JOIN dwd_f_crm_clue_info b ON a.clue_id=b.id`)
+	return m
 }
 
 // 根据mongodb userid 获取 线索id
@@ -361,8 +362,8 @@ func GetClueIdByUserId(userid string) (uuid string) {
 func AddOpenSea(m map[string]bool, level int, clue_level string) {
 	// createtime := time.Now().Format(date.Date_Full_Layout)
 	if len(m) > 0 {
-		for _, v := range m {
-			config.JianyuSubjectdb.Update("dwd_f_crm_open_sea", map[string]interface{}{"clue_id": v}, map[string]interface{}{
+		for k, _ := range m {
+			config.JianyuSubjectdb.Update("dwd_f_crm_open_sea", map[string]interface{}{"clue_id": k}, map[string]interface{}{
 				"clue_level": clue_level,
 				"LEVEL":      level,
 			})
@@ -433,16 +434,21 @@ func ReturnOpenSea() {
 		"01": t.AddDate(0, 0, -businessLeads),
 		"00": t,
 	} {
-		sql := `SELECT MAX(c.next_time) nexttime, b.id FROM dwd_f_crm_private_sea a 
+		sql := `SELECT a.clue_id,a.position_id,a.seatNumber FROM dwd_f_crm_private_sea a 
 				LEFT JOIN dwd_f_crm_clue_info b ON a.clue_id=b.id 
-				LEFT JOIN  dwd_f_crm_trail_content c ON b.id =c.clue_id 
 				WHERE b.trailstatus =?`
 		argsSelect := []interface{}{trailstatus}
+
+		intime := ""
 		if trailstatus != "00" {
-			sql += " AND c.createtime <? "
-			argsSelect = append(argsSelect, nexttime)
+			sql += " AND a.comeintime <?"
+			nt, _ := nexttime.(time.Time)
+			intime := nt.Format(date.Date_Full_Layout)
+			argsSelect = append(argsSelect, intime)
 		}
-		sql += " GROUP BY  b.id"
+		log.Println(sql)
+		log.Println(argsSelect)
+		//
 		config.JianyuSubjectdb.SelectByBath(500, func(l *[]map[string]interface{}) bool {
 			//1新增 2私海手动退回 3私海高意向客户自动退回 4私海意向客户退回 5私海潜在客户退回 6私海沉睡客户退回 7私海商机线索退回 8私海无意向客户退回
 			comeinsource := GetComeSource()[trailstatus]
@@ -453,16 +459,32 @@ func ReturnOpenSea() {
 			changeArgs3 := []interface{}{}
 			changeArgs4 := []interface{}{}
 			for _, v := range *l {
-				id := v["id"]
+				id := v["clue_id"] //
 				position_id := v["position_id"]
 				seatNumber := gconv.String(v["seatNumber"])
+
+				args2 := []interface{}{id}
+				//获取跟进内容
+				sql1 := `select COUNT(1) FROM dwd_f_crm_trail_content WHERE clue_id =?;`
+				sql2 := `SELECT COUNT(1) FROM dwd_f_crm_trail_content WHERE clue_id =? `
+				if intime != "" {
+					sql2 += ` createime < ?`
+					args2 = append(args2, intime)
+				}
+				//保留未跟进线索
+				if c1, c2 := config.JianyuSubjectdb.CountBySql(sql1, id),
+					config.JianyuSubjectdb.CountBySql(sql2, args2...); c1 != 0 && c2 <= 0 {
+					log.Println("不满足线索过滤", id)
+					continue
+				}
+
 				ids = append(ids, id)
 				//
 				args = append(args, id, now, comeinsource)
 				changeArgs1 = append(changeArgs1, id, position_id, "trailstatus", "线索状态变更", codeMap[trailstatus], "流失", now, -1)
 				changeArgs2 = append(changeArgs2, id, position_id, "position_id", "所属人变更", GetPositionName(seatNumber), "/", now, -1)
 				changeArgs3 = append(changeArgs3, id, position_id, "退出任务车", "未更新跟进记录自动退回公海", now, -1)
-				changeArgs3 = append(changeArgs3, id, position_id, "退回公海", "未更新跟进记录自动退回公海", now, -1)
+				changeArgs4 = append(changeArgs3, id, position_id, "退回公海", "未更新跟进记录自动退回公海", now, -1)
 
 			}
 			whs := []string{}
@@ -475,15 +497,31 @@ func ReturnOpenSea() {
 			//进入公海
 			config.JianyuSubjectdb.InsertIgnoreBatch("dwd_f_crm_open_sea", []string{"clue_id", "comeintime", "comeinsource"}, args)
 			//改线索表 职位id  坐席号 弄成空 分配状态改成未分配
-			config.JianyuSubjectdb.UpdateOrDeleteBySql(`UPDATE dwd_f_crm_clue_info SET seatNumber ='',position_id=0,is_assign=0  WHERE id in (`+wh+`)`, ids...)
-
+			up1 := config.JianyuSubjectdb.UpdateOrDeleteBySql(`UPDATE dwd_f_crm_clue_info SET seatNumber ='',position_id=0,is_assign=0  WHERE id in (`+wh+`)`, ids...)
+			if up1 == -1 {
+				log.Println("修改记录失败", `UPDATE dwd_f_crm_clue_info SET seatNumber ='',position_id=0,is_assign=0  WHERE id in (`+wh+`)`, ids)
+			}
 			// 添加线索修改记录
 
 			//线索状态变更记录
-			config.JianyuSubjectdb.InsertBatch("dwd_f_crm_clue_change_record", []string{"clue_id", "position_id", "change_field", "change_type", "old_value", "new_value", "createtime", "operator_id"}, changeArgs1)
-			config.JianyuSubjectdb.InsertBatch("dwd_f_crm_clue_change_record", []string{"clue_id", "position_id", "change_field", "change_type", "old_value", "new_value", "createtime", "operator_id"}, changeArgs2)
-			config.JianyuSubjectdb.InsertBatch("dwd_f_crm_clue_change_record", []string{"clue_id", "position_id", "change_type", "new_value", "createtime", "operator_id"}, changeArgs3)
-			config.JianyuSubjectdb.InsertBatch("dwd_f_crm_clue_change_record", []string{"clue_id", "position_id", "change_type", "new_value", "createtime", "operator_id"}, changeArgs4)
+			i1, _ := config.JianyuSubjectdb.InsertBatch("dwd_f_crm_clue_change_record", []string{"clue_id", "position_id", "change_field", "change_type", "old_value", "new_value", "createtime", "operator_id"}, changeArgs1)
+			i2, _ := config.JianyuSubjectdb.InsertBatch("dwd_f_crm_clue_change_record", []string{"clue_id", "position_id", "change_field", "change_type", "old_value", "new_value", "createtime", "operator_id"}, changeArgs2)
+			i3, _ := config.JianyuSubjectdb.InsertBatch("dwd_f_crm_clue_change_record", []string{"clue_id", "position_id", "change_type", "new_value", "createtime", "operator_id"}, changeArgs3)
+			i4, _ := config.JianyuSubjectdb.InsertBatch("dwd_f_crm_clue_change_record", []string{"clue_id", "position_id", "change_type", "new_value", "createtime", "operator_id"}, changeArgs4)
+			if i1 <= 0 || i2 <= 0 || i3 <= 0 || i4 <= 0 {
+				if i1 <= 0 {
+					log.Println("i1失败", changeArgs1)
+				}
+				if i2 <= 0 {
+					log.Println("i2失败", changeArgs2)
+				}
+				if i3 <= 0 {
+					log.Println("i3失败", changeArgs3)
+				}
+				if i4 <= 0 {
+					log.Println("i4失败", changeArgs4)
+				}
+			}
 			if count > 0 {
 				return true
 			}
@@ -505,11 +543,15 @@ func GetComeSource() map[string]int {
 	}
 }
 
-func GetOneSeaC() map[string]bool {
+func GetOneSeaC(oneClassA, oneClassB map[string]bool) map[string]bool {
 	m := map[string]bool{}
 	config.JianyuSubjectdb.SelectByBath(500, func(l *[]map[string]interface{}) bool {
 		for _, v := range *l {
-			m[gconv.String(v["clue_id"])] = true
+			clueid := gconv.String(v["clue_id"])
+			if oneClassA[clueid] || oneClassB[clueid] {
+				continue
+			}
+			m[clueid] = true
 		}
 		return true
 	}, `select clue_id from dwd_f_crm_open_sea where comeinsource in(2,3,4)`)
@@ -534,3 +576,51 @@ func GetPositionName(seatNumber string) string {
 	}
 	return ""
 }
+
+//（4）回收站：
+//A.5个自然日内被销售人员手动退回公海的客户；
+//B.3个自然日内有过“已接听”的通话记录且仍处于“商机线索”状态下的客户。
+func Recycle() {
+	log.Println("回收站开始")
+	ctx := gctx.New()
+	t := time.Now()
+	t = time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local)
+	handReturn := g.Cfg().MustGet(ctx, "handReturn").Int()
+	tA := t.AddDate(0, 0, -handReturn)
+	recycleB := g.Cfg().MustGet(ctx, "recycleB").Int()
+	tB := t.AddDate(0, 0, -recycleB)
+	for sql, _ := range map[string]interface{}{
+		fmt.Sprintf(`select id from dwd_f_crm_open_sea where comeinsource=%v and comeintime > "%s" ;`, 2, tA.Format(date.Date_Full_Layout)): 1,
+		fmt.Sprintf(`SELECT a.uid,b.phone,c.createTime,a.id FROM dwd_f_crm_clue_info a 
+					LEFT JOIN dwd_f_userbase_contacts b ON b.baseinfo_id=a.uid
+					LEFT JOIN Call_Accounting.voice_record c ON c.CalledNo = b.phone
+					WHERE a.trailstatus=01 AND a.uid !="" AND b.phone IS NOT NULL AND c.State="dealing" AND c.createTime >"%s"
+					`, tB.Format(date.Date_Full_Layout)): 2,
+	} {
+		func(sql string) {
+			config.JianyuSubjectdb.SelectByBath(500, func(l *[]map[string]interface{}) bool {
+				ids := []interface{}{}
+				for _, v := range *l {
+					id := v["id"]
+					ids = append(ids, id)
+				}
+				whs := []string{}
+				for i := 0; i < len(ids); i++ {
+					whs = append(whs, "?")
+				}
+				wh := strings.Join(whs, ",")
+				config.JianyuSubjectdb.UpdateOrDeleteBySql(`UPDATE dwd_f_crm_open_sea SET level = 4 WHERE id in (`+wh+`)`, ids...)
+				return true
+			}, sql)
+		}(sql)
+	}
+
+	log.Println("回收站结束")
+
+}
+
+/*
+dwd_f_crm_clue_info.uid    trailstatus==01
+dwd_f_userbase_contacts.baseinfo_id ==dwd_f_crm_clue_info.uid
+ >> dwd_f_userbase_contacts.phone== Call_Accounting.voice_record.CalledNo &&State=="dealing"
+*/

telemarketingEtl/entity/dwd_f_userbase_event_info.go

@@ -2,6 +2,7 @@ package entity
 
 import (
 	"context"
+	"fmt"
 	"log"
 	"net/url"
 	"regexp"
@@ -45,6 +46,7 @@ var (
 	PcEntPortraitNameReg  = regexp.MustCompile(".*/swordfish/page_big_pc/ent_portrait/(.*)")
 	AppEntPortraitNameReg = regexp.MustCompile(".*/jyapp/big/page/ent_portrait\\?eId=(.*)")
 	CollectionReg         = regexp.MustCompile(".*frontPage/collection/sess/index")
+	PcEntSearchReq        = regexp.MustCompile(".*/jylab/entSearch/index.html")
 )
 
 func init() {
@@ -81,7 +83,7 @@ func loading() {
 			case "pcForecast": //pc-潜在项目预测
 				PcForecastReg = req
 			case "appForecast": //app-潜在项目预测列表页
-				AppClientReg = req
+				AppForecastReg = req
 			case "pcCollection": //pc-标讯收藏
 				PcCollectionReg = req
 			case "appCollection": //app-标讯收藏
@@ -109,10 +111,6 @@ func filter(url string) bool {
 	return false
 }
 
-func In() {
-
-}
-
 // 查看事件
 func EventInfoAdd(start, end int64) {
 	//
@@ -135,7 +133,7 @@ func EventInfoAdd(start, end int64) {
 		go func(table string) {
 			defer th_mgo.Close()
 			of := options.Find()
-			of.SetProjection(g.Map{"url": 1, "userid": 1, "hour": 1, "month": 1, "day": 1, "date": 1, "client": 1})
+			of.SetProjection(g.Map{"url": 1, "userid": 1, "hour": 1, "month": 1, "day": 1, "date": 1, "client": 1, "refer": 1})
 			coll := config.MgoLog.C.Database(config.MgoLog.DbName).Collection(table)
 			cur, err := coll.Find(ctx, g.Map{"_id": g.Map{"$gte": util.MongoId(start), "$lt": util.MongoId(end)}}, of)
 			if err == nil && cur.Err() == nil {
@@ -167,6 +165,7 @@ func EventInfoAdd(start, end int64) {
 					if !mongodb.IsObjectIdHex(userid) {
 						userid = GetUserIdByPositionId(userid)
 					}
+					refer := gconv.String(v["refer"]) //用于取分客户监控,客户监控的url地址和采购单位画像是一个
 					eventtype := ""
 					name := ""
 					//根据不同url地址获取 标讯的公告名称、采购单位名称、企业名称
@@ -185,7 +184,7 @@ func EventInfoAdd(start, end int64) {
 								}
 							}
 						}
-					} else if pcBuyPortraitReg.MatchString(url_) {
+					} else if pcBuyPortraitReg.MatchString(url_) && refer != "/swordfish/page_big_pc/my_client" {
 						eventtype = BUYPORTRAIT
 						na := PcBuyPortraitNameReg.FindStringSubmatch(url_)
 						if len(na) > 1 {
@@ -213,9 +212,35 @@ func EventInfoAdd(start, end int64) {
 							entId := util.DecodeId(eId)
 							name = GetEntNameByEntId(entId)
 						}
-					} else if CollectionReg.MatchString(url_) {
-						eventtype = COLLECTION
+					} else if ProductIndexReg.MatchString(url_) {
+						eventtype = PRODUCTINDEX
+					} else if PcClientReg.MatchString(url_) && refer == "/swordfish/page_big_pc/my_client" {
+						eventtype = CLIENT
+						name = PcClient(url_)
+					} else if AppClientReg.MatchString(url_) {
+						eventtype = CLIENT
+						name = AppClient(url_)
+					} else if PcProjectMonitorReg.MatchString(url_) {
+						eventtype = PROJECTPROGRESS
+						name = ProjectProgress(url_)
+					} else if PcEntIntelligentceMonitorReg.MatchString(url_) {
+						eventtype = ENTFOLLOW
+						name = EntFollow(url_)
+					} else if PcForecastReg.MatchString(url_) || AppForecastReg.MatchString(url_) {
+						eventtype = FORECAST
+						name, _ = Forecast(url_)
+					} else if PcEntSearchReq.MatchString(url_) {
+						eventtype = ENTSEARCH
 					}
+
+					/*
+
+						CLIENT          = "客户监控"
+						PROJECTPROGRESS = "项目进度监控"
+						ENTFOLLOW       = "企业情报监控"
+						FORECAST        = "潜在项目预测"
+						ENTSEARCH       = "企业搜索"
+					*/
 					createtime := time.Unix(gconv.Int64(v["date"]), 0).Format(date.Date_Full_Layout)
 					values = append(values, userid, eventtype, name, url_, platform, createtime)
 					if n%index == 0 {
@@ -251,6 +276,33 @@ func EventInfoAdd(start, end int64) {
 			}
 		}(table)
 	}
+
+	//收藏
+	Collection(start, end, fields)
+
+}
+
+func Collection(start, end int64, fields []string) {
+	//单独存储收藏
+	st := time.Unix(start, 0).Format(date.Date_Full_Layout)
+	et := time.Unix(end, 0).Format(date.Date_Full_Layout)
+
+	config.JianyuMaindb.SelectByBath(500, func(l *[]map[string]interface{}) bool {
+		args := []interface{}{}
+		for _, v := range *l {
+			userid := gconv.String(v["userid"])
+			if !mongodb.IsObjectIdHex(userid) {
+				userid = GetUserIdByPositionId(userid)
+			}
+			createtime := gconv.String(v["createdate"])
+			bid := gconv.String(v["bid"])
+			//获取bid信息
+			title, href := GetInfoByBid(bid)
+			args = append(args, userid, COLLECTION, title, href, PC, createtime)
+		}
+		config.JianyuSubjectdb.InsertBatch(DWD_F_USERBASE_EVENT_INFO, fields, args)
+		return true
+	}, `select userid,bid,createdate from bdcollection where createdate >= ? and createdate < ?`, st, et)
 }
 
 // 根据企业id获取企业名称
@@ -266,3 +318,115 @@ func GetEntNameByEntId(entId string) (name string) {
 	}
 	return gconv.String((*entinfo)["company_name"])
 }
+
+//潜在项目预测
+//返回title 和 href
+func Forecast(urls string) (title, href string) {
+	//获取id
+	re := regexp.MustCompile(`id=([^&]+)`)
+	match := re.FindStringSubmatch(urls)
+
+	// 输出结果
+	if len(match) < 2 {
+		return "", ""
+	}
+	id, _ := url.QueryUnescape(match[1])
+	idstr := util.DecodeId(id)
+	if idstr == "" {
+		return "", ""
+	}
+	data, ok := config.MgoEnt.FindById("project_forecast", idstr, nil)
+	if data != nil && len(*data) > 0 && ok {
+		return gconv.String((*data)["title"]), gconv.String((*data)["jyhref"])
+	}
+	return "", ""
+}
+
+//关注项目
+func ProjectProgress(urls string) (title string) {
+	//获取id
+	re := regexp.MustCompile(`sid=([^&]+)`)
+	match := re.FindStringSubmatch(urls)
+	log.Println(match)
+	// 输出结果
+	if len(match) < 2 {
+		return ""
+	}
+	id, _ := url.QueryUnescape(match[1])
+	idstr := util.DecodeId(id)
+	log.Println(idstr)
+	if idstr == "" {
+		return ""
+	}
+	res, _ := config.MgoBid.FindById("bidding", idstr, `{"projectname":1,"href":1}`)
+
+	if res == nil || len(*res) == 0 {
+		res, _ = config.MgoBid.FindById("bidding_back", idstr, `{"projectname":1,"href":1}`)
+		if res == nil || len(*res) == 0 {
+			return ""
+		}
+	}
+	return gconv.String((*res)["projectname"])
+}
+
+//企业情报监控
+func EntFollow(urls string) string {
+	re := regexp.MustCompile(`/[^/]+/[^/]+/[^/]+/(.*)`)
+
+	// 使用正则表达式提取加密戳
+	match := re.FindStringSubmatch(urls)
+	// 输出结果
+	if len(match) < 2 {
+		return ""
+	}
+	if match[1] == "" {
+		return ""
+	}
+	id, _ := url.QueryUnescape(match[1])
+	idstr := util.DecodeId(id)
+	entinfo, _ := config.MgoEnt.FindOneByField("qyxy_std", map[string]interface{}{"_id": idstr}, map[string]interface{}{
+		"company_name": 1, //公司名称
+	})
+	if entinfo == nil || len(*entinfo) == 0 {
+		return ""
+	}
+	return gconv.String((*entinfo)["company_name"])
+}
+
+func PcClient(urls string) string {
+
+	re := regexp.MustCompile(`/[^/]+/[^/]+/[^/]+/(.*)`)
+
+	// 使用正则表达式提取加密戳
+	match := re.FindStringSubmatch(urls)
+	// 输出结果
+	if len(match) < 2 {
+		return ""
+	}
+	if match[1] == "" {
+		return ""
+	}
+
+	return util.Unescape(match[1])
+}
+
+func AppClient(urls string) string {
+	return util.GetQueryParam(urls, "entName")
+}
+
+//return title href
+func GetInfoByBid(bid string) (string, string) {
+	title, href := "", ""
+	res, _ := config.MgoBid.FindById("bidding", bid, `{"title":1,"href":1}`)
+
+	if res == nil || len(*res) == 0 {
+		res, _ = config.MgoBid.FindById("bidding_back", bid, `{"projectname":1,"href":1}`)
+		if res == nil || len(*res) == 0 {
+			return "", ""
+		}
+	}
+	title = gconv.String((*res)["title"])
+	href = fmt.Sprintf("/article/content/%s.html", util.EncodeId(bid))
+
+	return title, href
+}

telemarketingEtl/entity/entity.go

@@ -15,11 +15,17 @@ const (
 	DWD_F_USERBASE_EVENT_INFO = "dwd_f_userbase_event_info"
 	DWD_F_USERBASE_VISIT_INFO = "dwd_f_userbase_visit_info"
 
-	BUYPORTRAIT = "采购单位画像"
-	INFO        = "标讯三级页"
-	ENTPORTRAIT = "企业画像"
-	LOGIN       = "登录"
-	COLLECTION  = "标讯收藏"
+	BUYPORTRAIT     = "采购单位画像"
+	INFO            = "标讯三级页"
+	ENTPORTRAIT     = "企业画像"
+	LOGIN           = "登录"
+	COLLECTION      = "标讯收藏"
+	PRODUCTINDEX    = "会员介绍页面"
+	CLIENT          = "客户监控"
+	PROJECTPROGRESS = "项目进度监控"
+	ENTFOLLOW       = "企业情报监控"
+	FORECAST        = "潜在项目预测"
+	ENTSEARCH       = "企业搜索"
 )
 
 // 根据职位id获取mongodb userid

telemarketingEtl/timetask/task.go

@@ -65,6 +65,7 @@ func TaskOpenSea() {
 
 // 定时任务
 func Run() {
+
 	timetaskBl := gcfg.Instance().MustGet(gctx.New(), "timetaskBl", "").Bool()
 	if timetaskBl {
 		var (

telemarketingEtl/util/util.go

@@ -1,14 +1,15 @@
 package util
 
 import (
-	"app.yhyue.com/moapp/jybase/encrypt"
-	"app.yhyue.com/moapp/jybase/mongodb"
 	"fmt"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"gopkg.in/mgo.v2/bson"
 	"net/url"
 	"strings"
 	"time"
+
+	"app.yhyue.com/moapp/jybase/encrypt"
+	"app.yhyue.com/moapp/jybase/mongodb"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/mgo.v2/bson"
 )
 
 func MongoId(t int64) primitive.ObjectID {
@@ -35,6 +36,13 @@ func DecodeId(eid string) string {
 	return encrypt.DecodeArticleId2ByCheck(eid)[0]
 }
 
+func EncodeId(id string) string {
+	if id == "" {
+		return ""
+	}
+	return encrypt.EncodeArticleId2ByCheck(id)
+}
+
 func SpitObjectId(s string) string {
 	sArr := strings.Split(s, `("`)
 	s = strings.Split(sArr[1], `")`)[0]
@@ -47,3 +55,20 @@ func GetObjectId(timestamp int64) primitive.ObjectID {
 	tid := bson.NewObjectIdWithTime(t)
 	return mongodb.StringTOBsonId(SpitObjectId(tid.String()))
 }
+
+// 查询 URL 中的查询参数值
+func GetQueryParam(urlStr, paramName string) string {
+	// 解析 URL
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return ""
+	}
+
+	// 获取查询参数
+	queryParams := u.Query()
+
+	// 获取查询参数值
+	paramValue := queryParams.Get(paramName)
+
+	return paramValue
+}