wangshan před 4 měsíci
rodič
revize
f6328ba31a

+ 43 - 0
api/personnel/v1/personnel.go

@@ -0,0 +1,43 @@
+package v1
+
+import "github.com/gogf/gf/v2/frame/g"
+
+type PersonnelInsertReq struct {
+	g.Meta `path:"/statistics" tags:"contract" method:"post" summary:"insert records"`
+	Statistics
+}
+
+type PersonnelBehaviorReq struct {
+	g.Meta `path:"/behavior" tags:"contract" method:"post" summary:"insert records"`
+	BS     []*Statistics `json:"bs"`
+}
+
+type Statistics struct {
+	ActionId        string                 `json:"action_id" dc:"事件ID"`
+	ActionType      string                 `json:"action_type" dc:"事件类型"`
+	BreakerId       string                 `json:"breaker_id" dc:"断点ID"`
+	BreakerName     string                 `json:"breaker_name" dc:"断点名称"`
+	OrderId         string                 `json:"order_id" dc:"订单编号"`
+	OrderTime       string                 `json:"order_time" dc:"下单时间"`
+	PayTime         string                 `json:"pay_time" dc:"支付时间"`
+	PayWay          string                 `json:"pay_way" dc:"支付方式"`
+	Price           int                    `json:"price" dc:"金额(元)"`
+	Product         string                 `json:"product" dc:"产品类型"`
+	ProductName     string                 `json:"product_name" dc:"产品类型名称"`
+	PageId          string                 `json:"page_id" dc:"页面ID"`
+	PageName        string                 `json:"page_name" dc:"页面名称"`
+	BiddingId       string                 `json:"bidding_id" dc:"详情页id"`
+	Desc            string                 `json:"desc" dc:"页面功能简述"`
+	Source          string                 `json:"source" dc:"来源名称"`
+	Url             string                 `json:"url" dc:"地址"`
+	ChainName       string                 `json:"chain_name" v:"required"  dc:"公链名称"`
+	Port            string                 `json:"port" `
+	Refer           string                 `json:"pageRefer" dc:"跳转前页面"`
+	SearchWord      string                 `json:"search_word"  dc:"搜索词" `
+	Filter          string                 `json:"filter"  dc:"搜索条件"`
+	BreakData       map[string]interface{} `json:"breakData"  dc:"额外数据"`
+	AppId           string                 `json:"appId"  dc:"appid"`
+	AppVersion      string                 `json:"appVersion"  dc:"版本号"`
+	MiniProgramCode string                 `json:"miniProgramCode"  dc:"来源"`
+	Duration        float64                `json:"duration"  dc:"页面停留时长 秒"`
+}

+ 5 - 1
go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	app.yhyue.com/moapp/jybase v0.0.0-20231120053339-0b7406056861
-	app.yhyue.com/moapp/jylog v0.0.0-20230522075550-05d7230ca545
+	app.yhyue.com/moapp/jylogx v0.0.0-20230522075659-ae6fbedb92bc
 	github.com/gogf/gf/contrib/drivers/clickhouse/v2 v2.5.2
 	github.com/gogf/gf/contrib/drivers/mysql/v2 v2.5.2
 	github.com/gogf/gf/contrib/nosql/redis/v2 v2.5.2
@@ -13,6 +13,7 @@ require (
 )
 
 require (
+	app.yhyue.com/moapp/jylog v0.0.0-20230522075550-05d7230ca545 // indirect
 	github.com/BurntSushi/toml v1.2.0 // indirect
 	github.com/ClickHouse/clickhouse-go/v2 v2.0.15 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -44,6 +45,7 @@ require (
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sirupsen/logrus v1.8.3 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -54,10 +56,12 @@ require (
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	github.com/zeromicro/go-zero v1.5.2 // indirect
 	go.mongodb.org/mongo-driver v1.11.4 // indirect
 	go.opentelemetry.io/otel v1.15.1 // indirect
 	go.opentelemetry.io/otel/sdk v1.14.0 // indirect
 	go.opentelemetry.io/otel/trace v1.15.1 // indirect
+	go.uber.org/automaxprocs v1.5.2 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect

+ 9 - 0
go.sum

@@ -3,6 +3,8 @@ app.yhyue.com/moapp/jybase v0.0.0-20231120053339-0b7406056861 h1:5EwPJ7sFwG8e1sT
 app.yhyue.com/moapp/jybase v0.0.0-20231120053339-0b7406056861/go.mod h1:wcWbj8aQHrFOKO/Bydf9/3UcNlfZK15lrvv40xJSl8w=
 app.yhyue.com/moapp/jylog v0.0.0-20230522075550-05d7230ca545 h1:+Lak4m1zgsigQloOsvp8AJ+0XeX/+PGp9QP550xlbBQ=
 app.yhyue.com/moapp/jylog v0.0.0-20230522075550-05d7230ca545/go.mod h1:uFrsdUBFbETiJlEmr4PtJWPsZlUpPj2bHQRhryu6ggk=
+app.yhyue.com/moapp/jylogx v0.0.0-20230522075659-ae6fbedb92bc h1:QEwc+V6ZTvk3VMFiMgPYJpsAVqRvTeIMupVfpO5PQYk=
+app.yhyue.com/moapp/jylogx v0.0.0-20230522075659-ae6fbedb92bc/go.mod h1:5xAagkwCYnqG5VEHnOV2AqD6DiR169qvOjYKaHMHecU=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -308,6 +310,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
@@ -344,6 +347,8 @@ github.com/sirupsen/logrus v1.8.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs
 github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk=
 github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
 github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
@@ -392,6 +397,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+github.com/zeromicro/go-zero v1.5.2 h1:vpMlZacCMtgdtYzKI3OMyhS6mZ9UQctiAh0J7gIq31I=
+github.com/zeromicro/go-zero v1.5.2/go.mod h1:ndCd1nMMAdEMZgPfdm1fpavHUdBW0ykB6ckCRaSG10w=
 go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
 go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
@@ -415,6 +422,8 @@ go.opentelemetry.io/otel/trace v1.15.1/go.mod h1:IWdQG/5N1x7f6YUlmdLeJvH9yxtuJAf
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
+go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=

+ 98 - 2
internal/controller/personnel/personnel.go

@@ -6,7 +6,7 @@ import (
 	"context"
 	"encoding/json"
 	"github.com/gogf/gf/v2/frame/g"
-	v1 "personnelBehavior/api/hello/v1"
+	v1 "personnelBehavior/api/personnel/v1"
 	"personnelBehavior/internal/model"
 	"personnelBehavior/internal/service"
 	"personnelBehavior/internal/utils"
@@ -54,7 +54,7 @@ func (p *personnel) Statistics(ctx context.Context, req *v1.PersonnelInsertReq)
 	}
 	//匿名用户id
 	in.JYTrustedId = utils.GetTrustedId(g.RequestFromCtx(ctx))
-	in.Date = time.Now().Unix()
+	in.Date = time.Now()
 	//浏览器信息
 	r := g.RequestFromCtx(ctx).Request
 	in.Ip = common.GetIp(r)
@@ -108,3 +108,99 @@ func (p *personnel) Statistics(ctx context.Context, req *v1.PersonnelInsertReq)
 	g.RequestFromCtx(ctx).Response.WriteJson(res)
 	return
 }
+
+// Behaviors 数组
+func (p *personnel) Behaviors(ctx context.Context, req *v1.PersonnelBehaviorReq) (res *utils.ResponseRes, err error) {
+	res = &utils.ResponseRes{}
+	bs := req.BS
+	g.Log().Info(ctx, bs)
+	if len(bs) == 0 {
+		return
+	}
+	for _, b := range bs {
+		var in model.PersonnelInsertInput
+		var sessionGet *model.Context
+		g.Log().Info(model.Ctx, "--Statistics--接收参数--", b, g.RequestFromCtx(ctx).Header)
+		in.ActionId = common.If(b.ActionId == "undefined", "", b.ActionId).(string)
+		in.ActionType = common.If(b.ActionType == "undefined", "", b.ActionType).(string)
+		in.BreakerId = common.If(b.BreakerId == "undefined", "", b.BreakerId).(string)
+		in.BreakerName = common.If(b.BreakerName == "undefined", "", b.BreakerName).(string)
+		in.OrderId = common.If(b.OrderId == "undefined", "", b.OrderId).(string)
+		in.OrderTime = common.If(b.OrderTime == "undefined", "", b.OrderTime).(string)
+		in.PayTime = common.If(b.PayTime == "undefined", "", b.PayTime).(string)
+		in.PayWay = common.If(b.PayWay == "undefined", "", b.PayWay).(string)
+		in.Price = b.Price
+		in.Product = common.If(b.Product == "undefined", "", b.Product).(string)
+		in.ProductName = common.If(b.ProductName == "undefined", "", b.ProductName).(string)
+		in.PageId = common.If(b.PageId == "undefined", "", b.PageId).(string)
+		in.PageName = common.If(b.PageName == "undefined", "", b.PageName).(string)
+		in.Desc = common.If(b.Desc == "undefined", "", b.Desc).(string)
+		in.Source = common.If(b.Source == "undefined", "", b.Source).(string)
+		in.Url = common.If(b.Url == "undefined", "", b.Url).(string)
+		in.AppId = common.If(b.AppId == "undefined", "", b.AppId).(string)
+		in.AppVersion = common.If(b.AppVersion == "undefined", "", b.AppVersion).(string)
+		sessionGet, err = model.UserSessionCtx.Get(ctx)
+		if err == nil {
+			session := sessionGet.JySession
+			if session != nil && session.MgoUserId != "" {
+				in.UserId = session.MgoUserId
+				in.PositionId = session.PositionId
+				in.Phone = session.Phone
+			}
+		}
+		//匿名用户id
+		in.JYTrustedId = utils.GetTrustedId(g.RequestFromCtx(ctx))
+		in.Date = time.Now()
+		//浏览器信息
+		r := g.RequestFromCtx(ctx).Request
+		in.Ip = common.GetIp(r)
+		if in.Url == "" {
+			in.Url = r.Referer()
+		}
+		if strings.Contains(in.Url, "article") || strings.Contains(in.Url, "nologin") {
+			in.BiddingId, in.Source = utils.GetDetailPageId(in.Url)
+		}
+		userAgent := utils.GetUserAgentInfo(r)
+		in.Browser = userAgent.BrowserName
+		in.BrowserVersion = userAgent.BrowserVersion
+		in.Os = userAgent.OsName
+		in.OsVersion = userAgent.OsVersion
+		in.Platform = userAgent.Platform
+		in.UserAgent = r.UserAgent()
+		in.ChainName = b.ChainName
+		//A24112  新增字段
+		in.Port = utils.GetUserPort(r)                             // 行为端口 wx,Android,ios,pc,h5
+		in.Refer = g.RequestFromCtx(ctx).GetHeader("pageReferrer") // 跳转前页面
+		if b.BreakData != nil && len(b.BreakData) > 0 {
+			// 处理page_name 和product_name
+			if pageName, ok := b.BreakData["page_name"]; ok && pageName != "" && pageName != "undefined" && in.PageName == "" {
+				in.PageName = common.InterfaceToStr(pageName)
+			}
+			if productName, ok := b.BreakData["product_name"]; ok && productName != "" && productName != "undefined" && in.ProductName == "" {
+				in.ProductName = common.InterfaceToStr(productName)
+			}
+			// 小程序 如果有id 存解密后的id
+			if bid, ok := b.BreakData["id"]; ok && bid != "" && bid != "undefined" && in.BiddingId == "" {
+				biddingId := encrypt.CommonDecodeArticle("content", common.InterfaceToStr(bid))[0]
+				in.BiddingId = common.InterfaceToStr(biddingId)
+			}
+			breakData, _ := json.Marshal(b.BreakData)
+			in.BreakData = string(breakData)
+		}
+		// 小程序的refer是从body里面取的
+		if in.Refer == "" && b.Refer != "" && b.Refer != "undefined" {
+			in.Refer = b.Refer
+		}
+		in.MiniProgramCode = g.RequestFromCtx(ctx).GetHeader("miniProgramCode")
+		in.SearchWord = b.SearchWord // 搜索词
+		in.Filter = b.Filter         // 搜索条件
+		in.Duration = b.Duration
+		err = service.Personnel().BatchInsertRecords(ctx, &in)
+		if err != nil {
+			res.Msg = err.Error()
+			g.Log().Error(ctx, "数据传入异常:", err.Error())
+		}
+	}
+	res.Data = bs
+	return
+}

+ 2 - 2
internal/dao/internal/personnel.go

@@ -73,12 +73,12 @@ func (dao *PersonnelDao) BatchExecInsert(ctx context.Context, in []*model.Person
 		return
 	}
 	var (
-		timeNow    = time.Now()
 		insertData []g.Map
 	)
 	for _, i := range in {
 		var (
-			clickTime = timeNow
+			timeNow   = time.Now()
+			clickTime = i.Date
 			orderTIme time.Time
 			payTime   time.Time
 		)

+ 21 - 19
internal/model/personnel.go

@@ -1,5 +1,7 @@
 package model
 
+import "time"
+
 type PersonnelInsertInput struct {
 	ActionId        string
 	ActionType      string
@@ -21,23 +23,23 @@ type PersonnelInsertInput struct {
 	PositionId      int64
 	Phone           string
 	Ip              string
-	Platform        string //平台
-	Os              string //系统
-	OsVersion       string //系统版本
-	Browser         string //浏览器
-	BrowserVersion  string //浏览器版本
-	Date            int64  //访问时间
-	Url             string //页面url
-	UserAgent       string //header 信息
-	JYTrustedId     string //匿名用户id
-	ChainName       string //库信息
-	Port            string //行为端口
-	Refer           string //跳转前页面
-	SearchWord      string //搜索词
-	Filter          string //搜索条件
-	BreakData       string //"额外数据
-	AppId           string // appid"
-	AppVersion      string // 版本号
-	MiniProgramCode string // "来源"`
-	Duration        int64  //页面停留时长 秒
+	Platform        string    //平台
+	Os              string    //系统
+	OsVersion       string    //系统版本
+	Browser         string    //浏览器
+	BrowserVersion  string    //浏览器版本
+	Date            time.Time //访问时间
+	Url             string    //页面url
+	UserAgent       string    //header 信息
+	JYTrustedId     string    //匿名用户id
+	ChainName       string    //库信息
+	Port            string    //行为端口
+	Refer           string    //跳转前页面
+	SearchWord      string    //搜索词
+	Filter          string    //搜索条件
+	BreakData       string    //"额外数据
+	AppId           string    // appid"
+	AppVersion      string    // 版本号
+	MiniProgramCode string    // "来源"`
+	Duration        float64   //页面停留时长 秒
 }

+ 4 - 4
manifest/config/config.yaml

@@ -18,7 +18,7 @@ logger:
 # 数据库信息
 database:
   default:
-    link: "clickhouse:jytop:pwdTopJy123@tcp(192.168.3.207:19000)/data_analysis?dial_timeout=2000ms&max_execution_time=60"
+    link: "clickhouse:jytop:pwdTopJy123@tcp(172.20.45.129:19000)/data_analysis?dial_timeout=2000ms&max_execution_time=60"
     debug: true
 # 数据库信息
 group: "data_analysis"
@@ -28,16 +28,16 @@ ttlUnit: 1 #超时时间单位 1:月;2:周;3:天
 # Redis 配置示例
 redis:
   default:
-    address: 192.168.3.149:1712
+    address: 172.20.45.129:1712
   session: # session
-    address: 192.168.3.149:1713
+    address: 172.20.45.129:1713
 #其他
 authcode: "jy_personnel_abc"
 #缓存信息
 cacheChanInfo:
   name: "人员行为记录"
   dataSize: 10000   #数据缓存通道大小
-  bulkSize: 500     #批次- 每批的数量
+  bulkSize: 2     #批次- 每批的数量
   timeAfter: 2000   # 定时保存 毫秒
   timeout: 10000    # 缓存通道满时,超时丢弃
   sCSize: 3         # 数据库并发存储数据