tj

.idea/.gitignore

@@ -0,0 +1,8 @@
+# 默认忽略的文件
+/shelf/
+/workspace.xml
+# 基于编辑器的 HTTP 客户端请求
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/userActionEtl.iml" filepath="$PROJECT_DIR$/.idea/userActionEtl.iml" />
+    </modules>
+  </component>
+</project>

.idea/userActionEtl.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="Go" enabled="true" />
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

config.go

@@ -0,0 +1,258 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"regexp"
+	"time"
+
+	"app.yhyue.com/moapp/jybase/mysql"
+	"app.yhyue.com/moapp/jybase/redis"
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gcfg"
+
+	qu "app.yhyue.com/moapp/jybase/common"
+	mg "app.yhyue.com/moapp/jybase/mongodb"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"github.com/lionsoul2014/ip2region/binding/golang/xdb"
+	// "github.com/gogf/gf/v2/frame/g"
+	// "github.com/gogf/gf/v2/os/gcfg"
+)
+
+var (
+	Sysconfig       *config
+	Mgo             mg.MongodbSim
+	Mgo_Log         mg.MongodbSim
+	Ch_analysis     driver.Conn
+	Ch_userbehavior driver.Conn
+	Base            *mysql.Mysql
+	Reader          *xdb.Searcher
+	RegWx           *regexp.Regexp
+	Ch              *mysql.Mysql
+	DataService     *mysql.Mysql
+)
+
+// x
+func init() {
+	g.Cfg().GetAdapter().(*gcfg.AdapterFile).SetFileName("config.yaml")
+
+	qu.ReadConfig(&Sysconfig)
+
+	RegWx, _ = regexp.Compile("MicroMessenger")
+	if Sysconfig != nil {
+		//
+		//初始化redis
+		if Sysconfig.Redis.Main != nil {
+			log.Println("初始化 redis", Sysconfig.Redis.Main.Address)
+			redis.InitRedisBySize(Sysconfig.Redis.Main.Address, 20, 15, 300)
+		}
+		if Sysconfig.Mongodb.Main != nil {
+			log.Println("初始化 mongodb main")
+			Mgo = mg.MongodbSim{
+				MongodbAddr: Sysconfig.Mongodb.Main.Address,
+				Size:        Sysconfig.Mongodb.Main.Size,
+				DbName:      Sysconfig.Mongodb.Main.DbName,
+			}
+			Mgo.InitPool()
+		}
+		if Sysconfig.Mongodb.Log != nil {
+			log.Println("初始化 mongodb log")
+			Mgo_Log = mg.MongodbSim{
+				MongodbAddr: Sysconfig.Mongodb.Log.Address,
+				Size:        Sysconfig.Mongodb.Log.Size,
+				DbName:      Sysconfig.Mongodb.Log.DbName,
+				UserName:    Sysconfig.Mongodb.Log.UserName,
+				Password:    Sysconfig.Mongodb.Log.Password,
+			}
+			Mgo_Log.InitPool()
+		}
+		err := ConnectClickhouseAnalysis()
+		if err != nil {
+			log.Println("数据库连接失败", err)
+			return
+		}
+		err = ConnectClickhouseBehavior()
+		if err != nil {
+			log.Println("数据库连接失败", err)
+			return
+		}
+		if Sysconfig.Tidb.Base != nil {
+			log.Println("初始化 tidb")
+			Base = &mysql.Mysql{
+				Address:      Sysconfig.Tidb.Base.Address,
+				UserName:     Sysconfig.Tidb.Base.UserName,
+				PassWord:     Sysconfig.Tidb.Base.PassWord,
+				DBName:       Sysconfig.Tidb.Base.DbName,
+				MaxOpenConns: Sysconfig.Tidb.Base.MaxOpenConns,
+				MaxIdleConns: Sysconfig.Tidb.Base.MaxIdleConns,
+			}
+			Base.Init()
+		}
+
+		if Sysconfig.Tidb.DataService != nil {
+			log.Println("初始化 tidb")
+			DataService = &mysql.Mysql{
+				Address:      Sysconfig.Tidb.DataService.Address,
+				UserName:     Sysconfig.Tidb.DataService.UserName,
+				PassWord:     Sysconfig.Tidb.DataService.PassWord,
+				DBName:       Sysconfig.Tidb.DataService.DbName,
+				MaxOpenConns: Sysconfig.Tidb.DataService.MaxOpenConns,
+				MaxIdleConns: Sysconfig.Tidb.DataService.MaxIdleConns,
+			}
+			DataService.Init()
+		}
+
+		r, err := xdb.LoadContentFromFile(Sysconfig.IpdbPath)
+		if err != nil {
+			log.Printf("Error opening GeoIP database: %v", err)
+			os.Exit(1)
+		} else {
+			r1, e1 := xdb.NewWithBuffer(r)
+			if e1 != nil {
+				log.Printf("Error opening GeoIP database: %v", e1)
+				os.Exit(1)
+			} else {
+				Reader = r1
+			}
+		}
+
+		if Sysconfig.Tidb.Ch != nil {
+			log.Println("初始化 clickhouse tidb连接模式", mysql.CLICKHOUSE, Sysconfig.Tidb.Ch)
+			//clickhouse://jytop:pwdTopJy123@192.168.3.207:19000/information?dial_timeout=2000ms&max_execution_time=60s
+			//
+			Ch = mysql.NewInit(mysql.CLICKHOUSE, Sysconfig.Clickhouseaddr, Sysconfig.Tidb.Ch.MaxOpenConns, Sysconfig.Tidb.Ch.MaxIdleConns)
+
+		}
+	}
+}
+
+type config struct {
+	Mongodb struct {
+		Main *mgoConf
+		Log  *mgoConf
+	}
+	ClickHouse struct {
+		Analysis     *ClickhouseS
+		Userbehavior *ClickhouseS
+	}
+	CacheChanInfo struct {
+		Name      string
+		DataSize  int
+		BulkSize  int
+		TimeAfter int
+		Timeout   int
+		SCSize    int
+	}
+	Tidb struct {
+		Base                *mysqlConf
+		Ch                  *mysqlConf
+		DataService         *mysqlConf
+		DataServiceCollName string
+	}
+	IpdbPath string
+	Redis    struct {
+		Main *redisConf
+	}
+	LogTask        string
+	TaskIsRun      bool
+	TimeOut        int
+	Clickhouseaddr string
+}
+
+type mgoConf struct {
+	Address         string
+	Size            int
+	DbName          string
+	UserName        string
+	Password        string
+	Collection      string
+	Collection_back string
+}
+
+type redisConf struct {
+	Address string
+}
+type mysqlConf struct {
+	DbName       string
+	Address      string
+	UserName     string
+	PassWord     string
+	MaxOpenConns int
+	MaxIdleConns int
+}
+
+type ClickhouseS struct {
+	Username     string `json:"username"`
+	Password     string `json:"password"`
+	Address      string `json:"address"`
+	DbName       string `json:"dbName"`
+	MaxIdleConns int    `json:"maxIdleConns"`
+	MaxOpenConns int    `json:"maxOpenConns"`
+}
+
+func ConnectClickhouseAnalysis() error {
+	var (
+		ctx = context.Background()
+		err error
+	)
+	Ch_analysis, err = clickhouse.Open(&clickhouse.Options{
+		Addr:         []string{Sysconfig.ClickHouse.Analysis.Address},
+		DialTimeout:  10 * time.Second,
+		MaxIdleConns: Sysconfig.ClickHouse.Analysis.MaxIdleConns,
+		MaxOpenConns: Sysconfig.ClickHouse.Analysis.MaxOpenConns,
+		Auth: clickhouse.Auth{
+			Database: Sysconfig.ClickHouse.Analysis.DbName,
+			Username: Sysconfig.ClickHouse.Analysis.Username,
+			Password: Sysconfig.ClickHouse.Analysis.Password,
+		},
+		Debugf: func(format string, v ...interface{}) {
+			fmt.Printf(format, v)
+		},
+	})
+	if err != nil {
+		return err
+	}
+	if err := Ch_analysis.Ping(ctx); err != nil {
+		if exception, ok := err.(*clickhouse.Exception); ok {
+			fmt.Printf("Exception [%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
+		}
+		return err
+	}
+	log.Println("初始化用户行为日志clickhouse")
+	return nil
+}
+
+func ConnectClickhouseBehavior() error {
+	var (
+		ctx = context.Background()
+		err error
+	)
+	Ch_userbehavior, err = clickhouse.Open(&clickhouse.Options{
+		Addr:         []string{Sysconfig.ClickHouse.Userbehavior.Address},
+		DialTimeout:  10 * time.Second,
+		MaxIdleConns: Sysconfig.ClickHouse.Userbehavior.MaxIdleConns,
+		MaxOpenConns: Sysconfig.ClickHouse.Userbehavior.MaxOpenConns,
+		Auth: clickhouse.Auth{
+			Database: Sysconfig.ClickHouse.Userbehavior.DbName,
+			Username: Sysconfig.ClickHouse.Userbehavior.Username,
+			Password: Sysconfig.ClickHouse.Userbehavior.Password,
+		},
+		Debugf: func(format string, v ...interface{}) {
+			fmt.Printf(format, v)
+		},
+	})
+	if err != nil {
+		return err
+	}
+	if err := Ch_userbehavior.Ping(ctx); err != nil {
+		if exception, ok := err.(*clickhouse.Exception); ok {
+			fmt.Printf("Exception [%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
+		}
+		return err
+	}
+	log.Println("初始化用户行为日志-新clickhouse")
+	return nil
+}

config.json

@@ -0,0 +1,79 @@
+{
+	"mongodb": {
+		"main": {
+			"address": "172.20.45.129:27002",
+			"size": 10,
+			"dbName": "qfw"
+		},
+		"log": {
+			"address": "172.20.45.129:27002",
+			"size": 5,
+			"dbName": "qfw",
+			"userName": "",
+			"password": ""
+		}
+	},
+	"clickhouse": {
+		"analysis": {
+			"username": "jytop",
+			"password": "pwdTopJy123",
+			"address": "172.20.45.129:19000",
+			"dbName": "data_analysis",
+			"maxIdleConns": 5,
+			"maxOpenConns": 5
+		},
+		"userbehavior": {
+			"username": "jytop",
+			"password": "pwdTopJy123",
+			"address": "172.20.45.129:19000",
+			"dbName": "userbehavior",
+			"maxIdleConns": 5,
+			"maxOpenConns": 5
+		}
+	},
+	"cacheChanInfo": {
+		"name": "用户行为分析",
+		"dataSize": 10000,
+		"bulkSize": 500,
+		"timeAfter": 2000,
+		"timeout": 10000,
+		"sCSize": 3
+	},
+	"tidb": {
+		"base": {
+			"dbName": "base_service",
+			"address": "172.20.45.129:4000",
+			"userName": "root",
+			"passWord": "=PDT49#80Z!RVv52_z",
+			"maxOpenConns": 10,
+			"maxIdleConns": 5
+		},
+		"ch": {
+			"dbName": "data_analysis",
+			"address": "172.20.45.129:19000",
+			"userName": "jytop",
+			"passWord": "pwdTopJy123",
+			"maxOpenConns": 10,
+			"maxIdleConns": 5
+		},
+		"dataService": {
+			"dbName": "data_service",
+			"address": "127.0.0.1:4001",
+			"userName": "zhangxinlei",
+			"passWord": "Zxl#20220819",
+			"maxOpenConns": 10,
+			"maxIdleConns": 5
+		},
+		"dataServiceCollName": "user_system_cs"
+	},
+	"ipdbPath": "./ip2region.xdb",
+	"redis": {
+		"main": {
+			"address": "useraction=172.20.45.129:1712"
+		}
+	},
+	"logTask": "0 0 * * * ?",
+	"taskIsRun": true,
+	"timeOut": 60,
+	"clickhouseaddr":"clickhouse://jytop:pwdTopJy123@172.20.45.129:19000/userbehavior?dial_timeout=2000ms&max_execution_time=60s"
+}

config.yaml

@@ -0,0 +1,7 @@
+# 数据库信息
+database:
+  default:
+    link: "clickhouse:jytop:pwdTopJy123@tcp(172.20.45.129:19000)/data_analysis?dial_timeout=2000ms&max_execution_time=60"
+    debug: false
+
+test: "abc"

entity.go

@@ -0,0 +1,89 @@
+package main
+
+import "time"
+
+type UserBehaviorLog struct {
+	UserID           string    // 用户ID
+	ActionID         int64     // 动作ID
+	TrustedID        string    // 浏览器指纹
+	IP               string    // 用户IP地址
+	Area             string    // 省份
+	City             string    // 城市
+	OS               string    // 操作系统
+	Browser          string    // 浏览器类型
+	BrowserVersion   string    // 浏览器版本
+	Date             time.Time // 行为发生时间
+	Platform         string    // 平台（如移动端、PC端）
+	Subsystem        string    // 子系统
+	Module           string    // 模块
+	URL              string    // 访问的URL
+	URLName          string    // URL名称
+	URLElement       string    // 页面元素
+	AddField         string    // 附加数据
+	TimeStamp        int32     // 时间戳
+	Refer            string    // 引用来源
+	ReferName        string    // 引用名称
+	ReferType        string    // 引用类型
+	SessionID        string    // 会话ID
+	SessionStartTime time.Time // 会话开始时间
+	Phone            string    // 手机号码
+	PageName         string    // 页面名称
+	Desc             string    // 描述
+	BreakData        string    // 额外数据
+	BreakerName      string    //埋点名称
+	PageCodeId       uint64    //页面id
+	DataType         string    //前端页面动作
+	PositionId       string    //职位id
+}
+
+type Strategy struct {
+	Name      string //
+	DataSize  int    //#数据缓存通道大小
+	BulkSize  int    //#批次- 每批的数量
+	TimeAfter int    //# 定时保存 毫秒
+	Timeout   int    //# 缓存通道满时，超时丢弃
+	SCSize    int    //# 数据库并发存储数据
+	CacheData chan *UserBehaviorLog
+	SC        chan bool
+}
+
+type Personnel_behavior struct {
+	Jy_trusted_id     string
+	User_id           string
+	Position_id       int64
+	Phone             string
+	Ip                string
+	Platform          string
+	Os                string
+	Os_version        string
+	Browser           string
+	Browser_version   string
+	Date              time.Time // 修改为 time.Time
+	Action_id         string
+	Action_type       string
+	Breaker_id        string
+	Breaker_name      string
+	Click_time        *time.Time // 使用指针以支持 Nullable
+	Order_id          string
+	Order_time        *time.Time // 使用指针以支持 Nullable
+	Pay_time          *time.Time // 使用指针以支持 Nullable
+	Pay_way           string
+	Price             *int32
+	Product           string
+	Product_name      string
+	Page_id           string
+	Page_name         string
+	Bidding_id        string
+	Desc              string
+	Url               string
+	Source            string
+	User_agent        string
+	Port              string
+	Refer             string
+	Search_word       string
+	Filter            string
+	Break_data        string
+	App_id            string
+	App_version       string
+	Mini_program_code string
+}

go.mod

@@ -0,0 +1,72 @@
+module userActionEtl
+
+go 1.23
+
+require (
+	app.yhyue.com/moapp/jybase v0.0.0-20250320080310-ef9b45180b29
+	github.com/ClickHouse/clickhouse-go/v2 v2.33.1
+	github.com/gogf/gf/contrib/drivers/clickhouse/v2 v2.9.0
+	github.com/gogf/gf/v2 v2.9.0
+	github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20241220152942-06eb5c6e8230
+	github.com/robfig/cron v1.2.0
+	github.com/xuri/excelize/v2 v2.9.0
+	go.mongodb.org/mongo-driver v1.11.4
+)
+
+require (
+	github.com/BurntSushi/toml v1.4.0 // indirect
+	github.com/ClickHouse/ch-go v0.65.1 // indirect
+	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/clbanning/mxj/v2 v2.7.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/fatih/color v1.18.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/garyburd/redigo v1.6.2 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/gomodule/redigo v1.8.9 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/grokify/html-strip-tags-go v0.1.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.1 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/paulmach/orb v0.11.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/richardlehane/mscfb v1.0.4 // indirect
+	github.com/richardlehane/msoleps v1.0.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect
+	github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/otel v1.35.0 // indirect
+	go.opentelemetry.io/otel/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.34.0 // indirect
+	go.opentelemetry.io/otel/trace v1.35.0 // indirect
+	golang.org/x/crypto v0.33.0 // indirect
+	golang.org/x/net v0.35.0 // indirect
+	golang.org/x/sync v0.11.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/text v0.22.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gorm.io/driver/mysql v1.0.5 // indirect
+	gorm.io/gorm v1.21.3 // indirect
+)

go.sum

@@ -0,0 +1,213 @@
+app.yhyue.com/moapp/jybase v0.0.0-20250320080310-ef9b45180b29 h1:gm2cwBTPaEL+9nUxjz+og2JfjFMWzp0vUTT+uLJiwCQ=
+app.yhyue.com/moapp/jybase v0.0.0-20250320080310-ef9b45180b29/go.mod h1:/HT/UZ4dKuUKAQqqKrzBBfIZ4vD56DPV4u2QyfH+kbU=
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/ClickHouse/ch-go v0.65.1 h1:SLuxmLl5Mjj44/XbINsK2HFvzqup0s6rwKLFH347ZhU=
+github.com/ClickHouse/ch-go v0.65.1/go.mod h1:bsodgURwmrkvkBe5jw1qnGDgyITsYErfONKAHn05nv4=
+github.com/ClickHouse/clickhouse-go/v2 v2.33.1 h1:Z5nO/AnmUywcw0AvhAD0M1C2EaMspnXRK9vEOLxgmI0=
+github.com/ClickHouse/clickhouse-go/v2 v2.33.1/go.mod h1:cb1Ss8Sz8PZNdfvEBwkMAdRhoyB6/HiB6o3We5ZIcE4=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME=
+github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/garyburd/redigo v1.6.2 h1:yE/pwKCrbLpLpQICzYTeZ7JsTA/C53wFTJHaEtRqniM=
+github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
+github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
+github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/gogf/gf/contrib/drivers/clickhouse/v2 v2.9.0 h1:ml8lrFbSumZQjzQJGTQ2uvk1LY7NJ/FrKox/ITpYc3w=
+github.com/gogf/gf/contrib/drivers/clickhouse/v2 v2.9.0/go.mod h1:Eb5iTy2QypvexojIeb3LdP5VAN8sqNcV0nmHGGt19lk=
+github.com/gogf/gf/v2 v2.9.0 h1:semN5Q5qGjDQEv4620VzxcJzJlSD07gmyJ9Sy9zfbHk=
+github.com/gogf/gf/v2 v2.9.0/go.mod h1:sWGQw+pLILtuHmbOxoe0D+0DdaXxbleT57axOLH2vKI=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4=
+github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
+github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20241220152942-06eb5c6e8230 h1:B0oaMTAQKDZd8cwYT0qsAI7+c3KbFeBNA8GhgoBMXWw=
+github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20241220152942-06eb5c6e8230/go.mod h1:C5LA5UO2ZXJrLaPLYtE1wUJMiyd/nwWaCO5cw/2pSHs=
+github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM=
+github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
+github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
+github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
+github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00=
+github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
+github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d h1:llb0neMWDQe87IzJLS4Ci7psK/lVsjIS2otl+1WyRyY=
+github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
+github.com/xuri/excelize/v2 v2.9.0 h1:1tgOaEq92IOEumR1/JfYS/eR0KHOCsRv/rYXXh6YJQE=
+github.com/xuri/excelize/v2 v2.9.0/go.mod h1:uqey4QBZ9gdMeWApPLdhm9x+9o2lq4iVmjiLfBS5hdE=
+github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNhDCkJX25eik94Si9cTER4A=
+github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
+go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
+go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=
+go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
+go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A=
+go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
+go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
+go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
+golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
+golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ=
+golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
+golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.0.5 h1:WAAmvLK2rG0tCOqrf5XcLi2QUwugd4rcVJ/W3aoon9o=
+gorm.io/driver/mysql v1.0.5/go.mod h1:N1OIhHAIhx5SunkMGqWbGFVeh4yTNWKmMo1GOAsohLI=
+gorm.io/gorm v1.21.3 h1:qDFi55ZOsjZTwk5eN+uhAmHi8GysJ/qCTichM/yO7ME=
+gorm.io/gorm v1.21.3/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=

main.go

@@ -0,0 +1,351 @@
+package main
+
+import (
+	qu "app.yhyue.com/moapp/jybase/common"
+	"app.yhyue.com/moapp/jybase/mongodb"
+	"context"
+	"encoding/json"
+	"fmt"
+	_ "github.com/gogf/gf/contrib/drivers/clickhouse/v2"
+	"github.com/gogf/gf/v2/util/gconv"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"log"
+	"strings"
+	"time"
+	// "github.com/gogf/gf/v2/os/gctx"
+)
+
+func main() {
+
+	//正式环境
+	go InitInsertDataStrategy()
+
+	go Run()
+
+	select {}
+
+}
+
+func dwd_f_personnel_behavior(st, et int64) {
+
+	pageCodeMap := LoadPageCode()
+	s := time.Unix(st, 0).Format(time.DateTime)
+	e := time.Unix(et, 0).Format(time.DateTime)
+	modelMap := MatchRegexModule()
+	q := fmt.Sprintf(`select jy_trusted_id,user_id,position_id,phone,ip,platform,os,os_version,browser,browser_version,date,action_id,action_type,breaker_id,breaker_name,click_time,order_id,order_time,pay_time,pay_way,price,product,product_name,page_id,page_name,bidding_id,desc,url,source,user_agent,port,refer,search_word,filter,break_data,app_id,app_version,mini_program_code from data_analysis.dwd_f_personnel_behavior 
+					where click_time >= '%s'
+					and  click_time < '%s' order by click_time `, s, e)
+	log.Println(q)
+	ctx := context.Background()
+	rows, err := Ch_analysis.Query(ctx, q)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	acitonMap := GetActionMap()
+	filterCount := 0
+	for rows.Next() {
+		m := &Personnel_behavior{}
+		if err := rows.Scan(m); err != nil {
+			if err := rows.Scan(
+				&m.Jy_trusted_id,
+				&m.User_id,
+				&m.Position_id,
+				&m.Phone,
+				&m.Ip,
+				&m.Platform,
+				&m.Os,
+				&m.Os_version,
+				&m.Browser,
+				&m.Browser_version,
+				&m.Date,
+				&m.Action_id,
+				&m.Action_type,
+				&m.Breaker_id,
+				&m.Breaker_name,
+				&m.Click_time,
+				&m.Order_id,
+				&m.Order_time,
+				&m.Pay_time,
+				&m.Pay_way,
+				&m.Price,
+				&m.Product,
+				&m.Product_name,
+				&m.Page_id,
+				&m.Page_name,
+				&m.Bidding_id,
+				&m.Desc,
+				&m.Url,
+				&m.Source,
+				&m.User_agent,
+				&m.Port,
+				&m.Refer,
+				&m.Search_word,
+				&m.Filter,
+				&m.Break_data,
+				&m.App_id,
+				&m.App_version,
+				&m.Mini_program_code,
+			); err != nil {
+				log.Fatal(err)
+			}
+			//过滤策略 过滤三级页||搜索||
+			if (strings.Contains(m.Url, "/content/") ||
+				strings.Contains(m.Url, "/jylab/supsearch/index.html") ||
+				strings.Contains(m.Url, "%2Fcontent%2F")) && m.Action_id == "c_jy_open_page" ||
+				strings.Contains(m.Url, "/qmx/") {
+				filterCount++
+				continue
+			}
+			//获取动作id
+			action := ""
+			if m.Action_id == "c_jy_open_page" {
+				action = "访问页面"
+			} else if m.Action_id == "c_jyclick" {
+				action = "点击页面元素"
+			}
+			if m.Product_name == "下单" || m.Product_name == "支付" || m.Product_name == "绑定手机号" {
+				action = m.Product_name
+			}
+			if strings.Contains(m.Product_name, "登录") || strings.Contains(m.Breaker_name, "登录") {
+				action = "登录"
+			}
+			if strings.Contains(m.Breaker_name, "注册") {
+				action = "注册"
+			}
+			actionid := acitonMap[action]
+			//获取ip
+			area, city := GetIpSource(m.Ip)
+			urlinfo, _ := modelMap.MatchRegexMap(Unescape(m.Url))
+			module := ""
+			if urlinfo != nil {
+				module = urlinfo.Module
+			}
+			referinfo, _ := modelMap.MatchRegexMap(Unescape(m.Refer))
+			refername := ""
+			if referinfo != nil {
+				refername = referinfo.Page_name
+			}
+			//TODO 获取子系统
+			subsystem := ""
+			//TODO 获取附加词
+			fields := ""
+			if ordercode := ExtractOrderCode(m.Url); ordercode != "" {
+				filterMap := map[string]interface{}{
+					"ordercode": ordercode,
+				}
+				filterStr, _ := json.Marshal(filterMap)
+				fields = string(filterStr)
+			}
+			//Macintosh Windows iPhone Linux
+			platform := ""
+			if strings.Contains(m.Url, "wx.") {
+				platform = "wx"
+			} else if strings.Contains(m.Url, "app-") {
+				platform = "android"
+			} else if strings.Contains(m.Url, "ios-") {
+				platform = "ios"
+			} else {
+				platform = "pc"
+			}
+			//来源域内域外
+			referType := GetReferType(m.Refer)
+
+			//pagecodeid
+			pagecodeid := int64(0)
+			if m.Breaker_name == "" && m.Action_id == "c_jy_open_page" {
+				pagecodeid = getPageCodeId(platform, m.Url, m.Breaker_name, module, m.Page_name, actionid, pageCodeMap, modelMap)
+			}
+			StrategyInfo.CacheData <- &UserBehaviorLog{
+				UserID:         m.User_id,
+				ActionID:       actionid,             // 动作ID
+				TrustedID:      m.Jy_trusted_id,      // 浏览器指纹
+				IP:             m.Ip,                 // 用户IP地址
+				Area:           area,                 // 省份
+				City:           city,                 // 城市
+				OS:             m.Os,                 // 操作系统
+				Browser:        m.Browser,            // 浏览器类型
+				BrowserVersion: m.Browser_version,    // 浏览器版本
+				Date:           m.Date,               // 行为发生时间
+				Platform:       platform,             // 平台（如移动端、PC端）
+				Subsystem:      subsystem,            // 子系统
+				Module:         module,               // 模块
+				URL:            m.Url,                // 访问的URL
+				URLName:        m.Page_name,          // URL名称
+				URLElement:     m.Breaker_id,         // 页面元素
+				AddField:       fields,               // 附加数据
+				TimeStamp:      int32(m.Date.Unix()), // 时间戳
+				Refer:          m.Refer,              // 引用来源
+				ReferName:      refername,            // 引用名称
+				ReferType:      referType,            // 引用类型
+				SessionID:      "",                   // 会话ID
+				//SessionStartTime: nil,                  // 会话开始时间 //TODO 会话开始时间x
+				Phone:       m.Phone,        // 手机号码
+				PageName:    m.Page_name,    // 页面名称
+				Desc:        m.Desc,         // 描述
+				BreakData:   m.Break_data,   // 额外数据
+				BreakerName: m.Breaker_name, //埋点名称
+				DataType:    m.Action_id,
+				PositionId:  gconv.String(m.Position_id),
+				PageCodeId:  uint64(pagecodeid),
+			}
+		}
+	}
+	log.Println("clickhouse  end")
+	log.Println("~~", filterCount)
+}
+
+func getPageCodeId(platform, url, breaker_name, module, url_name string, actionid int64, pageCodeMap map[string]int64, moduleMap *CompiledRegexMap) int64 {
+	urlinfo, _ := moduleMap.MatchRegexMap(Unescape(url))
+	url1 := Unescape(url)
+	if urlinfo != nil {
+		url1 = urlinfo.Url
+	}
+	breaker_name = ""
+	key := fmt.Sprintf("%s_%s_%s", platform, url1, breaker_name)
+	id := int64(0)
+	if code := pageCodeMap[fmt.Sprintf("%s_%s_%s", platform, url1, breaker_name)]; code <= 0 {
+
+		id = AddPathCodeId()
+		q := `INSERT INTO path_page_code (id,url_name,url_element,action_id,platform,module,url) VALUES (?,?,?,?,?,?,?) `
+		Ch.InsertBySql(q, id, url_name, breaker_name, actionid, platform, module, url1)
+		pageCodeMap[key] = id
+	} else {
+		id = code
+	}
+	return id
+}
+
+// jy_mongodb_log 剑鱼日志
+func jy_mongodb_log(st, et int64) {
+	userPhoneMap := GetPhone()
+	modelMap := MatchRegexModule()
+	dbname := []string{
+		"jy_logs",
+		"jyapp_logs",
+		"subscribepay_logs",
+		"debris_product_logs",
+	}
+
+	pageCodeMap := LoadPageCode()
+
+	for _, v := range dbname {
+		q := GetQuery(v, st, et)
+		sess := Mgo_Log.GetMgoConn()
+		defer Mgo_Log.DestoryMongoConn(sess)
+		//data := []map[string]interface{}{}
+		it := sess.DB("qfw").C(v).Find(q).Sort("-_id").Select(map[string]interface{}{
+			"userid":     1,
+			"ip":         1,
+			"refer":      1,
+			"client":     1,
+			"url":        1,
+			"createtime": 1,
+			"date":       1,
+			"trustedId":  1,
+			"os":         1,
+			"positionid": 1,
+			"browse":     1,
+		}).Iter()
+		total := 0
+		for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+			if total%50000 == 0 {
+				log.Println("cur index ", total)
+			}
+			userid := qu.ObjToString(tmp["userid"])
+			client := qu.ObjToString(tmp["client"])
+			href := qu.ObjToString(tmp["url"])
+			if !mongodb.IsObjectIdHex(userid) {
+				userid = GetUserId(userid)
+			}
+			trustedId := qu.ObjToString(tmp["trustedId"])
+			ip := qu.ObjToString(tmp["ip"])
+			area, city := GetIpSource(ip)
+			os := qu.ObjToString(tmp["os"])
+			browse := qu.ObjToString(tmp["browse"])
+			positionid := qu.ObjToString(tmp["positionid"])
+
+			refer := qu.ObjToString(tmp["refer"])
+			//TODO
+			actionid := int64(0)
+			if strings.Contains(href, "/login") {
+				actionid = int64(3)
+			}
+			if strings.Contains(href, "/register") {
+				actionid = int64(4)
+			}
+			i_createtime := qu.Int64All(tmp["date"])
+			createtime := time.Unix(i_createtime, 0)
+			if v == "subscribepay_logs" {
+				ct, _ := tmp["createtime"].(primitive.DateTime)
+				createtime = ct.Time()
+				i_createtime = createtime.Unix()
+				actionid = int64(7)
+				if strings.Contains(href, "Pay") {
+					actionid = int64(8)
+				}
+			}
+			platform := GetPlatform(v, client, refer)
+			subsystem := ""
+			hrefinfo, _ := modelMap.MatchRegexMap(Unescape(href))
+			module := ""
+			if hrefinfo != nil {
+				module = hrefinfo.Module
+			}
+			referinfo, _ := modelMap.MatchRegexMap(Unescape(refer))
+			refermodule := ""
+			urlname := ""
+			urldesc := ""
+			if referinfo != nil {
+				refermodule = referinfo.Module
+				urlname = referinfo.Page_name
+				urldesc = referinfo.Desc
+			}
+			referType := GetReferType(refer)
+
+			fields := ""
+			if bid := GetArticleId(href); bid != "" {
+				filterMap := map[string]interface{}{
+					"bidding": bid,
+				}
+				filterStr, _ := json.Marshal(filterMap)
+				fields = string(filterStr)
+			}
+			pagecodeid := getPageCodeId(platform, href, "", module, "", actionid, pageCodeMap, modelMap)
+
+			StrategyInfo.CacheData <- &UserBehaviorLog{
+				UserID:         userid,
+				ActionID:       actionid,            // 动作ID
+				TrustedID:      trustedId,           // 浏览器指纹
+				IP:             ip,                  // 用户IP地址
+				Area:           area,                // 省份
+				City:           city,                // 城市
+				OS:             os,                  // 操作系统
+				Browser:        browse,              // 浏览器类型
+				BrowserVersion: "",                  // 浏览器版本
+				Date:           createtime,          // 行为发生时间
+				Platform:       platform,            // 平台（如移动端、PC端）
+				Subsystem:      subsystem,           // 子系统
+				Module:         module,              // 模块
+				URL:            href,                // 访问的URL
+				URLName:        urlname,             // URL名称
+				URLElement:     "",                  // 页面元素
+				AddField:       fields,              // 附加数据
+				TimeStamp:      int32(i_createtime), // 时间戳
+				Refer:          refer,               // 引用来源
+				ReferName:      refermodule,         // 引用名称
+				ReferType:      referType,           // 引用类型
+				SessionID:      "",                  // 会话ID
+				//SessionStartTime: time.Now(),           // 会话开始时间 //TODO 会话开始时间
+				Phone:      userPhoneMap[userid], // 手机号码
+				PageName:   urlname,              // 页面名称
+				Desc:       urldesc,              // 描述
+				BreakData:  "",                   // 额外数据
+				PageCodeId: uint64(pagecodeid),
+				PositionId: positionid,
+			}
+			tmp = make(map[string]interface{})
+		}
+	}
+	log.Println("mongodb end")
+}

module.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+	"regexp"
+)
+
+type Moduledetail struct {
+	Url          string
+	Module_id    int64
+	Desc         string
+	Platform     string
+	Product_name string
+	Page_name    string
+	Module       string
+}
+
+// RegexMap 是一个存储正则表达式和对应值的映射
+// type RegexMap map[string]string
+type RegexMap map[string]*Moduledetail
+
+// CompiledRegexMap 是一个封装了编译后的正则表达式的结构体
+type CompiledRegexMap struct {
+	RegexMap RegexMap
+	Compiled map[string]*regexp.Regexp
+}
+
+// NewCompiledRegexMap 创建一个新的 CompiledRegexMap
+func NewCompiledRegexMap(regexMap RegexMap) (*CompiledRegexMap, error) {
+	compiled := make(map[string]*regexp.Regexp)
+	for pattern := range regexMap {
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("invalid regex pattern: %s: %w", pattern, err)
+		}
+		compiled[pattern] = re
+	}
+	return &CompiledRegexMap{
+		RegexMap: regexMap,
+		Compiled: compiled,
+	}, nil
+}
+
+// MatchRegexMap 根据输入的字符串，查找匹配的正则表达式键
+// 如果找到匹配的键，则返回对应的值；否则返回 nil
+func (crm *CompiledRegexMap) MatchRegexMap(input string) (*Moduledetail, bool) {
+	for pattern, re := range crm.Compiled {
+		if re.MatchString(input) {
+			return crm.RegexMap[pattern], true
+		}
+	}
+	return nil, false
+}

path.go

@@ -0,0 +1,170 @@
+package main
+
+import (
+	qu "app.yhyue.com/moapp/jybase/common"
+	"context"
+	"fmt"
+	"log"
+	"regexp"
+)
+
+var (
+	ArticleId = regexp.MustCompile("/content/(.*?\\.html)(\\?.*)?")
+)
+
+// initPageCode 初始化页面code
+func initPageCode() {
+	st := "2025-04-01 00:00:00"
+	et := "2025-05-01 00:00:00"
+
+	m := map[string]bool{}
+	q2 := fmt.Sprintf(`select id,platform, module, url, url_name, url_element 
+								from userbehavior.path_page_code`)
+	ctx := context.Background()
+	rows2, err2 := Ch_userbehavior.Query(ctx, q2)
+	if err2 != nil {
+		log.Fatal(err2)
+	}
+	defer rows2.Close()
+	for rows2.Next() {
+		var (
+			id                                           uint64
+			platform, module, url, url_name, url_element string
+		)
+		if err := rows2.Scan(
+			&id, &platform, &module, &url, &url_name, &url_element,
+		); err != nil {
+			log.Fatal(err)
+		}
+		key := fmt.Sprintf("%s_%s_%s", platform, url, url_element)
+		m[key] = true
+	}
+
+	//加载用户行为表
+	q := fmt.Sprintf(`select platform, module, url, url_name, breaker_name 
+								from userbehavior.user_behavior_log where action_id ='%s'and
+								date >='%s' and date <'%s'`, "2", st, et)
+	rows, err := Ch_userbehavior.Query(ctx, q)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var (
+			platform, module, url, url_name, breaker_name string
+		)
+		if err := rows.Scan(
+			&platform, &module, &url, &url_name, &breaker_name,
+		); err != nil {
+			log.Fatal(err)
+		}
+		fu := ArticleId.FindStringSubmatch(url)
+		if len(fu) > 1 {
+			//三级页
+			url = ArticleId.String()
+		}
+		key := fmt.Sprintf("%s_%s_%s", platform, url, breaker_name)
+		if m[key] {
+			continue
+		}
+		m[key] = true
+		id := AddPathCodeId()
+
+		Ch.Insert("path_page_code", map[string]interface{}{
+			"id":          id,
+			"platform":    platform,
+			"module":      module,
+			"url":         url,
+			"url_name":    url_name,
+			"url_element": breaker_name,
+		})
+	}
+}
+
+// LoadPageCode 加载页面代码表
+func LoadPageCode() map[string]int64 {
+	m := map[string]int64{}
+	q := fmt.Sprintf(`select id,platform,module,url,url_name,url_element
+								from userbehavior.path_page_code`)
+	ctx := context.Background()
+	rows, err := Ch_userbehavior.Query(ctx, q)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var (
+			id                                           uint64
+			platform, module, url, url_name, url_element string
+		)
+		if err := rows.Scan(
+			&id, &platform, &module, &url, &url_name, &url_element,
+		); err != nil {
+			log.Fatal(err)
+		}
+		key := fmt.Sprintf("%s_%s_%s", platform, url, url_element)
+		if m[key] > 0 {
+			continue
+		}
+		m[key] = int64(id)
+	}
+	return m
+}
+
+func GetPathBehavior(pathArr []string, url string) []string {
+	if len(pathArr) == 0 {
+		return []string{url}
+	}
+	newarr := getLastElements(pathArr, 5)
+	m := ArrToMap(newarr)
+	//前面几个路径没有,追加路径
+	if !m[url] {
+		pathArr = append(pathArr, url)
+	}
+	return pathArr
+}
+
+// getLastElements 获取最后几个数组
+func getLastElements(arr []string, count int) []string {
+	length := len(arr)
+	if length <= count {
+		// 如果数组长度不足X个，直接返回整个数组
+		return arr
+	}
+	// 返回数组的最后X元素
+	return arr[length-count:]
+}
+
+func ArrToMap(arr []string) map[string]bool {
+	m := map[string]bool{}
+	for _, v := range arr {
+		m[v] = true
+	}
+	return m
+}
+
+// SavePathBehavior 路径存储
+func SavePathBehavior(path string) int64 {
+	data := Ch.SelectBySql(`select id from userbehavior.user_behavior_path where path =?`, path)
+	pid := int64(0)
+	if data != nil && len(*data) > 0 {
+		id := qu.Int64All((*data)[0]["id"])
+		return id
+	} else {
+		udata := Ch.SelectBySql(`select id from userbehavior.path_behavior_id order by id desc limit 1`)
+		savemap := map[string]interface{}{
+			"path": path,
+		}
+		if udata != nil && len(*udata) > 0 {
+			id := qu.Int64All((*udata)[0]["id"])
+			savemap["id"] = id + 1
+			pid = id + 1
+		} else {
+			savemap["id"] = 1
+			pid = 1
+		}
+		Ch.Insert("user_behavior_path", savemap)
+		Ch.Insert("path_behavior_id", map[string]interface{}{"id": pid})
+	}
+	return pid
+}

save.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"fmt"
+	"github.com/gogf/gf/v2/frame/g"
+	"log"
+	"time"
+)
+
+var (
+	StrategyInfo *Strategy
+)
+
+func init() {
+	StrategyInfo = &Strategy{
+		Name:      Sysconfig.CacheChanInfo.Name,
+		DataSize:  Sysconfig.CacheChanInfo.DataSize,  //#数据缓存通道大小
+		BulkSize:  Sysconfig.CacheChanInfo.BulkSize,  //#批次- 每批的数量
+		TimeAfter: Sysconfig.CacheChanInfo.TimeAfter, //# 定时保存 毫秒
+		Timeout:   Sysconfig.CacheChanInfo.Timeout,   //# 缓存通道满时，超时丢弃
+		SCSize:    Sysconfig.CacheChanInfo.SCSize,    //# 数据库并发存储数据
+		CacheData: make(chan *UserBehaviorLog, Sysconfig.CacheChanInfo.DataSize),
+		SC:        make(chan bool, Sysconfig.CacheChanInfo.SCSize),
+	}
+}
+
+// InitInsertDataStrategy 数据存库
+func InitInsertDataStrategy() {
+	arr := make([]*UserBehaviorLog, StrategyInfo.BulkSize)
+	index := 0
+	timer := time.NewTimer(time.Duration(StrategyInfo.TimeAfter) * time.Millisecond)
+	defer timer.Stop()
+	for {
+		timer.Reset(time.Duration(StrategyInfo.TimeAfter) * time.Millisecond)
+		select {
+		case v := <-StrategyInfo.CacheData:
+			arr[index] = v
+			index++
+			if index == StrategyInfo.BulkSize {
+				StrategyInfo.SC <- true
+				go func(cacheData []*UserBehaviorLog) {
+					defer func() {
+						<-StrategyInfo.SC
+					}()
+					// 存储数据
+					BatchExecInsert(cacheData)
+
+				}(arr)
+				arr = make([]*UserBehaviorLog, StrategyInfo.BulkSize)
+				index = 0
+			}
+		case <-timer.C:
+			if index > 0 {
+				StrategyInfo.SC <- true
+				go func(cacheData []*UserBehaviorLog) {
+					defer func() {
+						<-StrategyInfo.SC
+					}()
+					// 存储数据
+					BatchExecInsert(cacheData)
+
+				}(arr[:index])
+				arr = make([]*UserBehaviorLog, StrategyInfo.BulkSize)
+				index = 0
+			}
+		}
+	}
+}
+
+// BatchExecInsert 批量 定时保存
+func BatchExecInsert(in []*UserBehaviorLog) (err error) {
+	if len(in) == 0 {
+		err = fmt.Errorf("data empty")
+		return
+	}
+	insertData := []map[string]interface{}{}
+
+	for _, i := range in {
+
+		insertData = append(insertData, map[string]interface{}{
+			"user_id":           i.UserID,           // 用户ID
+			"action_id":         i.ActionID,         // 动作ID
+			"trusted_id":        i.TrustedID,        // 浏览器指纹
+			"ip":                i.IP,               // 用户IP地址
+			"area":              i.Area,             // 省份
+			"city":              i.City,             // 城市
+			"os":                i.OS,               // 操作系统
+			"browser":           i.Browser,          // 浏览器类型
+			"browser_version":   i.BrowserVersion,   // 浏览器版本
+			"date":              i.Date,             // 行为发生时间
+			"platform":          i.Platform,         // 平台
+			"subsystem":         i.Subsystem,        // 子系统
+			"module":            i.Module,           // 模块
+			"url":               i.URL,              // 访问的URL
+			"url_name":          i.URLName,          // URL名称
+			"url_element":       i.URLElement,       // 页面元素
+			"add_field":         i.AddField,         // 附加数据
+			"time_stamp":        i.TimeStamp,        // 时间戳
+			"refer":             i.Refer,            // 引用来源
+			"refer_name":        i.ReferName,        // 引用名称
+			"refer_type":        i.ReferType,        // 引用类型
+			"session_id":        i.SessionID,        // 会话ID
+			"session_starttime": i.SessionStartTime, // 会话开始时间
+			"phone":             i.Phone,            // 手机号码
+			"page_name":         i.PageName,         // 页面名称
+			"desc":              i.Desc,             // 描述
+			"break_data":        i.BreakData,        // 额外数据
+			"breaker_name":      i.BreakerName,      //埋点名称
+			"data_type":         i.DataType,
+			"page_code_id":      i.PageCodeId,
+			"position_id":       i.PositionId,
+		})
+	}
+	_, err = g.DB().Model("userbehavior.user_behavior_log").Data(insertData).Insert()
+
+	if err != nil {
+		log.Println("batch insert false， --err:", err)
+		return
+	}
+	return
+}

session.go

@@ -0,0 +1,456 @@
+package main
+
+import (
+	qu "app.yhyue.com/moapp/jybase/common"
+	"app.yhyue.com/moapp/jybase/date"
+	"context"
+	"fmt"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/util/gconv"
+	"log"
+	"strings"
+	"sync"
+	"time"
+)
+
+type SessionLog struct {
+	User_id        string
+	Session_id     string
+	Start_time     time.Time
+	End_time       time.Time
+	Duration       string
+	In_platform    string
+	In_module      string
+	In_refer       string
+	In_refer_name  string
+	In_refer_type  string
+	In_url         string
+	In_url_name    string
+	In_url_element string
+	Out_platform   string
+	Out_module     string
+	Out_url        string
+	out_url_name   string
+	IsActive       bool
+	Path           string
+	PathArr        []string
+}
+
+type SessionDetaiLog struct {
+	User_id     string
+	Session_id  string
+	Create_time time.Time
+	Duration    string
+	Platform    string
+	Module      string
+	Count       int32
+}
+
+// UserSessions 用户会话映射
+type UserSessions struct {
+	sync.RWMutex
+	sessions map[string]*SessionLog
+}
+
+// UserSessions 用户会话映射
+type UserSessionsDetails struct {
+	sync.RWMutex
+	sessionsDetails map[string]*SessionDetaiLog
+}
+
+// Session 获取会话
+func Session(st, et int64) {
+	//初始化页码code
+	pageCodeMap := LoadPageCode()
+	//先加载未结束的会话
+	userSessionMap := LoadUserSession(st, et)
+	//加载虚拟id
+	userTrustedMap := LoadUserTrustedId(st, et)
+	//初始化用户会话明细映射
+	// 初始化用户会话映射
+	userSessionDetails := &UserSessionsDetails{
+		sessionsDetails: make(map[string]*SessionDetaiLog),
+	}
+
+	modelMap := MatchRegexModule()
+	starttime := time.Unix(st, 0).Format(date.Date_Full_Layout)
+	endtime := time.Unix(et, 0).Format(date.Date_Full_Layout)
+	//加载用户行为表
+	q := fmt.Sprintf(`select date,user_id,trusted_id,platform,module,url,url_name,url_element,refer,refer_name,refer_type,session_id,page_name,breaker_name,data_type,action_id
+								from userbehavior.user_behavior_log where 
+								date >='%s' and date <'%s'`, starttime, endtime)
+	// and user_id ='67d7e09285045c492114c634'
+	ctx := context.Background()
+	//log.Println(q)
+	rows, err := Ch_userbehavior.Query(ctx, q)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var (
+			date                                                                                                                                             time.Time
+			user_id, trusted_id, platform, module, url, url_name, url_element, refer, refer_name, refer_type, session_id, page_name, breaker_name, data_type string
+			action_id                                                                                                                                        int64
+		)
+		if err := rows.Scan(
+			&date, &user_id, &trusted_id, &platform, &module, &url, &url_name, &url_element, &refer, &refer_name, &refer_type, &session_id, &page_name, &breaker_name, &data_type, &action_id,
+		); err != nil {
+			log.Fatal(err)
+		}
+		//处理会话逻辑
+		HandleSession(pageCodeMap, userSessionDetails, userTrustedMap, userSessionMap, date, user_id, trusted_id, platform, module, url, url_name, url_element, refer, refer_name, refer_type, session_id, page_name, breaker_name, data_type, action_id, modelMap)
+	}
+	//清理过期session
+	CleanupSessions(userSessionMap)
+
+	//存储
+	StoreSessions(userSessionMap, userSessionDetails)
+}
+
+// GetSessionId 获取sessionid
+func GetSessionId(platform, user_id string, date time.Time) string {
+	timestr := date.Format("20060102150405")
+	return fmt.Sprintf("%s_%s_%s", platform, user_id, timestr)
+}
+
+// CalculateMinutesDifference 计算两个时间点之间的分钟差
+func CalculateMinutesDifference(t1, t2 time.Time) int {
+	// 计算两个时间之间的差值
+	duration := t2.Sub(t1)
+	// 将时间差转换为分钟
+	minutes := int(duration.Minutes())
+	return minutes
+}
+
+func HandleSession(pageCodeMap map[string]int64, sessionDetails *UserSessionsDetails, userTrustedMap map[string]string, sessions *UserSessions, date1 time.Time,
+	user_id, trusted_id, platform, module, url, url_name, url_element, refer, refer_name, refer_type, session_id, page_name, breaker_name, data_type string, action_id int64, modelMap *CompiledRegexMap) {
+	if user_id == "" && trusted_id == "" {
+		return
+	}
+	if user_id == "" && trusted_id != "" {
+		user_id = userTrustedMap[trusted_id]
+	}
+	if user_id == "" {
+		return
+	}
+
+	keyname := fmt.Sprintf("%s_%s", platform, user_id)
+	sessions.RLock()
+	session, exists := sessions.sessions[keyname]
+	sessions.RUnlock()
+	sessId := GetSessionId(platform, user_id, date1)
+
+	id := getPageCodeId(platform, url, breaker_name, module, url_name, action_id, pageCodeMap, modelMap)
+	// 如果用户没有活跃会话，则创建新会话
+	if !exists || !session.IsActive {
+		patharr := []string{}
+		if data_type == "c_jy_open_page" {
+			//patharr = GetPathBehavior([]string{}, gconv.String(pageCodeMap[key]))
+			patharr = GetPathBehavior([]string{}, gconv.String(id))
+		}
+		newSession := &SessionLog{
+			User_id:        user_id,
+			Session_id:     sessId,
+			Start_time:     date1,
+			End_time:       date1,
+			Duration:       "0",
+			In_platform:    platform,
+			In_module:      module,
+			In_refer:       refer,
+			In_refer_name:  refer_name,
+			In_refer_type:  refer_type,
+			In_url:         url,
+			In_url_name:    url_name,
+			In_url_element: url_element,
+			Out_platform:   platform,
+			Out_module:     module,
+			Out_url:        url,
+			out_url_name:   url_name,
+			IsActive:       true,
+			PathArr:        patharr,
+		}
+
+		sessions.Lock()
+		sessions.sessions[keyname] = newSession
+		sessions.Unlock()
+	} else {
+		// 计算时间差
+		timeDiff := date1.Sub(session.End_time)
+		if timeDiff > time.Duration(Sysconfig.TimeOut)*time.Minute {
+			patharr := []string{}
+			if data_type == "c_jy_open_page" {
+				//patharr = GetPathBehavior([]string{}, gconv.String(pageCodeMap[key]))
+				patharr = GetPathBehavior([]string{}, gconv.String(id))
+			}
+			// 如果时间差超过60分钟，创建新会话
+			newSession := &SessionLog{
+				User_id:        user_id,
+				Session_id:     sessId,
+				Start_time:     date1,
+				End_time:       date1,
+				Duration:       "0",
+				In_platform:    platform,
+				In_module:      module,
+				In_refer:       refer,
+				In_refer_name:  refer_name,
+				In_refer_type:  refer_type,
+				In_url:         url,
+				In_url_name:    url_name,
+				In_url_element: url_element,
+				Out_platform:   platform,
+				Out_module:     module,
+				Out_url:        url,
+				out_url_name:   url_name,
+				IsActive:       true,
+				PathArr:        patharr,
+			}
+
+			sessions.Lock()
+			sessions.sessions[keyname] = newSession
+			sessions.Unlock()
+		} else {
+
+			endtime := session.End_time
+			if endtime.Before(date1) {
+				endtime = date1
+			}
+			duration := endtime.Sub(session.Start_time)
+
+			// 否则更新当前会话
+			sessions.Lock()
+			session.End_time = endtime
+			session.Out_platform = platform
+			session.Out_module = module
+			session.Out_url = url
+			session.out_url_name = url_name
+			session.Duration = duration.String()
+			//session.PathArr = append(session.PathArr, gconv.String(pageCodeMap[key]))
+			if data_type == "c_jy_open_page" {
+				//session.PathArr = GetPathBehavior(session.PathArr, gconv.String(pageCodeMap[key]))
+				session.PathArr = GetPathBehavior(session.PathArr, gconv.String(id))
+			}
+			sessions.Unlock()
+		}
+	}
+
+	//
+	sessdetailkey := fmt.Sprintf("%s_%s", sessions.sessions[keyname].Session_id, module)
+
+	sessionDetails.RLock()
+	sessdetal, exists := sessionDetails.sessionsDetails[sessdetailkey]
+	sessionDetails.RUnlock()
+	if !exists {
+		//新增
+		newSessDetail := &SessionDetaiLog{
+			User_id:     user_id,
+			Session_id:  sessions.sessions[keyname].Session_id,
+			Create_time: sessions.sessions[keyname].Start_time,
+			Duration:    sessions.sessions[keyname].Duration,
+			Platform:    platform,
+			Module:      module,
+			Count:       1,
+		}
+		sessionDetails.Lock()
+		sessionDetails.sessionsDetails[sessdetailkey] = newSessDetail
+		sessionDetails.Unlock()
+	} else {
+		//更新
+		sessionDetails.Lock()
+		sessdetal.Count++
+		sessdetal.Duration = sessions.sessions[keyname].Duration
+		sessionDetails.Unlock()
+	}
+}
+
+// LoadUserSession 加载未结束会话
+func LoadUserSession(st, et int64) *UserSessions {
+	// 初始化用户会话映射
+	userSessions := &UserSessions{
+		sessions: make(map[string]*SessionLog),
+	}
+	starttime := time.Unix(st, 0).Add(-time.Duration(Sysconfig.TimeOut) * time.Minute).Format(date.Date_Full_Layout)
+	endtime := time.Unix(et, 0).Add(-time.Duration(Sysconfig.TimeOut) * time.Minute).Format(date.Date_Full_Layout)
+	q := fmt.Sprintf(`select user_id,session_id,start_time,end_time,duration,in_platform,in_subsystem,
+       					in_module,in_refer,in_refer_name,in_refer_type,in_url,in_url_name,in_url_element,
+       					out_platform,out_subsystem,out_module,out_url,out_url_name,path 
+								from userbehavior.user_session_log where end_time >='%s' and end_time <'%s'`, starttime, endtime)
+
+	ctx := context.Background()
+	rows, err := Ch_userbehavior.Query(ctx, q)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var (
+			start_time, end_time                                                                                                                                                                                                   time.Time
+			user_id, session_id, duration, in_platform, in_subsystem, in_module, in_refer, in_refer_name, in_refer_type, in_url, in_url_name, in_url_element, out_platform, out_subsystem, out_module, out_url, out_url_name, path string
+		)
+
+		if err := rows.Scan(
+			&user_id, &session_id, &start_time, &end_time, &duration, &in_platform, &in_subsystem, &in_module, &in_refer, &in_refer_name, &in_refer_type, &in_url, &in_url_name, &in_url_element, &out_platform, &out_subsystem, &out_module, &out_url, &out_url_name, &path,
+		); err != nil {
+			log.Fatal(err)
+		}
+		if path != "" {
+			da := Ch.SelectBySql(`select path from userbehavior.user_behavior_path where id =? limit 1`, path)
+			if da != nil && len(*da) > 0 {
+				path = qu.ObjToString((*da)[0]["path"])
+			}
+		}
+		userSessions.Lock()
+		userSessions.sessions[fmt.Sprintf("%s_%s", in_platform, user_id)] = &SessionLog{
+			User_id:        user_id,
+			Session_id:     session_id,
+			Start_time:     start_time,
+			End_time:       end_time,
+			Duration:       duration,
+			In_platform:    in_platform,
+			In_module:      in_module,
+			In_refer:       in_refer,
+			In_refer_name:  in_refer_name,
+			In_refer_type:  in_refer_type,
+			In_url:         in_url,
+			In_url_name:    in_url_name,
+			In_url_element: in_url_element,
+			Out_platform:   out_platform,
+			Out_module:     out_module,
+			Out_url:        out_url,
+			out_url_name:   out_url_name,
+			IsActive:       true,
+			PathArr:        strings.Split(path, "_"),
+		}
+		userSessions.Unlock()
+	}
+	return userSessions
+}
+
+// CleanupSessions 清理过期的会话
+func CleanupSessions(sessions *UserSessions) {
+	sessions.RLock()
+	defer sessions.RUnlock()
+	// 当前时间
+	now := time.Now()
+	// 遍历并清理过期会话
+	for userID, session := range sessions.sessions {
+		// 如果会话不活跃，并且最后更新时间超过24小时
+		if !session.IsActive && now.Sub(session.End_time) > time.Duration(Sysconfig.TimeOut)*time.Minute {
+			sessions.Lock()
+			delete(sessions.sessions, userID)
+			sessions.Unlock()
+		}
+	}
+}
+
+// StoreSessions 存储会话数据到 ClickHouse
+func StoreSessions(sessions *UserSessions, sessionDetails *UserSessionsDetails) {
+	sessions.RLock()
+	defer sessions.RUnlock()
+	ctx := context.Background()
+	// 遍历所有用户会话
+	c := 0
+	for _, session := range sessions.sessions {
+		if session.IsActive {
+			c++
+			if c%1000 == 0 {
+				time.Sleep(time.Second * 3)
+			}
+			//var results
+			//Ch_userbehavior.Select(ctx, query)
+			count := Ch.CountBySql(`select count(1) from user_session_log where session_id =?`, session.Session_id)
+			if count > 0 {
+				//修改user_session_log
+				query := fmt.Sprintf(`ALTER TABLE user_session_log 
+					UPDATE out_platform='%s' , out_module ='%s' , 
+					out_url ='%s' , out_url_name='%s' ,
+					end_time ='%v' , duration ='%s' ,path='%s'
+					WHERE session_id = '%s'`, session.Out_platform, session.Out_module, session.Out_url,
+					session.out_url_name, session.End_time.Format(date.Date_Full_Layout), session.Duration, gconv.String(SavePathBehavior(strings.Join(session.PathArr, "_"))),
+					session.Session_id)
+				err := Ch_userbehavior.Exec(ctx, query)
+				if err != nil {
+					log.Println(session.Session_id, "更新失败", err)
+				}
+				//修改user_session_detail_log
+
+			} else {
+				//不存在会话 创建
+				insertData := map[string]interface{}{
+					"user_id":        session.User_id,
+					"session_id":     session.Session_id,
+					"start_time":     session.Start_time,
+					"end_time":       session.End_time,
+					"duration":       session.Duration,
+					"in_platform":    session.In_platform,
+					"in_subsystem":   "",
+					"in_module":      session.In_module,
+					"in_refer":       session.In_refer,
+					"in_refer_name":  session.In_refer_name,
+					"in_refer_type":  session.In_refer_type,
+					"in_url":         session.In_url,
+					"in_url_name":    session.In_url_name,
+					"in_url_element": session.In_url_element,
+					"out_platform":   session.Out_platform,
+					"out_subsystem":  "",
+					"out_module":     session.Out_module,
+					"out_url":        session.Out_url,
+					"out_url_name":   session.out_url_name,
+					"path":           gconv.String(SavePathBehavior(strings.Join(session.PathArr, "_"))),
+				}
+				_, err := g.DB().Model("userbehavior.user_session_log").Data(insertData).Insert()
+				if err != nil {
+					log.Println("session_log存储失败", err)
+				}
+			}
+		}
+		//修改归集表
+		query := fmt.Sprintf(`ALTER TABLE user_behavior_log 
+					UPDATE session_id='%s' , session_starttime ='%s'
+					WHERE date >= '%s' and date <'%s' and user_id ='%s' and platform ='%s'`, session.Session_id, session.Start_time.Format(date.Date_Full_Layout), session.Start_time.Format(date.Date_Full_Layout), session.End_time.Format(date.Date_Full_Layout), session.User_id, session.In_platform)
+		err := Ch_userbehavior.Exec(ctx, query)
+		if err != nil {
+			log.Println("详情更新失败", err)
+		}
+	}
+
+	sessionDetails.RLock()
+	defer sessionDetails.RUnlock()
+	c2 := 0
+	for _, detail := range sessionDetails.sessionsDetails {
+		c2++
+		if c2%1000 == 0 {
+			time.Sleep(time.Second * 3)
+		}
+		ddata := Ch.SelectBySql(`select count from user_session_detail_log 
+             where session_id =? and module = ?`, detail.Session_id, detail.Module)
+		if ddata != nil && len(*ddata) > 0 {
+			c := qu.IntAll((*ddata)[0]["count"])
+			//修改user_session_log
+			query := fmt.Sprintf(`ALTER TABLE user_session_detail_log 
+					UPDATE duration='%s' , count ='%v'
+					WHERE session_id = '%s' and module ='%s'`, detail.Duration, detail.Count+int32(c), detail.Session_id, detail.Module)
+			err := Ch_userbehavior.Exec(ctx, query)
+			if err != nil {
+				log.Println(detail.Session_id, "详情更新失败", err)
+			}
+		} else {
+			//不存在会话 创建
+			insertData := map[string]interface{}{
+				"user_id":     detail.User_id,
+				"session_id":  detail.Session_id,
+				"create_time": detail.Create_time,
+				"duration":    detail.Duration,
+				"platform":    detail.Platform,
+				"subsystem":   "",
+				"module":      detail.Module,
+				"count":       detail.Count,
+			}
+			_, err := g.DB().Model("userbehavior.user_session_detail_log").Data(insertData).Insert()
+			if err != nil {
+				log.Println("user_session_detail_log存储失败", err)
+			}
+		}
+	}
+	log.Println("处理数据结束")
+}

sql.sql

@@ -0,0 +1,252 @@
+CREATE TABLE userbehavior.action
+(
+    `id` UInt64 COMMENT '唯一标识符',
+    `name` Nullable(String) DEFAULT NULL COMMENT '动作id'
+)
+    ENGINE = MergeTree
+ORDER BY id
+SETTINGS index_granularity = 8192
+COMMENT '动作表'
+
+INSERT INTO userbehavior.`action` (id,name) VALUES
+	 (1,'访问页面'),
+	 (2,'点击页面元素'),
+	 (3,'登录'),
+	 (4,'注册'),
+	 (5,'注销'),
+	 (6,'下单'),
+	 (7,'绑定手机号'),
+	 (8,'支付');
+
+
+CREATE TABLE userbehavior.user_behavior_log
+(
+    `user_id` Nullable(String) DEFAULT NULL COMMENT '用户ID',
+    `action_id` Nullable(Int64) DEFAULT NULL COMMENT '动作ID',
+    `trusted_id` Nullable(String) DEFAULT NULL COMMENT '浏览器指纹',
+    `ip` Nullable(String) DEFAULT NULL COMMENT '用户IP地址',
+    `area` Nullable(String) DEFAULT NULL COMMENT '省份',
+    `city` Nullable(String) DEFAULT NULL COMMENT '城市',
+    `os` Nullable(String) DEFAULT NULL COMMENT '操作系统',
+    `browser` Nullable(String) DEFAULT NULL COMMENT '浏览器类型',
+    `browser_version` Nullable(String) DEFAULT NULL COMMENT '浏览器版本',
+    `date` DateTime COMMENT '行为发生时间',
+    `platform` Nullable(String) DEFAULT NULL COMMENT '平台（如移动端、PC端）',
+    `subsystem` Nullable(String) DEFAULT NULL COMMENT '子系统',
+    `module` Nullable(String) DEFAULT NULL COMMENT '模块',
+    `url` Nullable(String) DEFAULT NULL COMMENT '访问的URL',
+    `url_name` Nullable(String) DEFAULT NULL COMMENT 'URL名称',
+    `url_element` Nullable(String) DEFAULT NULL COMMENT '页面元素',
+    `add_field` Nullable(String) DEFAULT NULL COMMENT '附加数据',
+    `time_stamp` Nullable(Int32) DEFAULT NULL COMMENT '时间戳',
+    `refer` Nullable(String) DEFAULT NULL COMMENT '引用来源',
+    `refer_name` Nullable(String) DEFAULT NULL COMMENT '引用名称',
+    `refer_type` Nullable(String) DEFAULT NULL COMMENT '引用类型',
+    `session_id` Nullable(String) DEFAULT NULL COMMENT '会话ID',
+    `session_starttime` Nullable(DateTime) DEFAULT NULL COMMENT '会话开始时间',
+    `phone` Nullable(String) DEFAULT NULL COMMENT '手机号码',
+    `page_name` Nullable(String) DEFAULT NULL COMMENT '页面名称',
+    `desc` Nullable(String) DEFAULT NULL COMMENT '描述',
+    `break_data` Nullable(String) DEFAULT NULL COMMENT '额外数据'
+)
+    ENGINE = MergeTree
+ORDER BY date
+TTL date + INTERVAL 6 MONTH
+SETTINGS index_granularity = 8192
+COMMENT '用户行为归集表'
+
+
+CREATE TABLE userbehavior.user_session_detail_log
+(
+    `user_id` Nullable(String) DEFAULT NULL COMMENT '用户ID',
+    `session_id` String COMMENT '会话id',
+    `create_time` DateTime  COMMENT '日期',
+    `duration` Nullable(String) DEFAULT NULL COMMENT '会话时长',
+    `platform` Nullable(String) DEFAULT NULL COMMENT '平台（如移动端、PC端）',
+    `subsystem` Nullable(String) DEFAULT NULL COMMENT '子系统',
+    `module` Nullable(String) DEFAULT NULL COMMENT '模块',
+    `count` Nullable(Int32) DEFAULT NULL COMMENT '访问次数'
+)
+    ENGINE = MergeTree
+ORDER BY session_id
+TTL create_time + INTERVAL 6 MONTH
+SETTINGS index_granularity = 8192
+COMMENT '用户会话（明细表）'
+
+
+
+CREATE TABLE userbehavior.user_session_log
+(
+    `user_id` Nullable(String) DEFAULT NULL COMMENT '用户ID',
+    `session_id` String COMMENT '会话id',
+    `start_time` DateTime COMMENT '会话开始时间',
+    `end_time` Nullable(DateTime) DEFAULT NULL COMMENT '会话结束时间',
+    `duration` Nullable(String) DEFAULT NULL COMMENT '会话时长',
+    `in_platform` Nullable(String) DEFAULT NULL COMMENT '平台（如移动端、PC端）',
+    `in_subsystem` Nullable(String) DEFAULT NULL COMMENT '子系统',
+    `in_module` Nullable(String) DEFAULT NULL COMMENT '模块',
+    `in_refer` Nullable(String) DEFAULT NULL COMMENT '引用来源',
+    `in_refer_name` Nullable(String) DEFAULT NULL COMMENT '引用名称',
+    `in_refer_type` Nullable(String) DEFAULT NULL COMMENT '引用类型',
+    `in_url` Nullable(String) DEFAULT NULL COMMENT '访问的U    RL',
+    `in_url_name` Nullable(String) DEFAULT NULL COMMENT 'URL名称',
+    `in_url_element` Nullable(String) DEFAULT NULL COMMENT '页面元素',
+    `out_platform` Nullable(String) DEFAULT NULL COMMENT '平台（如移动端、PC端）',
+    `out_subsystem` Nullable(String) DEFAULT NULL COMMENT '子系统',
+    `out_module` Nullable(String) DEFAULT NULL COMMENT '模块',
+    `out_url` Nullable(String) DEFAULT NULL COMMENT '访问的URL',
+    `out_url_name` Nullable(String) DEFAULT NULL COMMENT 'URL名称'
+)
+ENGINE = MergeTree
+ORDER BY session_id
+TTL start_time + INTERVAL 6 MONTH
+SETTINGS index_granularity = 8192
+COMMENT '用户会话（主表）'
+--创建数据库
+--CREATE DATABASE IF NOT EXISTS userbehavior;
+
+
+CREATE TABLE userbehavior.user_behavior_log (
+	id  UInt64 NOT NULL,
+	user_id Nullable(String) DEFAULT NULL,  --用户id
+	action_id  Nullable(Int64) DEFAULT NULL, --动作id
+	trusted_id Nullable(String) DEFAULT NULL, --浏览器指纹
+	ip Nullable(String) DEFAULT NULL,   --ip
+	area Nullable(String) DEFAULT NULL,  --省份
+	city Nullable(String) DEFAULT NULL,  --城市
+	os  Nullable(String) DEFAULT NULL,  --操作系统
+	browser Nullable(String) DEFAULT NULL,
+	browser_version Nullable(String) DEFAULT NULL,
+	`date` DateTime NOT NULL,
+	platform Nullable(String) DEFAULT NULL,  --平台
+	subsystem Nullable(String) DEFAULT NULL,  --子系统
+	module Nullable(String) DEFAULT NULL, --模块
+	url  Nullable(String) DEFAULT NULL,
+	url_name Nullable(DateTime) DEFAULT NULL,
+	add_field Nullable(String) DEFAULT NULL, --附加数据
+	time_stamp Nullable(Int32) DEFAULT NULL, --时间戳
+	refer Nullable(String) DEFAULT NULL,
+	refer_name Nullable(String) DEFAULT NULL,
+	refer_type Nullable(String) DEFAULT NULL,
+	session_id Nullable(String) DEFAULT NULL,
+	session_starttime Nullable(DateTime) DEFAULT NULL,
+	phone Nullable(String) default null,
+	page_name Nullable(String) DEFAULT NULL,
+	`desc` Nullable(String) DEFAULT NULL
+) ENGINE = MergeTree ORDER BY id COMMENT '用户行为归集表'  
+
+
+
+--DROP TABLE userbehavior.user_behavior_log 
+
+
+CREATE TABLE userbehavior.user_behavior_log (
+    id UInt64 NOT NULL COMMENT '唯一标识符',
+    user_id Nullable(String) DEFAULT NULL COMMENT '用户ID',  -- 用户id
+    action_id Nullable(Int64) DEFAULT NULL COMMENT '动作ID', -- 动作id
+    trusted_id Nullable(String) DEFAULT NULL COMMENT '浏览器指纹', -- 浏览器指纹
+    ip Nullable(String) DEFAULT NULL COMMENT '用户IP地址',   -- ip
+    area Nullable(String) DEFAULT NULL COMMENT '省份',  -- 省份
+    city Nullable(String) DEFAULT NULL COMMENT '城市',  -- 城市
+    os Nullable(String) DEFAULT NULL COMMENT '操作系统',  -- 操作系统
+    browser Nullable(String) DEFAULT NULL COMMENT '浏览器类型',
+    browser_version Nullable(String) DEFAULT NULL COMMENT '浏览器版本',
+    `date` DateTime NOT NULL COMMENT '行为发生时间',
+    platform Nullable(String) DEFAULT NULL COMMENT '平台（如移动端、PC端）',  -- 平台
+    subsystem Nullable(String) DEFAULT NULL COMMENT '子系统',  -- 子系统
+    module Nullable(String) DEFAULT NULL COMMENT '模块', -- 模块
+    url Nullable(String) DEFAULT NULL COMMENT '访问的URL',
+    url_name Nullable(String) DEFAULT NULL COMMENT 'URL名称',
+    url_element Nullable(String) DEFAULT NULL COMMENT '页面元素',
+    add_field Nullable(String) DEFAULT NULL COMMENT '附加数据', -- 附加数据
+    time_stamp Nullable(Int32) DEFAULT NULL COMMENT '时间戳', -- 时间戳
+    refer Nullable(String) DEFAULT NULL COMMENT '引用来源',
+    refer_name Nullable(String) DEFAULT NULL COMMENT '引用名称',
+    refer_type Nullable(String) DEFAULT NULL COMMENT '引用类型',
+    session_id Nullable(String) DEFAULT NULL COMMENT '会话ID',
+    session_starttime Nullable(DateTime) DEFAULT NULL COMMENT '会话开始时间',
+    phone Nullable(String) DEFAULT NULL COMMENT '手机号码',
+    page_name Nullable(String) DEFAULT NULL COMMENT '页面名称',
+    `desc` Nullable(String) DEFAULT NULL COMMENT '描述',
+    break_data Nullable(String) DEFAULT NULL COMMENT '额外数据'
+) ENGINE = MergeTree ORDER BY id COMMENT '用户行为归集表';
+
+
+
+
+CREATE TABLE userbehavior.user_session_log (
+    id UInt64 NOT NULL COMMENT '唯一标识符',
+    user_id Nullable(String) DEFAULT NULL COMMENT '用户ID',  -- 用户id
+    session_id Nullable(String) DEFAULT NULL COMMENT '会话id', -- 动作id
+    start_time Nullable(DateTime) DEFAULT NULL COMMENT '会话开始时间', -- 浏览器指纹
+    end_time Nullable(DateTime) DEFAULT NULL COMMENT '会话结束时间',   -- ip
+    duration Nullable(String) DEFAULT NULL COMMENT '会话时长',  -- 省份
+    city Nullable(String) DEFAULT NULL COMMENT '城市',  -- 城市
+    os Nullable(String) DEFAULT NULL COMMENT '操作系统',  -- 操作系统
+    browser Nullable(String) DEFAULT NULL COMMENT '浏览器类型',
+    browser_version Nullable(String) DEFAULT NULL COMMENT '浏览器版本',
+    `date` DateTime NOT NULL COMMENT '行为发生时间',
+    platform Nullable(String) DEFAULT NULL COMMENT '平台（如移动端、PC端）',  -- 平台
+    subsystem Nullable(String) DEFAULT NULL COMMENT '子系统',  -- 子系统
+    module Nullable(String) DEFAULT NULL COMMENT '模块', -- 模块
+    url Nullable(String) DEFAULT NULL COMMENT '访问的URL',
+    url_name Nullable(String) DEFAULT NULL COMMENT 'URL名称',
+    url_element Nullable(String) DEFAULT NULL COMMENT '页面元素',
+    add_field Nullable(String) DEFAULT NULL COMMENT '附加数据', -- 附加数据
+    time_stamp Nullable(Int32) DEFAULT NULL COMMENT '时间戳', -- 时间戳
+    refer Nullable(String) DEFAULT NULL COMMENT '引用来源',
+    refer_name Nullable(String) DEFAULT NULL COMMENT '引用名称',
+    refer_type Nullable(String) DEFAULT NULL COMMENT '引用类型',
+    session_id Nullable(String) DEFAULT NULL COMMENT '会话ID',
+    session_starttime Nullable(DateTime) DEFAULT NULL COMMENT '会话开始时间',
+    phone Nullable(String) DEFAULT NULL COMMENT '手机号码',
+    page_name Nullable(String) DEFAULT NULL COMMENT '页面名称',
+    `desc` Nullable(String) DEFAULT NULL COMMENT '描述',
+    break_data Nullable(String) DEFAULT NULL COMMENT '额外数据'
+) ENGINE = MergeTree ORDER BY id COMMENT '用户行为归集表';
+
+
+
+CREATE TABLE userbehavior.path_page_code
+(
+    `id` UInt64 NOT NULL COMMENT '唯一标识符',
+    `platform` String COMMENT '平台',
+    `module` String COMMENT '模块',
+    `url` String COMMENT '页面url',
+    `url_name` String COMMENT 'url名称',
+    `url_element` String COMMENT '元素',
+
+)
+    ENGINE = MergeTree
+ORDER BY id
+COMMENT '用户会话（页面维表）';
+
+
+
+
+CREATE TABLE userbehavior.path_code_id (
+                                           id Int64  NOT NULL COMMENT '维表id',
+) ENGINE = MergeTree ORDER BY id COMMENT '用户行为归集表';
+
+
+CREATE TABLE userbehavior.path_behavior_id (
+                                               id Int64  NOT NULL COMMENT '维表id',
+) ENGINE = MergeTree ORDER BY id COMMENT '用户行为路径id';
+
+
+CREATE TABLE userbehavior.user_behavior_path (
+                                                 id Int64 NOT NULL,
+                                                 `path` String NOT NULL
+)ENGINE = MergeTree
+ORDER BY path;
+
+
+
+
+ALTER TABLE userbehavior.user_behavior_log ADD COLUMN breaker_name String;
+
+ALTER TABLE userbehavior.path_page_code  ADD COLUMN action_id UInt64 ;
+
+ALTER TABLE userbehavior.user_behavior_log  ADD COLUMN page_code_id UInt64 ;
+ALTER TABLE userbehavior.user_behavior_log  ADD COLUMN data_type String  ;
+ALTER TABLE userbehavior.user_behavior_log  ADD COLUMN position_id String  ;

timetask.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"app.yhyue.com/moapp/jybase/date"
+	"github.com/robfig/cron"
+	"log"
+	"time"
+)
+
+func Run() {
+	if Sysconfig.TaskIsRun {
+		go behaviorTask()
+	}
+}
+
+// 定时检查是否有需要解读的文件 定时任务
+func behaviorTask() {
+	c := cron.New()
+	c.AddFunc(Sysconfig.LogTask, func() {
+		et := time.Now()
+		st := et.Add(-time.Minute * 60)
+		dwd_f_personnel_behavior(st.Unix(), et.Unix())
+		log.Println("dwd_f_personnel_behavior end")
+		jy_mongodb_log(st.Unix(), et.Unix())
+		log.Println("jy_mongodb_log end")
+		User(st.Unix(), et.Unix())
+		//pm := InitPositionId()
+		salesLeads_logs(st.Unix(), et.Unix())
+		Session(st.Unix(), et.Unix())
+		log.Println(st.Format(date.Date_Full_Layout), "behaviorTask end")
+	})
+	c.Start()
+	lock := make(chan bool)
+	<-lock
+}
+
+// 定时检查是否有需要解读的文件 定时任务
+func MongodbTask() {
+	c := cron.New()
+	c.AddFunc(Sysconfig.LogTask, func() {
+		et := time.Now()
+		st := et.Add(-time.Minute * 60)
+		jy_mongodb_log(st.Unix(), et.Unix())
+		log.Println("mongodb end")
+	})
+	c.Start()
+	lock := make(chan bool)
+	<-lock
+}
+
+func SessionTask() {
+	c := cron.New()
+	c.AddFunc(Sysconfig.LogTask, func() {
+		et := time.Now().Add(-time.Minute * 60)
+		st := et.Add(-time.Minute * 60)
+		Session(st.Unix(), et.Unix())
+		log.Println("SessionTask end")
+	})
+	c.Start()
+	lock := make(chan bool)
+	<-lock
+}

user.go

@@ -0,0 +1,358 @@
+package main
+
+import (
+	qu "app.yhyue.com/moapp/jybase/common"
+	"app.yhyue.com/moapp/jybase/date"
+	"app.yhyue.com/moapp/jybase/mongodb"
+	"app.yhyue.com/moapp/jybase/redis"
+	"fmt"
+	"log"
+	"strings"
+	"time"
+)
+
+// getUserTrustedId 获取mgo中的匿名id
+func getUserTrustedId(userid string) string {
+	sess := Mgo_Log.GetMgoConn()
+	defer Mgo_Log.DestoryMongoConn(sess)
+	query := map[string]interface{}{
+		"userid": userid,
+	}
+	it := sess.DB("qfw").C("jy_logs").Find(query).Select(map[string]interface{}{
+		"trustedId": 1,
+	}).Iter()
+	total := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		trustedId := qu.ObjToString(tmp["trustedId"])
+		if trustedId != "" {
+			return MgoTrustedIdToCkTrustedId(trustedId)
+		}
+		tmp = make(map[string]interface{})
+	}
+	return ""
+}
+
+func User(st, et int64) {
+	s, e := GetCurTimePiInfo(st, et)
+	query := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": mongodb.StringTOBsonId(s),
+			"$lt":  mongodb.StringTOBsonId(e),
+		},
+	}
+	sess := Mgo_Log.GetMgoConn()
+	defer Mgo_Log.DestoryMongoConn(sess)
+	it := sess.DB("qfw").C("register_log").Find(query).Sort("-_id").Select(map[string]interface{}{
+		"userid":      1,
+		"system":      1,
+		"source":      1,
+		"create_time": 1,
+		"phone":       1,
+	}).Iter()
+	total := 0
+
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		userid := qu.ObjToString(tmp["userid"])
+		source := qu.ObjToString(tmp["source"]) //pc wx app h5 entbase background vipgift
+		create_time := qu.ObjToString(tmp["create_time"])
+		t, _ := time.ParseInLocation(date.Date_Full_Layout, create_time, time.Local)
+		if source == "pc" {
+			if tid := getUserTrustedId(userid); tid != "" {
+				sql := fmt.Sprintf(`select * from userbehavior.user_behavior_log 
+         				where trusted_id='%s' and user_id ='' and date<='%s' limit 1`, tid, create_time)
+				data := Ch.SelectBySql(sql)
+				if data != nil && len(*data) > 0 {
+					d := (*data)[0]
+					ip := qu.ObjToString(d["ip"])
+					area := qu.ObjToString(d["area"])
+					city := qu.ObjToString(d["city"])
+					os := qu.ObjToString(d["os"])
+					browser := qu.ObjToString(d["browser"])
+					module := qu.ObjToString(d["module"])
+					href := qu.ObjToString(d["href"])
+					urlname := qu.ObjToString(d["url_name"])
+					refer := qu.ObjToString(d["refer"])
+					refer_name := qu.ObjToString(d["refer_name"])
+					refer_type := qu.ObjToString(d["refer_type"])
+					phone := qu.ObjToString(d["phone"])
+					urldesc := qu.ObjToString(d["desc"])
+					session_id := qu.ObjToString(d["session_id"])
+					page_code_id := qu.Int64All(d["page_code_id"])
+					position_id := qu.ObjToString(d["position_id"])
+					StrategyInfo.CacheData <- &UserBehaviorLog{
+						UserID:         userid,
+						ActionID:       int64(4),        // 动作ID
+						TrustedID:      tid,             // 浏览器指纹
+						IP:             ip,              // 用户IP地址
+						Area:           area,            // 省份
+						City:           city,            // 城市
+						OS:             os,              // 操作系统
+						Browser:        browser,         // 浏览器类型
+						BrowserVersion: "",              // 浏览器版本
+						Date:           t,               // 行为发生时间
+						Platform:       "pc",            // 平台（如移动端、PC端）
+						Subsystem:      "",              // 子系统
+						Module:         module,          // 模块
+						URL:            href,            // 访问的URL
+						URLName:        urlname,         // URL名称
+						URLElement:     "",              // 页面元素
+						AddField:       "",              // 附加数据
+						TimeStamp:      int32(t.Unix()), // 时间戳
+						Refer:          refer,           // 引用来源
+						ReferName:      refer_name,      // 引用名称
+						ReferType:      refer_type,      // 引用类型
+						SessionID:      session_id,      // 会话ID
+						Phone:          phone,           // 手机号码
+						PageName:       urlname,         // 页面名称
+						Desc:           urldesc,         // 描述
+						BreakData:      "",              // 额外数据
+						PageCodeId:     uint64(page_code_id),
+						PositionId:     position_id,
+					}
+				}
+			}
+		} else {
+			StrategyInfo.CacheData <- &UserBehaviorLog{
+				UserID:         userid,
+				ActionID:       int64(4),                     // 动作ID
+				TrustedID:      "",                           // 浏览器指纹
+				IP:             "",                           // 用户IP地址
+				Area:           "",                           // 省份
+				City:           "",                           // 城市
+				OS:             "",                           // 操作系统
+				Browser:        "",                           // 浏览器类型
+				BrowserVersion: "",                           // 浏览器版本
+				Date:           t,                            // 行为发生时间
+				Platform:       "",                           // 平台（如移动端、PC端）
+				Subsystem:      "",                           // 子系统
+				Module:         "",                           // 模块
+				URL:            "",                           // 访问的URL
+				URLName:        "",                           // URL名称
+				URLElement:     "",                           // 页面元素
+				AddField:       "",                           // 附加数据
+				TimeStamp:      int32(t.Unix()),              // 时间戳
+				Refer:          "",                           // 引用来源
+				ReferName:      "",                           // 引用名称
+				ReferType:      "",                           // 引用类型
+				SessionID:      "",                           // 会话ID
+				Phone:          qu.ObjToString(tmp["phone"]), // 手机号码
+				PageName:       "",                           // 页面名称
+				Desc:           "",                           // 描述
+				BreakData:      "",                           // 额外数据
+				PageCodeId:     0,
+				PositionId:     "",
+			}
+		}
+		tmp = make(map[string]interface{})
+	}
+}
+
+func salesLeads_logs(st, et int64) {
+	userPhoneMap := GetPhone()
+	modelMap := MatchRegexModule()
+
+	s, e := GetCurTimePiInfo(st, et)
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": mongodb.StringTOBsonId(s),
+			"$lt":  mongodb.StringTOBsonId(e),
+		},
+		"url": "/salesLeads/retainedCapital",
+	}
+	log.Println(q)
+	sess := Mgo_Log.GetMgoConn()
+	defer Mgo_Log.DestoryMongoConn(sess)
+	it := sess.DB("qfw").C("salesLeads_logs").Find(q).Sort("-_id").Select(map[string]interface{}{
+		"userid":     1,
+		"ip":         1,
+		"refer":      1,
+		"client":     1,
+		"url":        1,
+		"createtime": 1,
+		"date":       1,
+		"trustedId":  1,
+		"os":         1,
+		"browse":     1,
+		"positionId": 1,
+	}).Iter()
+	total := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%50000 == 0 {
+			log.Println("cur saleLeads index ", total)
+		}
+		userid := qu.ObjToString(tmp["userid"])
+		client := qu.ObjToString(tmp["client"])
+		href := qu.ObjToString(tmp["url"])
+
+		trustedId := qu.ObjToString(tmp["trustedId"])
+		ip := qu.ObjToString(tmp["ip"])
+		area, city := GetIpSource(ip)
+		os := qu.ObjToString(tmp["os"])
+		browse := qu.ObjToString(tmp["browse"])
+
+		refer := qu.ObjToString(tmp["refer"])
+
+		i_createtime := qu.Int64All(tmp["date"])
+		createtime := time.Unix(i_createtime, 0)
+
+		platform := ""
+		if strings.Contains(refer, "www.jianyu") {
+			platform = "pc"
+		} else if strings.Contains(refer, "wx.jianyu") {
+			platform = "wx"
+		} else {
+			if strings.Contains(client, "Android") {
+				platform = "android"
+			}
+			// 检查是否为 iOS 客户端
+			if strings.Contains(client, "iPhone") || strings.Contains(client, "iPad") || strings.Contains(client, "iPod") {
+				platform = "ios"
+			}
+		}
+
+		hrefinfo, _ := modelMap.MatchRegexMap(Unescape(href))
+		module := ""
+		if hrefinfo != nil {
+			module = hrefinfo.Module
+		}
+		referinfo, _ := modelMap.MatchRegexMap(Unescape(refer))
+		refermodule := ""
+		urlname := ""
+		urldesc := ""
+		if referinfo != nil {
+			refermodule = referinfo.Module
+			urlname = referinfo.Page_name
+			urldesc = referinfo.Desc
+		}
+		referType := GetReferType(refer)
+		positionid := qu.ObjToString(tmp["positionId"])
+		//if !mongodb.IsObjectIdHex(userid) {
+		//	userid = GetUserId(userid)
+		//	positionid = userid
+		//} else {
+		//	positionid = gconv.String(positionMap[userid])
+		//}
+
+		StrategyInfo.CacheData <- &UserBehaviorLog{
+			UserID:         userid,
+			ActionID:       12,                  // 动作ID
+			TrustedID:      trustedId,           // 浏览器指纹
+			IP:             ip,                  // 用户IP地址
+			Area:           area,                // 省份
+			City:           city,                // 城市
+			OS:             os,                  // 操作系统
+			Browser:        browse,              // 浏览器类型
+			BrowserVersion: "",                  // 浏览器版本
+			Date:           createtime,          // 行为发生时间
+			Platform:       platform,            // 平台（如移动端、PC端）
+			Subsystem:      "",                  // 子系统
+			Module:         module,              // 模块
+			URL:            href,                // 访问的URL
+			URLName:        urlname,             // URL名称
+			URLElement:     "",                  // 页面元素
+			AddField:       "",                  // 附加数据
+			TimeStamp:      int32(i_createtime), // 时间戳
+			Refer:          refer,               // 引用来源
+			ReferName:      refermodule,         // 引用名称
+			ReferType:      referType,           // 引用类型
+			SessionID:      "",                  // 会话ID
+			//SessionStartTime: time.Now(),           // 会话开始时间 //TODO 会话开始时间
+			Phone:      userPhoneMap[userid], // 手机号码
+			PageName:   urlname,              // 页面名称
+			Desc:       urldesc,              // 描述
+			BreakData:  "",                   // 额外数据
+			PositionId: positionid,
+		}
+		tmp = make(map[string]interface{})
+
+	}
+	log.Println("mongodb end")
+}
+
+// InitPositionId 通过用户归集获取职位id和mgoid的对应关系
+func InitPositionId() map[string]int64 {
+	positionIdMap := map[string]int64{}
+	DataService.SelectByBath(20, func(l *[]map[string]interface{}) bool {
+		for _, v := range *l {
+			user_id := qu.ObjToString(v["user_id"])
+			id := qu.Int64All(v["id"])
+			positionIdMap[user_id] = id
+		}
+		return true
+	}, fmt.Sprintf("SELECT userid,position_id FROM data_service.%s WHERE TYPE=0;", Sysconfig.Tidb.DataServiceCollName))
+	return positionIdMap
+}
+
+// LoadUserTrustedId 获取userid和虚拟id对应关系
+func LoadUserTrustedId(st, et int64) map[string]string {
+	userTrusted := map[string]string{}
+	starttime := time.Unix(st, 0).Format(date.Date_Full_Layout)
+	endtime := time.Unix(et, 0).Format(date.Date_Full_Layout)
+	Ch.SelectByBath(20, func(l *[]map[string]interface{}) bool {
+		for _, v := range *l {
+			userid := qu.ObjToString(v["user_id"])
+			trusted_id := qu.ObjToString(v["trusted_id"])
+			userTrusted[trusted_id] = userid
+		}
+		return true
+	}, fmt.Sprintf(`select user_id,trusted_id from user_behavior_log where date>='%s' and date< '%s' and user_id !='' and trusted_id!=''`, starttime, endtime))
+	return userTrusted
+}
+
+func GetPhone() map[string]string {
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{
+		"i_appid": 2,
+		"$or": []map[string]interface{}{
+			map[string]interface{}{"s_phone": map[string]interface{}{"$exists": true}},
+			map[string]interface{}{"s_m_phone": map[string]interface{}{"$exists": true}},
+		},
+	}
+	userPhoneMap := map[string]string{}
+
+	it := sess.DB("qfw").C("user").Find(q).Select(map[string]interface{}{
+		"userid":    1,
+		"s_phone":   1,
+		"s_m_phone": 1,
+	}).Iter()
+	total := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%50000 == 0 {
+			log.Println("cur index ", total)
+		}
+		userid := mongodb.BsonIdToSId(tmp["_id"])
+		phone := qu.ObjToString(tmp["s_phone"])
+		if phone == "" {
+			phone = qu.ObjToString(tmp["s_m_phone"])
+		}
+		if phone != "" {
+			userPhoneMap[userid] = phone
+		}
+
+		tmp = make(map[string]interface{})
+	}
+	return userPhoneMap
+}
+
+// 获取用户id
+func GetUserId(userid string) string {
+	keyname := fmt.Sprintf("position_%s", userid)
+	mgoUserid := redis.GetStr("useraction", keyname)
+	if mgoUserid != "" {
+		return mgoUserid
+	}
+	data := Base.SelectBySql(`select user_id from base_position where id =?`, userid)
+	if data != nil && len(*data) > 0 {
+		base_user_id := qu.Int64All((*data)[0]["user_id"])
+		udata, _ := Mgo.FindOne("user", map[string]interface{}{
+			"base_user_id": base_user_id,
+		})
+		if udata != nil && len(*udata) > 0 {
+			mgoUserid := mongodb.BsonIdToSId((*udata)["_id"])
+			redis.Put("useraction", keyname, mgoUserid, 3600*24)
+			return mgoUserid
+		}
+	}
+	return ""
+}

util.go

@@ -0,0 +1,659 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	qu "app.yhyue.com/moapp/jybase/common"
+	"app.yhyue.com/moapp/jybase/encrypt"
+	"app.yhyue.com/moapp/jybase/mongodb"
+	"github.com/gogf/gf/v2/frame/g"
+	excelize "github.com/xuri/excelize/v2"
+)
+
+// 解析ip归属
+func AnalyIp(ip string) (country, p, c, isp string) {
+	res, err := Reader.SearchByStr(ip)
+	if err != nil {
+		log.Printf("GeoIP lookup error: %v", err)
+		return
+	}
+	arr := strings.Split(res, "|")
+	if len(arr) > 3 {
+		return arr[0], arr[2], arr[3], arr[4]
+	} else {
+		return
+	}
+}
+
+// 初始化模块
+func Initmodule() {
+	m := map[string]string{
+		"/jy_mobile/common/order/create/svip":                                         "下单",
+		"/jy_mobile/common/order/create/areapack":                                     "下单",
+		"/page_entbase_mobile/page/business_management/people_management_branch.html": "工作台",
+		"/jyapp/front/dataExport/buyerEdit":                                           "工作台",
+		"/front/wx_dataExport/buyerEdit":                                              "工作台",
+		"/jyapp/big/page/unit_portrayal":                                              "画像",
+		"/swordfish/page_big_pc/unit_portrayal/":                                      "画像",
+		"/weixin/pay/checkout_buyerPortraitPack":                                      "支付",
+		"/jyapp/pay/checkout_buyerPortraitPack":                                       "支付",
+		"/jy_mobile/common/order/create/buyerpack":                                    "下单",
+		"/jy_mobile/search/middle/buyer":                                              "搜索",
+		"/jy_mobile/search/result/buyer":                                              "搜索",
+		"/front/invoice/check_invoice.html":                                           "订单",
+		"/jyapp/front/invoice/check_invoice.html":                                     "订单",
+		"/jyapp/vipsubscribe/toSubVipSetPage":                                         "订阅",
+		"/front/vipsubscribe/toSubVipSetPage":                                         "订阅",
+		"/weixin/pay/checkout_subvip":                                                 "支付",
+		"/jyapp/pay/checkout_subvip":                                                  "支付",
+		"/jy_mobile/common/order/create/datapack":                                     "下单",
+		"/weixin/pay/checkout_member":                                                 "支付",
+		"/jyapp/pay/checkout_member":                                                  "支付",
+		"/jy_mobile/common/order/create/bigmember":                                    "下单",
+		"/swordfish/page_big_pc/my_client":                                            "订阅",
+		"/page_workDesktop/work-bench/app/big/big_subscribe?vt=q":                     "订阅",
+		"/swordfish/page_big_pc/free/ent_follow":                                      "订阅",
+		"/page_workDesktop/work-bench/app/big/big_subscribe?vt=v":                     "订阅",
+		"/page_workDesktop/work-bench/app/big/big_subscribe?vt=m":                     "订阅",
+		"/page_workDesktop/work-bench/app/big/big_subscribe?vt=f":                     "订阅",
+		"/page_workDesktop/work-bench/app/big/big_subscribe?vt=s":                     "订阅",
+		"/swordfish/page_big_pc/free/project_progress":                                "订阅",
+		"/jyapp/vipsubscribe/toVIPViewPage":                                           "订阅",
+		"/front/vipsubscribe/toVIPViewPage":                                           "订阅",
+		"/weixin/pay/checkout_filePack":                                               "支付",
+		"/jyapp/pay/checkout_filePack":                                                "支付",
+		"/jy_mobile/common/order/create/filepack":                                     "下单",
+		"/page_workDesktop/work-bench/app/big/workspace/dashboard":                    "工作台",
+		"/jyapp/article/content/.*html":                                               "标讯",
+		"/article/content/.*html":                                                     "标讯",
+		"/jy_mobile/search/middle/supplier":                                           "搜索",
+		"/jy_mobile/search/result/supplier":                                           "搜索",
+		"/jyapp/vipsubscribe/toSetKeyWordPage":                                        "订阅",
+		"/front/vipsubscribe/toSetKeyWordPage":                                        "订阅",
+		"/weixin/pay/checkout_integral":                                               "支付",
+		"/jyapp/pay/checkout_integral":                                                "支付",
+		"/jy_mobile/tabbar/home":                                                      "首页",
+		"/jyapp/jylab/mainSearch?isLogin=1":                                           "首页",
+		"/jylab/mainSearch":                                                           "首页",
+		"/page_docs_mobile/search":                                                    "搜索",
+		"/page_docs_mobile/details/":                                                  "标讯",
+		"/page_docs_mobile/purchase/":                                                 "下单",
+		"/front/swordfish/toMyOrder":                                                  "订单",
+		"/front/invoice/showpage":                                                     "订单",
+		"/jyapp/front/invoice/showpage":                                               "订单",
+		"/page_entniche_new/page/sub_management/sub_management_system.html":           "订阅",
+		"/page_entniche_new/page/subsetting/sub_entrance.html":                        "订阅",
+		"/jyapp/big/page/ent_portrait":                                                "画像",
+		"/swordfish/page_big_pc/ent_portrait/":                                        "画像",
+		"/jy_mobile/search/middle/company":                                            "搜索",
+		"/jy_mobile/search/result/company":                                            "搜索",
+		"/jyapp/pay/checkout_entniche":                                                "支付",
+		"/front/invoice/cantInvoice":                                                  "订单",
+		"/jyapp/front/cantInvoice":                                                    "订单",
+		"/weixin/pay/checkout_areaPack":                                               "支付",
+		"/jyapp/pay/checkout_areaPack":                                                "支付",
+		"/jy_mobile/common/order/create/datareport":                                   "下单",
+		"/weixin/pay/checkout_dataexport":                                             "支付",
+		"/jyapp/pay/checkout_dataexport":                                              "支付",
+		"/jy_mobile/common/order/create/dataexport":                                   "下单",
+		"/weixin/pay/checkout_dataPack":                                               "支付",
+		"/jyapp/pay/checkout_dataPack":                                                "支付",
+		"/jyapp/free/me":                                                              "菜单",
+		"/jyapp/front/myOrder/toMyOrder":                                              "订单",
+		"/front/myOrder/toMyOrder":                                                    "订单",
+		"/jyapp/vipsubscribe/toOrderDetailPage":                                       "订单",
+		"/front/vipsubscribe/toOrderDetailPage":                                       "订单",
+		"/jyapp/front/myorder/tointegralDetail":                                       "订单",
+		"/front/wxMyOrder/integralDetail/":                                            "订单",
+		"/front/wx_dataExport/wxToOrderDetail":                                        "订单",
+		"/jyapp/front/dataExport/toOrderDetail":                                       "订单",
+		"/weixin/common/medical/orderDetail":                                          "订单",
+		"/jyapp/common/medical/orderDetail":                                           "订单",
+		"/weixin/pay/xs_payNow":                                                       "订单",
+		"/jyxsapp/orderMsg":                                                           "订单",
+		"/datareport/page/order/detail":                                               "订单",
+		"/jyapp/front/myorder/toEntnicheDetail":                                       "订单",
+		"/front/wxMyOrder/memberDetail":                                               "订单",
+		"/jyapp/big/page/orderdetail_member":                                          "订单",
+		"/front/wxMyOrder/aiForecastPackdetail":                                       "订单",
+		"/jyapp/big/page/orderdetail_aiForecastPack":                                  "订单",
+		"/front/wxMyOrder/subAccountDetail":                                           "订单",
+		"/jyapp/big/page/orderdetail_subAccount":                                      "订单",
+		"/front/wxMyOrder/bidfileDetail":                                              "订单",
+		"/jyapp/front/myorder/toBidfileDetail":                                        "订单",
+		"/weixin/common/dataPack/orderDetail":                                         "订单",
+		"/jyapp/common/dataPack/orderDetail":                                          "订单",
+		"/weixin/common/filePack/orderDetail":                                         "订单",
+		"/jyapp/common/filePack/orderDetail":                                          "订单",
+		"/weixin/common/areaPack/orderDetail":                                         "订单",
+		"/jyapp/common/areaPack/orderDetail":                                          "订单",
+		"/weixin/common/buyerPortraitPack/orderDetail":                                "订单",
+		"/jyapp/common/buyerPortraitPack/orderDetail":                                 "订单",
+		"/weixin/common/dataFile/orderDetail":                                         "订单",
+		"/jyapp/common/dataFile/orderDetail":                                          "订单",
+		"/jy_mobile/tabbar/subscribe":                                                 "订阅",
+		"/jyapp/swordfish/historypush":                                                "订阅",
+		"/swordfish/newhistorypush":                                                   "订阅",
+		"/swordfish/frontPage/collection/sess/index":                                  "标讯",
+		"/weixin/frontPage/messageCenter/sess/index":                                  "工作台",
+		"/jyapp/frontPage/messageCenter/sess/index":                                   "工作台",
+		"/jy_mobile/tabbar/message":                                                   "工作台",
+		"/swordfish/frontPage/messageCenter/sess/index":                               "工作台",
+		"/jylab/purSearch/index.html":                                                 "搜索",
+		"/search/issued":                                                              "搜索",
+		"/succbi/nzj/app/nzj.app/nzj_search":                                          "搜索",
+		"/jylab/entSearch/index.html":                                                 "搜索",
+		"/jylab/supsearch/index.html":                                                 "搜索",
+		"/jylab/medical/index.html":                                                   "搜索",
+		"/page_workDesktop/work-bench/app/big/big_subscribe?vt=m&v":                   "推送",
+		"/jy_mobile/search/filter-history/bidding":                                    "搜索",
+		"jy_mobile/search/middle/bidding":                                             "搜索",
+		"/jy_mobile/search/result/bidding":                                            "搜索",
+		"/weixin/pay/checkout_onlineCourse":                                           "支付",
+		"/jyapp/pay/checkout_onlineCourse":                                            "支付",
+		"/jy_mobile/common/order/create/course":                                       "支付",
+		"/jyapp/free/register":                                                        "注册",
+		"/jyapp/free/login":                                                           "登录",
+		"/jyapp/free/setPwd":                                                          "注册",
+		"/jyapp/account/phone/bind":                                                   "注册",
+		"/front/account/phone/bind":                                                   "注册",
+		"/swordfish/frontPage/userMerge/sess/bind":                                    "注册",
+		"/succbi/YXT/app/medical.app/zkh_search.spg":                                  "搜索",
+		"/succbi/YXT/app/medical.app/zkh_detail.spg?JGDM=":                            "搜索",
+		"/succbi/YXT/app/medical.app/zhzhb_search.spg":                                "搜索",
+		"/succbi/YXT/app/medical.app/zhzhb_detail.spg?jxs_hx=":                        "搜索",
+		"/succbi/YXT/app/medical.app/scdc_detail.spg":                                 "搜索",
+		"/succbi/YXT/app/medical.app/zsj_search.spg":                                  "搜索",
+		"/jyapp/frontPage/activity/free/app-register":                                 "注册",
+		"/jyapp/followent/detail/.*":                                                  "订阅",
+		"/jyapp/followent/newInfo/.*":                                                 "订阅",
+		"/jy_mobile/order/create/dataexport?source=d&id=.*":                           "下单",
+		"/jy_mobile/redirect/unit_portrayal/":                                         "画像",
+		"/jyapp/swordfish/historypush?f=push&t=free&pushtime=":                        "订阅",
+		"/jyapp/swordfish/historypush?f=push&t=vip&pushtime=":                         "订阅",
+		"/jy_mobile/order/create/svip?tip=":                                           "下单",
+		"/swordfish/page_big_pc/recommen-list?vt=":                                    "标讯",
+		"/jy_mobile/tabbar/recommendedlist":                                           "订阅",
+		"/swordfish/page_big_pc/search/sun":                                           "搜索",
+		"/page_workDesktop/work-bench/app/big/search/sun":                             "搜索",
+		"/jy_mobile/search/middle/sun":                                                "搜索",
+		"/jy_mobile/search/result/sun":                                                "搜索",
+		"/search/construction":                                                        "搜索",
+		"jy_mobile/ai/search":                                                         "搜索",
+		"/debrisproduct/free/login":                                                   "登录",
+		"/debrisproduct/orderDetail":                                                  "订单",
+		"/debrisproduct/orderList":                                                    "订单",
+		"/debrisproduct/getBiddingDetail":                                             "标讯",
+		"/debrisproduct/createOrder":                                                  "订单",
+		"/debrisproduct/info":                                                         "订阅",
+		"/debrisproduct/updateArea":                                                   "订阅",
+		"/debrisproduct/userInfo":                                                     "订阅",
+		"/front/subscribeTransfer":                                                    "订阅",
+		"/article/entservice":                                                         "标讯",
+		"/biddetail/normal":                                                           "标讯",
+		"/front/pcAjaxReq":                                                            "搜索",
+		"/front/tenderSubscribe":                                                      "订阅",
+		"/front/vipsubscribe":                                                         "订阅",
+		"/jylab/entSearch":                                                            "搜索",
+		"/jylab/purSearch":                                                            "搜索",
+		"/jylab/supsearch":                                                            "搜索",
+		"/wxkeyset/keyset":                                                            "订阅",
+		"/user/phone/bind":                                                            "注册",
+	}
+
+	insertData := []map[string]interface{}{}
+	ctx := context.Background()
+	rows, err := Ch_userbehavior.Query(ctx, `select name,id from module`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	moduleMap := map[string]int64{}
+	for rows.Next() {
+		var module_id uint64
+		var href string
+
+		if err := rows.Scan(m); err != nil {
+			if err := rows.Scan(
+				&href,
+				&module_id,
+			); err != nil {
+				log.Fatal(err)
+			}
+			log.Println(href, module_id)
+			moduleMap[href] = int64(module_id)
+		}
+	}
+	for k, v := range m {
+		insertData = append(insertData, map[string]interface{}{
+			"url":       k,
+			"module_id": moduleMap[v],
+		})
+	}
+	g.DB().Model("userbehavior.module_details").Data(insertData).Insert()
+}
+
+// GetModule 模块
+func GetModule() map[string]*Moduledetail {
+	ctx := context.Background()
+	rows, err := Ch_userbehavior.Query(ctx, `select name,id from module`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	moduleIdMap := map[int64]string{}
+	for rows.Next() {
+		var module_id uint64
+		var href string
+
+		if err := rows.Scan(
+			&href,
+			&module_id,
+		); err != nil {
+			log.Fatal(err)
+		}
+		moduleIdMap[int64(module_id)] = href
+
+	}
+
+	rows, err = Ch_userbehavior.Query(ctx, `select module_id,url,desc,platform,product_name,page_name from module_details`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	moduleMap := map[string]*Moduledetail{}
+	for rows.Next() {
+		var module_id uint64
+		var href string
+		var desc string
+		var platform string
+		var product_name string
+		var page_name string
+
+		if err := rows.Scan(
+			&module_id,
+			&href,
+			&desc,
+			&platform,
+			&product_name,
+			&page_name,
+		); err != nil {
+			log.Fatal(err)
+		}
+		//moduleMap[href] = moduleIdMap[int64(module_id)]
+		moduleMap[href] = &Moduledetail{
+			href,
+			int64(module_id),
+			desc,
+			platform,
+			product_name,
+			page_name,
+			moduleIdMap[int64(module_id)],
+		}
+	}
+	return moduleMap
+}
+
+func MatchRegexModule() *CompiledRegexMap {
+	m := GetModule()
+	rm := RegexMap{}
+	for k, v := range m {
+		rm[k] = v
+	}
+	compiledRegexMap, err := NewCompiledRegexMap(rm)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return nil
+	}
+	return compiledRegexMap
+}
+
+func GetActionMap() map[string]int64 {
+	m := map[string]int64{}
+	ctx := context.Background()
+	rows, err := Ch_userbehavior.Query(ctx, `select name,id from action`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var id uint64
+		var name string
+
+		if err := rows.Scan(
+			&name,
+			&id,
+		); err != nil {
+			log.Fatal(err)
+		}
+		m[name] = int64(id)
+	}
+
+	return m
+}
+
+func GetIpSource(ip string) (area, city string) {
+	if strings.Contains(ip, ",") {
+		for _, v := range strings.Split(ip, ",") {
+			if strings.Contains(v, "172.17.") {
+				continue
+			}
+			_, area, city, _ = AnalyIp(v)
+			return area, city
+		}
+	}
+	return area, city
+}
+
+func GetReferType(refer string) string {
+	refertype := "域外"
+	if strings.Contains(refer, "jianyu360") {
+		refertype = "域内"
+	}
+	return refertype
+}
+
+// debris_product_logs 查询语句
+func debris_product_logs(st, et int64) map[string]interface{} {
+	s, e := GetCurTimePiInfo(st, et)
+	query := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": mongodb.StringTOBsonId(s),
+			"$lt":  mongodb.StringTOBsonId(e),
+		},
+		"url": map[string]interface{}{
+			"$in": []string{
+				"/debrisproduct/free/login",
+				"/debrisproduct/orderDetail",
+				"/debrisproduct/orderList",
+				"/debrisproduct/getBiddingDetail",
+				"/debrisproduct/createOrder",
+				"/debrisproduct/info",
+				"/debrisproduct/updateArea",
+				"/debrisproduct/userInfo",
+			},
+		},
+	}
+	return query
+}
+
+func subscribepay(st, et int64) map[string]interface{} {
+	s, e := GetCurTimePiInfo(st, et)
+	query := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": mongodb.StringTOBsonId(s),
+			"$lt":  mongodb.StringTOBsonId(e),
+		},
+		"url": map[string]interface{}{
+			"$in": []string{
+				"/jypay/user/phone/bind",
+				"/jypay/callback/aliPay",
+				"/jypay/callback/wxPay",
+			},
+		},
+	}
+	return query
+}
+
+func jylog_jyapplog(st, et int64) map[string]interface{} {
+	s, e := GetCurTimePiInfo(st, et)
+	query := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": mongodb.StringTOBsonId(s),
+			"$lt":  mongodb.StringTOBsonId(e),
+		},
+		"$or": []map[string]interface{}{
+			//map[string]interface{}{
+			//	"url": map[string]interface{}{
+			//		"$in": []string{
+			//			"/jypay/user/phone/bind",
+			//		},
+			//	},
+			//},
+			map[string]interface{}{
+				"url": map[string]interface{}{
+					"$regex": "/content/",
+				},
+			},
+			map[string]interface{}{
+				"url": map[string]interface{}{
+					"$regex": "/login",
+				},
+			},
+			map[string]interface{}{
+				"url": map[string]interface{}{
+					"$regex": "/register",
+				},
+			},
+		},
+	}
+	return query
+}
+
+// 获取每天0点数据
+func GetCurTimePiInfo(start, end int64) (string, string) {
+	gteid := strconv.FormatInt(start, 16) + "0000000000000000"
+	ltid := strconv.FormatInt(end, 16) + "0000000000000000"
+	return gteid, ltid
+}
+
+// GetQuery 获取查询语句
+func GetQuery(dbname string, s, e int64) (query map[string]interface{}) {
+	if dbname == "subscribepay_logs" {
+		return subscribepay(s, e)
+	}
+	if dbname == "jy_logs" || dbname == "jyapp_logs" {
+		return jylog_jyapplog(s, e)
+	}
+	if dbname == "debris_product_logs" {
+		return debris_product_logs(s, e)
+	}
+	return nil
+}
+
+// TimeStampToDate时间戳转时间
+func TimeStampToDate(timestamp int64) time.Time {
+	// 判断时间戳的长度
+	if timestamp > 9999999999 { // 毫秒级时间戳通常大于10位
+		// 毫秒级时间戳，除以1000转换为秒级时间戳
+		return time.Unix(timestamp/1000, (timestamp%1000)*int64(time.Millisecond)).UTC()
+	}
+	return time.Unix(timestamp, 0).UTC()
+
+}
+
+// Unescape解析url
+func Unescape(href string) string {
+	str, err := url.QueryUnescape(href)
+	if err != nil {
+		log.Println(err)
+	}
+	return str
+}
+
+func GetPlatform(dbname, client, refer string) string {
+	if dbname == "jyapp_logs" {
+		// 检查是否为 Android 客户端
+		if strings.Contains(client, "Android") {
+			return "android"
+		}
+		// 检查是否为 iOS 客户端
+		if strings.Contains(client, "iPhone") || strings.Contains(client, "iPad") || strings.Contains(client, "iPod") {
+			return "ios"
+		}
+		return "ios"
+	} else {
+		if RegWx.MatchString(client) {
+			return "wx"
+		} else {
+			if strings.Contains(refer, "h5.") {
+				return "h5"
+			} else {
+				return "pc"
+			}
+		}
+	}
+	return ""
+}
+
+// 初始化模块
+func InitUrl() {
+
+	insertData := []map[string]interface{}{}
+	ctx := context.Background()
+	rows, err := Ch_userbehavior.Query(ctx, `select name,id from module`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	moduleMap := map[string]int64{}
+	for rows.Next() {
+		var module_id uint64
+		var href string
+
+		if err := rows.Scan(
+			&href,
+			&module_id,
+		); err != nil {
+			log.Fatal(err)
+		}
+		//log.Println(href, module_id)
+		moduleMap[href] = int64(module_id)
+	}
+	f, err := excelize.OpenFile("./url.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		// 关闭文件
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	// 获取工作表名称
+	sheetName := "sheet1"
+	// 获取工作表中的所有行
+	rowsx, err := f.GetRows(sheetName)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for i, row := range rowsx {
+		log.Println("count:", i)
+		if i == 0 {
+			continue
+		}
+		href := row[0]
+		mid := int64(0)
+		if len(row) > 1 {
+			mid = qu.Int64All(qu.If(row[1] == "", 0, moduleMap[row[1]]))
+		}
+		desc := ""
+		if len(row) > 2 {
+			desc = qu.If(row[2] == "", "", row[2]).(string)
+		}
+		platform := ""
+		productname := ""
+		pagename := ""
+		if len(row) > 3 {
+			platform = qu.If(row[3] == "", "", row[3]).(string)
+		}
+		if len(row) > 4 {
+			productname = qu.If(row[4] == "", "", row[4]).(string)
+		}
+		if len(row) > 5 {
+			pagename = qu.If(row[5] == "", "", row[5]).(string)
+		}
+		insertData = append(insertData, map[string]interface{}{
+			"url":          href,
+			"module_id":    mid,
+			"desc":         desc,
+			"platform":     platform,
+			"product_name": productname,
+			"page_name":    pagename,
+		})
+	}
+
+	g.DB().Model("userbehavior.module_details").Data(insertData).Insert()
+}
+
+// ExtractOrderCode 从URL中提取order code
+func ExtractOrderCode(url string) string {
+	// 正则表达式，匹配 odercode=xxx, orderCode=xxx 或 order_code=xxx
+	matches := OderCodeReg.FindStringSubmatch(url)
+	if len(matches) >= 3 {
+		//log.Println(matches)
+		return matches[2]
+	}
+	// 没有找到order code
+	return ""
+}
+
+var (
+	ArticleIdReg = regexp.MustCompile(".*article/content/(.*)\\.html")
+	OderCodeReg  = regexp.MustCompile(`(?i)(odercode|orderCode|order_code)=([a-zA-Z0-9]+)`)
+)
+
+// GetArticleId 获取三级页id
+func GetArticleId(url string) string {
+	ur := ArticleIdReg.FindStringSubmatch(url)
+	if len(ur) > 1 {
+		uarr := BDecodeArticleId2ByCheck(ur[1], encrypt.SE, encrypt.SE2)
+		if len(uarr) > 0 {
+			return uarr[0]
+		}
+	}
+	return ""
+}
+
+// 短地址解密,二次解密带校验和
+func BDecodeArticleId2ByCheck(id string, s1, s2 *encrypt.SimpleEncrypt) []string {
+	if !strings.Contains(id, "+") { //新加密算法解密
+		id, _ = url.QueryUnescape(id)
+	}
+	if len(id) > 2 {
+		kstr := s2.DecodeStringByCheck(id[3:])
+		return strings.Split(s1.DecodeStringByCheck(kstr), ",")
+	}
+	return []string{}
+}
+
+//TODO
+//select * from data_analysis.dwd_f_personnel_behavior where breaker_name='登录'
+//平台  应用  模块 页面 元素
+
+// AddPathCodeId 获取id，自增
+func AddPathCodeId() int64 {
+	data := Ch.SelectBySql(`select id from userbehavior.path_code_id order by id desc limit 1`)
+	if data != nil && len(*data) > 0 {
+		id := qu.Int64All((*data)[0]["id"])
+		newId := id + 1
+		Ch.Insert("path_code_id", map[string]interface{}{
+			"id": newId,
+		})
+		return newId
+	} else {
+		Ch.Insert("path_code_id", map[string]interface{}{
+			"id": 1,
+		})
+		return 1
+	}
+}
+
+// MgoTrustedIdToCkTrustedId mgo的匿名id和clickhouse的不一致 需要转换
+func MgoTrustedIdToCkTrustedId(s string) string {
+	if s == "" {
+		return ""
+	}
+	return encrypt.SE.DecodeString(s)
+}