优化订单回款，批量保存

account_order/clickhouse.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"fmt"
+	"go.uber.org/zap"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"time"
+)
+
+// truncateClickhouse 清空Clickhouse数据表
+func truncateClickhouse() {
+	var (
+		f_order DwdFOrder
+		//account_order_change  DwdFAccountOrderChange
+		//account_return        DwdFAccountReturn
+		//account_return_change DwdFAccountReturnChange
+	)
+
+	table1 := f_order.TableName()
+	//table2 := account_order_change.TableName()
+	//table3 := account_return.TableName()
+	//table4 := account_return_change.TableName()
+
+	// 清空表 1
+	db, err := ClickhouseDB.DB()
+	if err != nil {
+		panic("获取数据库连接对象失败：" + err.Error())
+	}
+	_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s.%s", GF.Clickhouse.DB, table1))
+	if err != nil {
+		log.Info("清空失败", zap.String("数据表", table1))
+	} else {
+		log.Info("清空成功", zap.String("数据表", table1))
+	}
+
+	//_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", table2))
+	//if err != nil {
+	//	log.Info("清空失败", zap.String("数据表", table2))
+	//}
+	//
+	//_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", table3))
+	//if err != nil {
+	//	log.Info("清空失败", zap.String("数据表", table3))
+	//}
+	//
+	//_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", table4))
+	//if err != nil {
+	//	log.Info("清空失败", zap.String("数据表", table4))
+	//}
+	//
+	//log.Info("所有数据表清空完毕", zap.String("数据表是：", fmt.Sprintf("%s,%s,%s,%s", table1, table2, table3, table4)))
+}
+
+// DwdFOrder 订单表-Clickhouse
+type DwdFOrder struct {
+	ID                  int        `gorm:"primaryKey"`                       // 自增 ID
+	OrderCode           string     `gorm:"column:order_code"`                // 订单编号
+	SalerName           string     `gorm:"column:saler_name"`                // 销售人员
+	SalerDept           string     `gorm:"column:saler_dept"`                // 部门
+	CompanyName         string     `gorm:"column:company_name"`              // 公司名称
+	UserRegtime         *time.Time `gorm:"column:user_regtime"`              // 用户注册时间
+	CreateTime          *time.Time `gorm:"column:create_time"`               // 订单创建时间
+	ReturnTime          *time.Time `gorm:"column:return_time"`               // 回款时间
+	SaleTime            *time.Time `gorm:"column:sale_time"`                 // 业绩统计
+	RefundTime          *time.Time `gorm:"column:refund_time"`               // 退款日期
+	OriginalPrice       int        `gorm:"column:original_price"`            // 标准售价
+	ContractMoney       int        `gorm:"column:contract_money"`            // 合同金额
+	Commission          int        `gorm:"column:commission"`                // 佣金
+	ProceduresMoney     int        `gorm:"column:procedures_money"`          // 手续费
+	ReceivableAmount    int        `gorm:"column:receivable_amount"`         // 应收金额
+	TotalReceived       int        `gorm:"column:total_received"`            // 累计已收
+	ProductType         string     `gorm:"column:product_type"`              // 产品类型
+	DataSpec            string     `gorm:"column:data_spec"`                 // 规格
+	OrderStatus         string     `gorm:"column:order_status"`              // 订单状态
+	ReturnStatus        string     `gorm:"column:return_status"`             // 回款状态
+	RefundStatus        string     `gorm:"column:refund_status"`             // 退款状态
+	VipType             string     `gorm:"column:vip_type"`                  // 付费类型
+	UserPhone           string     `gorm:"column:user_phone"`                // 手机号
+	UserID              string     `gorm:"column:user_id"`                   // 用户 ID
+	VipStarttime        *time.Time `gorm:"column:vip_starttime"`             // 服务开始时间
+	VipEndtime          *time.Time `gorm:"column:vip_endtime"`               // 服务结束时间
+	ContractStatus      string     `gorm:"column:contract_status"`           // 合同状态
+	ContractCode        string     `gorm:"column:contract_code"`             // 合同编号
+	ContractTime        *time.Time `gorm:"column:contract_time"`             // 合同时间
+	SigningSubject      string     `gorm:"column:signing_subject"`           // 签约主体
+	OrderChannel        string     `gorm:"column:order_channel"`             // 下单渠道
+	DistributionChannel string     `gorm:"column:distribution_channel"`      // 销售渠道
+	IsBackstageOrder    string     `gorm:"column:is_backstage_order"`        // 是否是后台订单
+	PayWay              string     `gorm:"column:pay_way"`                   // 付款方式
+	Comeintime          *time.Time `gorm:"column:comeintime;autoCreateTime"` // 入库时间
+	Updatetime          *time.Time `gorm:"column:updatetime;autoUpdateTime"` // 更新时间
+}
+
+func (DwdFOrder) TableName() string {
+	return "dwd_f_order"
+}
+
+// DwdFOrderChange 归集后的-剑鱼业绩变更表结构体;
+type DwdFOrderChange struct {
+	ID                  int        `gorm:"primaryKey"`
+	OrderCode           string     `gorm:"column:order_code;not null;comment:'订单编号'"`
+	SalerName           string     `gorm:"column:saler_name;comment:'销售人员'"`
+	SalerDept           string     `gorm:"column:saler_dept;comment:'部门'"`
+	CompanyName         string     `gorm:"column:company_name;comment:'公司名称'"`
+	UserRegtime         *time.Time `gorm:"column:user_regtime;comment:'用户注册时间'"`
+	CreateTime          *time.Time `gorm:"column:create_time;comment:'订单创建时间'"`
+	SaleTime            *time.Time `gorm:"column:sale_time;comment:'业绩统计时间'"`
+	OriginalPrice       int        `gorm:"column:original_price;comment:'标准售价'"`
+	ContractMoney       int        `gorm:"column:contract_money;comment:'合同金额'"`
+	Commission          int        `gorm:"column:commission;comment:'佣金'"`
+	ProceduresMoney     int        `gorm:"column:procedures_money;comment:'手续费'"`
+	ReceivableAmount    int        `gorm:"column:receivable_amount;comment:'应收金额'"`
+	ChangeValue         int        `gorm:"column:change_value;comment:'业绩变动额'"`
+	ChangeReason        string     `gorm:"column:change_reason;comment:'变更类型'"`
+	TotalReceived       int        `gorm:"column:total_received;comment:'累计已收'"`
+	ProductType         string     `gorm:"column:product_type;comment:'产品类型'"`
+	DataSpec            string     `gorm:"column:data_spec;comment:'规格'"`
+	OrderStatus         string     `gorm:"column:order_status;comment:'订单状态'"`
+	ReturnStatus        string     `gorm:"column:return_status;comment:'回款状态'"`
+	RefundStatus        string     `gorm:"column:refund_status;comment:'退款状态'"`
+	VipType             string     `gorm:"column:vip_type;comment:'付费类型'"`
+	UserPhone           string     `gorm:"column:user_phone;comment:'手机号'"`
+	UserID              string     `gorm:"column:user_id;comment:'用户ID'"`
+	VipStarttime        *time.Time `gorm:"column:vip_starttime;comment:'服务开始时间'"`
+	VipEndtime          *time.Time `gorm:"column:vip_endtime;comment:'服务结束时间'"`
+	ContractStatus      string     `gorm:"column:contract_status;comment:'合同状态'"`
+	ContractCode        string     `gorm:"column:contract_code;comment:'合同编号'"`
+	ContractTime        *time.Time `gorm:"column:contract_time;comment:'合同时间'"`
+	SigningSubject      string     `gorm:"column:signing_subject;comment:'签约主体'"`
+	OrderChannel        string     `gorm:"column:order_channel;comment:'下单渠道'"`
+	DistributionChannel string     `gorm:"column:distribution_channel;comment:'销售渠道'"`
+	IsBackstageOrder    string     `gorm:"column:is_backstage_order;comment:'是否是后台订单'"`
+	PayWay              string     `gorm:"column:pay_way;comment:'付款方式'"`
+	Comeintime          *time.Time `gorm:"column:comeintime;default:CURRENT_TIMESTAMP;comment:'数据进入时间'"`
+	Updatetime          *time.Time `gorm:"column:updatetime;default:CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP;comment:'更新时间'"`
+	OrderSaleRecordID   int        `gorm:"column:order_sale_record_id;comment:'业绩变更表ID'"`
+	RefundRecordID      int        `gorm:"column:refund_record_id;comment:'退款表ID'"`
+}
+
+func (DwdFOrderChange) TableName() string {
+	return "dwd_f_performance_change"
+}
+
+// DwdFReturn 回款表
+type DwdFReturn struct {
+	ID                     int        `gorm:"primaryKey"`
+	OrderCode              string     `gorm:"column:order_code;comment:'订单编号'"`
+	SalerName              string     `gorm:"column:saler_name;comment:'销售人员'"`
+	SalerDept              string     `gorm:"column:saler_dept;comment:'部门'"`
+	UserPhone              string     `gorm:"column:user_phone;comment:'用户手机号'"`
+	CompanyName            string     `gorm:"column:company_name;comment:'公司名称'"`
+	ProductType            string     `gorm:"column:product_type;comment:'产品类型'"`
+	DataSpec               string     `gorm:"column:data_spec;comment:'规格'"`
+	ReturnType             string     `gorm:"column:return_type;comment:'回款方式'"`
+	SaleTime               *time.Time `gorm:"column:sale_time;comment:'业绩统计日期'"`
+	ReturnTime             *time.Time `gorm:"column:return_time;comment:'回款日期'"`
+	ReturnMoney            int        `gorm:"column:return_money;comment:'回款金额'"`
+	SigningSubject         string     `gorm:"column:signing_subject;comment:'签约主体'"`
+	ReturnSubject          string     `gorm:"column:return_subject;comment:'回款主体'"`
+	SubjectCheck           string     `gorm:"column:subject_check;comment:'主体校验'"`
+	PaymentNumber          string     `gorm:"column:payment_number;comment:'支付单号'"`
+	BankFlow               string     `gorm:"column:bank_flow;comment:'银行流水号'"`
+	BankName               string     `gorm:"column:bank_name;comment:'银行名称'"`
+	OrderChannel           string     `gorm:"column:order_channel;comment:'下单渠道'"`
+	DistributionChannel    string     `gorm:"column:distribution_channel;comment:'销售渠道'"`
+	RefundStatus           string     `gorm:"column:refund_status;comment:'退款状态'"`
+	OperateType            string     `gorm:"column:operate_type;comment:'关联方式'"`
+	OperatePerson          string     `gorm:"column:operate_person;comment:'创建人'"`
+	ReturnMoneyRecordState int        `gorm:"column:return_money_record_state;comment:'数据状态'"`
+	Comeintime             *time.Time `gorm:"column:comeintime;default:CURRENT_TIMESTAMP;comment:'入库时间'"`
+	Updatetime             *time.Time `gorm:"column:updatetime;default:CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP;comment:'更新时间'"`
+	ReturnMoneyRecordID    int        `gorm:"column:return_money_record_id;comment:'回款表 id'"`
+	DataexportOrderID      int        `gorm:"column:dataexport_order_id;comment:'订单表 id'"`
+}
+
+func (DwdFReturn) TableName() string {
+	return "dwd_f_return"
+}
+
+// DwdFReturnChange 回款变更表
+type DwdFReturnChange struct {
+	ID                  int        `gorm:"primaryKey;autoIncrement;comment:'自增唯一标识'"`
+	OrderCode           string     `gorm:"column:order_code;comment:'订单编号'"`
+	SalerName           string     `gorm:"column:saler_name;comment:'销售人员'"`
+	SalerDept           string     `gorm:"column:saler_dept;comment:'部门'"`
+	UserPhone           string     `gorm:"column:user_phone;comment:'用户手机号'"`
+	CompanyName         string     `gorm:"column:company_name;comment:'公司名称'"`
+	ProductType         string     `gorm:"column:product_type;comment:'产品类型'"`
+	DataSpec            string     `gorm:"column:data_spec;comment:'规格'"`
+	ReturnType          string     `gorm:"column:return_type;comment:'回款方式'"`
+	SaleTime            *time.Time `gorm:"column:sale_time;comment:'业绩变更日期'"`
+	ReturnTime          *time.Time `gorm:"column:return_time;comment:'回款日期'"`
+	ReturnMoney         int        `gorm:"column:return_money;comment:'回款金额'"`
+	ChangeValue         int        `gorm:"column:change_value;comment:'业绩变动额'"`
+	ChangeReason        string     `gorm:"column:change_reason;comment:'变更类型'"`
+	SigningSubject      string     `gorm:"column:signing_subject;comment:'签约主体'"`
+	ReturnSubject       string     `gorm:"column:return_subject;comment:'回款主体'"`
+	SubjectCheck        string     `gorm:"column:subject_check;comment:'主体校验'"`
+	PaymentNumber       string     `gorm:"column:payment_number;comment:'支付单号'"`
+	BankFlow            string     `gorm:"column:bank_flow;comment:'银行流水号'"`
+	BankName            string     `gorm:"column:bank_name;comment:'银行名称'"`
+	OrderChannel        string     `gorm:"column:order_channel;comment:'下单渠道'"`
+	DistributionChannel string     `gorm:"column:distribution_channel;comment:'销售渠道'"`
+	RefundStatus        string     `gorm:"column:refund_status;comment:'退款状态'"`
+	OperateType         string     `gorm:"column:operate_type;comment:'关联方式'"`
+	OperatePerson       string     `gorm:"column:operate_person;comment:'创建人'"`
+	OrderSaleRecordID   int        `gorm:"column:order_sale_record_id;comment:'业绩变更表 ID'"`
+	RefundRecordID      int        `gorm:"column:refund_record_id;comment:'退款表 ID'"`
+	Comeintime          *time.Time `gorm:"column:comeintime;default:CURRENT_TIMESTAMP;comment:'入库时间'"`
+	Updatetime          *time.Time `gorm:"column:updatetime;default:CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP;comment:'更新时间'"`
+}
+
+func (DwdFReturnChange) TableName() string {
+	return "dwd_f_return_change"
+}

account_order/config.go

@@ -12,6 +12,7 @@ type GlobalConf struct {
 	Log         Log
 	Mongospider MgoConf
 	Email       EmailConf
+	Clickhouse  CkConf
 }
 
 type MgoConf struct {
@@ -32,11 +33,11 @@ type CronConf struct {
 	Delete int
 }
 
-type EspConf struct {
-	URL      string
+type CkConf struct {
+	Host     string
 	Username string
 	Password string
-	Index    string
+	DB       string
 }
 
 type MysqlConf struct {

account_order/config.toml

@@ -43,6 +43,16 @@
     test = true
 
 
+[clickhouse] ## clickhouse 数据库
+#    host = "cc-2ze9tv451wov14w9e.clickhouse.ads.aliyuncs.com:9000"
+#    username = "biservice"
+#    password = "Bi_top95215#"
+#    db = "orderData"
+## 测试环境
+host = "192.168.3.207:19000"
+username = "jytop"
+password = "pwdTopJy123"
+db = "wcc"
 
     ## 测试环境
 #    address = "192.168.3.14:4000"

account_order/go.mod

@@ -5,30 +5,42 @@ go 1.22.6
 require (
 	github.com/spf13/viper v1.19.0
 	go.uber.org/zap v1.27.0
+	gorm.io/driver/clickhouse v0.6.1
 	gorm.io/driver/mysql v1.5.7
 	gorm.io/gorm v1.25.12
 	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3
 )
 
 require (
+	github.com/ClickHouse/ch-go v0.61.5 // indirect
+	github.com/ClickHouse/clickhouse-go/v2 v2.23.2 // indirect
 	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/dchest/captcha v1.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
-	github.com/golang/snappy v0.0.1 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
-	github.com/klauspost/compress v1.17.2 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/paulmach/orb v0.11.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
@@ -38,13 +50,15 @@ require (
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.mongodb.org/mongo-driver v1.10.1 // indirect
-	go.uber.org/multierr v1.10.0 // indirect
+	go.mongodb.org/mongo-driver v1.11.4 // indirect
+	go.opentelemetry.io/otel v1.26.0 // indirect
+	go.opentelemetry.io/otel/trace v1.26.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect

account_order/go.sum

@@ -1,9 +1,16 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4=
+github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg=
+github.com/ClickHouse/clickhouse-go/v2 v2.23.2 h1:+DAKPMnxLS7pduQZsrJc8OhdLS2L9MfDEJ2TS+hpYDM=
+github.com/ClickHouse/clickhouse-go/v2 v2.23.2/go.mod h1:aNap51J1OM3yxQJRgM+AlP/MPkGBCL8A74uQThoQhR0=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
@@ -26,11 +33,16 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
+github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -44,8 +56,10 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -54,10 +68,15 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@@ -67,9 +86,11 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -87,8 +108,13 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
+github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -98,12 +124,16 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
 github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
@@ -141,19 +171,26 @@ github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCO
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs=
+go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4=
 go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA=
+go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
-go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
@@ -170,6 +207,8 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -177,6 +216,8 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -188,6 +229,8 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
@@ -201,8 +244,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -217,6 +260,8 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -241,10 +286,12 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
@@ -257,6 +304,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/clickhouse v0.6.1 h1:t7JMB6sLBXxN8hEO6RdzCbJCwq/jAEVZdwXlmQs1Sd4=
+gorm.io/driver/clickhouse v0.6.1/go.mod h1:riMYpJcGZ3sJ/OAZZ1rEP1j/Y0H6cByOAnwz7fo2AyM=
 gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
 gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
 gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=

account_order/init.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
+	"gorm.io/driver/clickhouse"
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"net/url"
 	"os"
 	"time"
 )
@@ -45,7 +47,7 @@ func init() {
 	//InitMgo()
 	//InitEs()
 	InitMysql()
-
+	InitClickhouse()
 	//
 
 }
@@ -166,3 +168,37 @@ func InitMgo() {
 
 	log.Info("InitMgo", zap.Any("duration", time.Since(now).Seconds()))
 }
+
+// InitClickhouse 初始化Clickhouse
+func InitClickhouse() {
+	username := GF.Clickhouse.Username
+	password := GF.Clickhouse.Password
+	host := GF.Clickhouse.Host
+	dbname := GF.Clickhouse.DB
+
+	var err error
+	// 设置时区为 Asia/Shanghai
+	loc, err := time.LoadLocation("Asia/Shanghai")
+	if err != nil {
+		log.Info("failed to load location: ", zap.Error(err))
+	}
+
+	encodedPassword := url.QueryEscape(password)
+	// 移除连接字符串中的 loc 参数
+	dn := fmt.Sprintf("clickhouse://%s:%s@%s/%s?dial_timeout=10s&read_timeout=20s", username, encodedPassword, host, dbname)
+	ClickhouseDB, err = gorm.Open(clickhouse.Open(dn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		log.Info("打开数据库失败：", zap.Error(err))
+	} else {
+		log.Info("连接数据库成功", zap.String("", ClickhouseDB.Name()))
+	}
+
+	// 确保 GORM 使用本地时区
+	ClickhouseDB = ClickhouseDB.Set("gorm:time_loc", loc)
+
+	// 检查时间是否正确
+	now := time.Now().In(loc) // 使用本地时间
+	log.Info("Current local time: ", zap.String("time", now.String()))
+}

account_order/main.go

@@ -21,6 +21,7 @@ var (
 	AnalysisDB              *gorm.DB
 	JianyuDB                *gorm.DB
 	JianyuSubjectDB         *gorm.DB
+	ClickhouseDB            *gorm.DB
 	endTime                 string //取数据，时间范围的截止时间；如果为空，就取当前时间
 	accountOrderPool        = make(chan DwdFAccountOrder, 5000)
 	accountOrderChangePool  = make(chan DwdFAccountOrderChange, 5000)
@@ -28,12 +29,11 @@ var (
 	accountReturnChangePool = make(chan DwdFAccountReturnChange, 5000)
 )
 
-// var endTime = "2024-11-02"
-
 func main() {
 	local, _ := time.LoadLocation("Asia/Shanghai")
 	c := cron.New(cron.WithLocation(local), cron.WithSeconds())
-	eid, err := c.AddFunc(GF.Cron.Spec, dealData)
+	//eid, err := c.AddFunc(GF.Cron.Spec, dealData)		//tidb 数据库
+	eid, err := c.AddFunc(GF.Cron.Spec, dealDataForClickhouse) //Clickhouse数据库
 	if err != nil {
 		log.Info("main", zap.Any("AddFunc err", err))
 	}
@@ -85,6 +85,45 @@ func dealData() {
 	go dealAllDataAccountReturnChange()      //4.处理归集后-剑鱼回款变更表-dwd_f_account_return_change
 }
 
+func dealDataForClickhouse() {
+	//1.清空数据表
+	endTime = time.Now().Format("2006-01-02 15:04:05")
+	if GF.Cron.Url != "" {
+		// 请求头
+		headers := map[string]string{
+			"Content-Type": "application/json",
+		}
+
+		// 请求体
+		body := []byte(`{
+			"msgtype": "text",
+			"text": {
+				"content": "开始清空 Clickhouse 订单回款相关数据",
+				"mentioned_list":["@all"]
+			}
+		}`)
+		_, err := HTTPRequest("POST", GF.Cron.Url, headers, body)
+		if err != nil {
+			log.Info("HTTPRequest ", zap.Error(err), zap.String("body", string(body)))
+		}
+	}
+	//clickhouse 数据清空
+	truncateClickhouse()
+	log.Info("配置取值时间范围为空", zap.String("默认取值截止时间为：", endTime))
+	//重新打开通道
+	accountOrderPool = make(chan DwdFAccountOrder, 5000)
+	accountOrderChangePool = make(chan DwdFAccountOrderChange, 5000)
+	accountReturnPool = make(chan DwdFAccountReturn, 5000)
+	accountReturnChangePool = make(chan DwdFAccountReturnChange, 5000)
+
+	go saveDataClickhouse() //保存订单数据
+	go dealAllDataAccountOrder2()
+	go dealAllDataAccountOrderChangeRecord() //2.处理归集后-存量业绩表更表-dwd_f_account_order_change
+	go dealAllDataAccountReturn()            //3.处理归集后-存量剑鱼回款表-dwd_f_account_return
+	go dealAllDataAccountReturnChange()      //4.处理归集后-剑鱼回款变更表-dwd_f_account_return_change
+
+}
+
 // dealAllDataAccountOrder2 处理归集后-存量剑鱼订单表-dwd_f_account_order
 func dealAllDataAccountOrder2() {
 	defer func() {
@@ -379,72 +418,228 @@ func dealAllDataAccountOrder2() {
 	log.Info("dealAllDataAccountOrder", zap.Any("处理时长", time.Since(now).Minutes()))
 }
 
-// saveAccountOrder 保存归集订单表
-func saveAccountOrder() {
-	defer func() {
-		if r := recover(); r != nil {
-			log.Info("saveAccountOrder Panic recovered", zap.Any("reason", r))
+// saveData 保存数据
+func saveData() {
+	var wg sync.WaitGroup
+	// 启动保存每个通道数据的goroutine
+	wg.Add(4) // 因为有4个通道
+
+	//1.DwdFAccountOrder
+	go func() {
+		defer wg.Done()  // 完成时减少计数器
+		batchSize := 100 // 设置批量插入大小
+		var batch []DwdFAccountOrder
+		for accountOrder := range accountOrderPool {
+			batch = append(batch, accountOrder)
+			if len(batch) >= batchSize {
+				err := AnalysisDB.Model(DwdFAccountOrder{}).CreateInBatches(batch, batchSize).Error
+				if err != nil {
+					log.Info("saveData DwdFAccountOrder", zap.Error(err))
+				}
+				batch = nil // 清空当前批次
+			}
+		}
+		// 插入剩余的批次
+		if len(batch) > 0 {
+			err := AnalysisDB.Model(DwdFAccountOrder{}).CreateInBatches(batch, len(batch)).Error
+			if err != nil {
+				log.Info("saveData DwdFAccountOrder", zap.Error(err))
+			}
 		}
-		log.Info("saveAccountOrder over")
 	}()
-
-	for {
-		select {
-		case accountOrder, ok := <-accountOrderPool:
-			if !ok {
-				// 通道已关闭，退出循环
-				return
+	//2.DwdFAccountOrderChange
+	go func() {
+		defer wg.Done()
+		batchSize := 100 // 设置批量插入大小
+		var batch []DwdFAccountOrderChange
+		for accountOrderChange := range accountOrderChangePool {
+			batch = append(batch, accountOrderChange)
+			if len(batch) >= batchSize {
+				err := AnalysisDB.Model(DwdFAccountOrderChange{}).CreateInBatches(batch, batchSize).Error
+				if err != nil {
+					log.Info("saveData DwdFAccountOrderChange", zap.Error(err))
+				}
+				batch = nil // 清空当前批次
+			}
+		}
+		// 插入剩余的批次
+		if len(batch) > 0 {
+			err := AnalysisDB.Model(DwdFAccountOrderChange{}).CreateInBatches(batch, len(batch)).Error
+			if err != nil {
+				log.Info("saveData DwdFAccountOrderChange", zap.Error(err))
+			}
+		}
+	}()
+	//3.DwdFAccountReturn
+	go func() {
+		defer wg.Done()
+		batchSize := 100 // 设置批量插入大小
+		var batch []DwdFAccountReturn
+		for accountReturn := range accountReturnPool {
+			batch = append(batch, accountReturn)
+			if len(batch) >= batchSize {
+				err := AnalysisDB.Model(DwdFAccountReturn{}).CreateInBatches(batch, batchSize).Error
+				if err != nil {
+					log.Info("saveData DwdFAccountReturn", zap.Error(err))
+				}
+				batch = nil // 清空当前批次
 			}
-			err := AnalysisDB.Create(&accountOrder).Error
+		}
+		// 插入剩余的批次
+		if len(batch) > 0 {
+			err := AnalysisDB.Model(DwdFAccountReturn{}).CreateInBatches(batch, len(batch)).Error
 			if err != nil {
-				log.Info("dealAllDataAccountOrder Create ", zap.Error(err), zap.String("order_code", accountOrder.OrderCode))
+				log.Info("saveData DwdFAccountReturn", zap.Error(err))
+			}
+		}
+	}()
+	//4.DwdFAccountReturnChange
+	go func() {
+		defer wg.Done()
+		batchSize := 100 // 设置批量插入大小
+		var batch []DwdFAccountReturnChange
+		for accountReturnChange := range accountReturnChangePool {
+			batch = append(batch, accountReturnChange)
+			if len(batch) >= batchSize {
+				err := AnalysisDB.Model(DwdFAccountReturnChange{}).CreateInBatches(batch, batchSize).Error
+				if err != nil {
+					log.Info("saveData DwdFAccountReturnChange", zap.Error(err))
+				}
+				batch = nil // 清空当前批次
+			}
+		}
+		// 插入剩余的批次
+		if len(batch) > 0 {
+			err := AnalysisDB.Model(DwdFAccountReturnChange{}).CreateInBatches(batch, len(batch)).Error
+			if err != nil {
+				log.Info("saveData DwdFAccountReturnChange", zap.Error(err))
+			}
+		}
+	}()
+
+	// 等待所有goroutine完成
+	wg.Wait()
+
+	// 所有通道的数据都处理完毕后打印日志
+	log.Info("All account data has been processed and saved successfully.")
+	if GF.Cron.Url != "" {
+		// 请求头
+		headers := map[string]string{
+			"Content-Type": "application/json",
+		}
+		// 请求体
+		body := []byte(`{
+			"msgtype": "text",
+			"text": {
+				"content": "订单回款数据处理完毕",
+				"mentioned_list":["@all"]
 			}
+		}`)
+		_, err := HTTPRequest("POST", GF.Cron.Url, headers, body)
+		if err != nil {
+			log.Info("HTTPRequest ", zap.Error(err), zap.String("body", string(body)))
 		}
 	}
+
 }
 
-// saveData 保存数据
-func saveData() {
+// saveDataClickhouse 保存数据，Clickhouse
+func saveDataClickhouse() {
 	var wg sync.WaitGroup
-	// 启动保存每个通道数据的goroutine
+	//// 启动保存每个通道数据的goroutine
 	wg.Add(4) // 因为有4个通道
-
+	//1.DwdFAccountOrder
 	go func() {
-		defer wg.Done() // 完成时减少计数器
+		defer wg.Done()
+		batchSize := 100 // 设置批量插入大小
+		var batch []DwdFAccountOrder
 		for accountOrder := range accountOrderPool {
-			err := AnalysisDB.Create(&accountOrder).Error
+			batch = append(batch, accountOrder)
+			if len(batch) >= batchSize {
+				err := ClickhouseDB.Model(DwdFOrder{}).CreateInBatches(batch, batchSize).Error
+				if err != nil {
+					log.Info("saveDataClickhouse DwdFOrder", zap.Error(err))
+				}
+				batch = nil // 清空当前批次
+			}
+		}
+		// 插入剩余的批次
+		if len(batch) > 0 {
+			err := ClickhouseDB.Model(DwdFOrder{}).CreateInBatches(batch, len(batch)).Error
 			if err != nil {
-				log.Info("dealAllDataAccountOrder Create", zap.Error(err), zap.String("order_code", accountOrder.OrderCode))
+				log.Info("saveDataClickhouse DwdFOrder", zap.Error(err))
 			}
 		}
 	}()
+	//
 
+	//2.DwdFAccountOrderChange
 	go func() {
 		defer wg.Done()
+		batchSize := 100 // 设置批量插入大小
+		var batch []DwdFAccountOrderChange
 		for accountOrderChange := range accountOrderChangePool {
-			err := AnalysisDB.Create(&accountOrderChange).Error
+			batch = append(batch, accountOrderChange)
+			if len(batch) >= batchSize {
+				err := ClickhouseDB.Model(DwdFOrderChange{}).CreateInBatches(batch, batchSize).Error
+				if err != nil {
+					log.Info("saveDataClickhouse DwdFOrderChange", zap.Error(err))
+				}
+				batch = nil // 清空当前批次
+			}
+		}
+		// 插入剩余的批次
+		if len(batch) > 0 {
+			err := ClickhouseDB.Model(DwdFOrderChange{}).CreateInBatches(batch, len(batch)).Error
 			if err != nil {
-				log.Info("dealAllDataAccountOrderChange Create", zap.Error(err), zap.String("order_code", accountOrderChange.OrderCode))
+				log.Info("saveDataClickhouse DwdFOrderChange", zap.Error(err))
 			}
 		}
 	}()
-
+	//3.DwdFAccountReturn
 	go func() {
 		defer wg.Done()
+		batchSize := 100 // 设置批量插入大小
+		var batch []DwdFAccountReturn
 		for accountReturn := range accountReturnPool {
-			err := AnalysisDB.Create(&accountReturn).Error
+			batch = append(batch, accountReturn)
+			if len(batch) >= batchSize {
+				err := ClickhouseDB.Model(DwdFReturn{}).CreateInBatches(batch, batchSize).Error
+				if err != nil {
+					log.Info("saveDataClickhouse DwdFReturn", zap.Error(err))
+				}
+				batch = nil // 清空当前批次
+			}
+		}
+		// 插入剩余的批次
+		if len(batch) > 0 {
+			err := ClickhouseDB.Model(DwdFReturn{}).CreateInBatches(batch, len(batch)).Error
 			if err != nil {
-				log.Info("dealAllDataAccountReturn Create", zap.Error(err), zap.String("order_code", accountReturn.OrderCode))
+				log.Info("saveDataClickhouse DwdFReturn", zap.Error(err))
 			}
 		}
-	}()
 
+	}()
+	//4.DwdFAccountReturnChange
 	go func() {
 		defer wg.Done()
+		batchSize := 100 // 设置批量插入大小
+		var batch []DwdFAccountReturnChange
 		for accountReturnChange := range accountReturnChangePool {
-			err := AnalysisDB.Create(&accountReturnChange).Error
+			batch = append(batch, accountReturnChange)
+			if len(batch) >= batchSize {
+				err := ClickhouseDB.Model(DwdFReturnChange{}).CreateInBatches(batch, batchSize).Error
+				if err != nil {
+					log.Info("saveDataClickhouse DwdFReturnChange", zap.Error(err))
+				}
+				batch = nil // 清空当前批次
+			}
+		}
+		// 插入剩余的批次
+		if len(batch) > 0 {
+			err := ClickhouseDB.Model(DwdFReturnChange{}).CreateInBatches(batch, len(batch)).Error
 			if err != nil {
-				log.Info("dealAllDataAccountReturnChange Create", zap.Error(err), zap.String("order_code", accountReturnChange.OrderCode))
+				log.Info("saveDataClickhouse DwdFReturnChange", zap.Error(err))
 			}
 		}
 	}()
@@ -453,7 +648,7 @@ func saveData() {
 	wg.Wait()
 
 	// 所有通道的数据都处理完毕后打印日志
-	log.Info("All account data has been processed and saved successfully.")
+	log.Info("All clickhouse data has been processed and saved successfully.")
 	if GF.Cron.Url != "" {
 		// 请求头
 		headers := map[string]string{
@@ -463,7 +658,7 @@ func saveData() {
 		body := []byte(`{
 			"msgtype": "text",
 			"text": {
-				"content": "订单回款数据处理完毕",
+				"content": "Clickhouse 订单回款数据处理完毕",
 				"mentioned_list":["@all"]
 			}
 		}`)