Merge branch 'master' of https://jygit.jydev.jianyu360.cn/data_processing/timed_tasks

account_order/account_order_change.go

@@ -30,9 +30,9 @@ func allDataOrderSaleRecordForAccountOrderChange() {
 
 		log.Info("aaaaaaaaaaaa")
 	}()
-
+	now := time.Now()
 	var total int64
-	JianyuDB.Debug().Model(&OrderSaleRecord{}).Where("state IN  ?", []string{"1", "-1"}).Where("autoUpdate < ?", "2024-10-16").Count(&total)
+	JianyuDB.Debug().Model(&OrderSaleRecord{}).Where("state IN  ?", []string{"1", "-1"}).Where("autoUpdate < ?", endTime).Count(&total)
 	log.Info("allDataOrderSaleRecordForAccountOrderChange", zap.Any("OrderSaleRecord 总数是", total))
 
 	rowsPerPage := 100                           // 每页的数量
@@ -40,7 +40,7 @@ func allDataOrderSaleRecordForAccountOrderChange() {
 
 	for page := 0; page < totalPages; page++ {
 		offset := page * rowsPerPage
-		rows, err := JianyuDB.Debug().Model(&OrderSaleRecord{}).Where("state IN  ?", []string{"1", "-1"}).Where("autoUpdate < ?", "2024-10-16").Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
+		rows, err := JianyuDB.Debug().Model(&OrderSaleRecord{}).Where("state IN  ?", []string{"1", "-1"}).Where("autoUpdate < ?", endTime).Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
 		defer func() {
 			err = rows.Close()
 			if err != nil {
@@ -196,9 +196,9 @@ func allDataOrderSaleRecordForAccountOrderChange() {
 				// refund_status 回款状态
 				if data_exOrder.IsBackstageOrder == 0 {
 					if account_order.OrderStatus == "已支付" {
-						account_order_change.RefundStatus = "全额回款"
+						account_order_change.ReturnStatus = "全额回款"
 					} else {
-						account_order_change.RefundStatus = "未回款"
+						account_order_change.ReturnStatus = "未回款"
 					}
 				} else if data_exOrder.IsBackstageOrder == 1 {
 					switch data_exOrder.ReturnStatus {
@@ -213,11 +213,11 @@ func allDataOrderSaleRecordForAccountOrderChange() {
 				//退款状态
 				switch data_exOrder.RefundStatus {
 				case 1:
-					account_order_change.ReturnStatus = "全额回款"
+					account_order_change.RefundStatus = "全额回款"
 				case 2:
-					account_order_change.ReturnStatus = "部分回款"
+					account_order_change.RefundStatus = "部分回款"
 				case 0:
-					account_order_change.ReturnStatus = "未回款"
+					account_order_change.RefundStatus = "未回款"
 				}
 
 				//付费类型 vip_type
@@ -248,8 +248,10 @@ func allDataOrderSaleRecordForAccountOrderChange() {
 				//
 				account_order_change.UserPhone = data_exOrder.UserPhone
 				account_order_change.UserID = data_exOrder.UserID
-				account_order_change.VipStarttime = data_exOrder.VipStarttime
-				account_order_change.VipEndtime = data_exOrder.VipEndtime
+				if data_exOrder.VipStarttime == nil || data_exOrder.VipStarttime.Year() > 1000 {
+					account_order_change.VipStarttime = data_exOrder.VipStarttime
+					account_order_change.VipEndtime = data_exOrder.VipEndtime
+				}
 				//合同
 				if contract.ContractStatus == 1 {
 					account_order_change.ContractStatus = "已签协议"
@@ -295,6 +297,7 @@ func allDataOrderSaleRecordForAccountOrderChange() {
 	}
 
 	log.Info("allDataOrderSaleRecordForAccountOrderChange 迭代结束", zap.Int64("数据总量", total))
+	log.Info("allDataOrderSaleRecordForAccountOrderChange", zap.Any("处理时长", time.Since(now).Minutes()))
 }
 
 // allDataRefundRecordForAccountOrderChange 退款表数据处理 -refund_record
@@ -306,9 +309,9 @@ func allDataRefundRecordForAccountOrderChange() {
 
 		log.Info("aaaaaaaaaaaa")
 	}()
-
+	now := time.Now()
 	var total int64
-	AnalysisDB.Debug().Model(&RefundRecord{}).Where("refund_time < ?", "2024-10-16").Count(&total)
+	AnalysisDB.Debug().Model(&RefundRecord{}).Where("refund_time < ?", endTime).Count(&total)
 	log.Info("allDataRefundRecordForAccountOrderChange", zap.Any("RefundRecord 退款表 总数是", total))
 
 	rowsPerPage := 100                           // 每页的数量
@@ -316,7 +319,7 @@ func allDataRefundRecordForAccountOrderChange() {
 
 	for page := 0; page < totalPages; page++ {
 		offset := page * rowsPerPage
-		rows, err := AnalysisDB.Debug().Model(&RefundRecord{}).Where("refund_time < ?", "2024-10-16").Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
+		rows, err := AnalysisDB.Debug().Model(&RefundRecord{}).Where("refund_time < ?", endTime).Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
 		defer func() {
 			err = rows.Close()
 			if err != nil {
@@ -476,9 +479,9 @@ func allDataRefundRecordForAccountOrderChange() {
 				// refund_status 回款状态
 				if data_exOrder.IsBackstageOrder == 0 {
 					if account_order.OrderStatus == "已支付" {
-						account_order_change.RefundStatus = "全额回款"
+						account_order_change.ReturnStatus = "全额回款"
 					} else {
-						account_order_change.RefundStatus = "未回款"
+						account_order_change.ReturnStatus = "未回款"
 					}
 				} else if data_exOrder.IsBackstageOrder == 1 {
 					switch data_exOrder.ReturnStatus {
@@ -493,11 +496,11 @@ func allDataRefundRecordForAccountOrderChange() {
 				//退款状态
 				switch data_exOrder.RefundStatus {
 				case 1:
-					account_order_change.ReturnStatus = "全额回款"
+					account_order_change.RefundStatus = "全额回款"
 				case 2:
-					account_order_change.ReturnStatus = "部分回款"
+					account_order_change.RefundStatus = "部分回款"
 				case 0:
-					account_order_change.ReturnStatus = "未回款"
+					account_order_change.RefundStatus = "未回款"
 				}
 
 				//付费类型 vip_type
@@ -528,8 +531,10 @@ func allDataRefundRecordForAccountOrderChange() {
 				//
 				account_order_change.UserPhone = data_exOrder.UserPhone
 				account_order_change.UserID = data_exOrder.UserID
-				account_order_change.VipStarttime = data_exOrder.VipStarttime
-				account_order_change.VipEndtime = data_exOrder.VipEndtime
+				if data_exOrder.VipStarttime == nil || data_exOrder.VipStarttime.Year() > 1000 {
+					account_order_change.VipStarttime = data_exOrder.VipStarttime
+					account_order_change.VipEndtime = data_exOrder.VipEndtime
+				}
 				//合同
 				if contract.ContractStatus == 1 {
 					account_order_change.ContractStatus = "已签协议"
@@ -576,5 +581,6 @@ func allDataRefundRecordForAccountOrderChange() {
 	}
 
 	log.Info("allDataRefundRecordForAccountOrderChange 迭代结束", zap.Int64("数据总量", total))
+	log.Info("allDataRefundRecordForAccountOrderChange", zap.Any("处理时长", time.Since(now).Minutes()))
 
 }

account_order/account_return.go

@@ -24,16 +24,16 @@ func allDataReturnMoneyRecordForAccountReturn() {
 		}
 		log.Info("aaaaaaaaaaaa")
 	}()
-
+	now := time.Now()
 	var total int64
-	AnalysisDB.Debug().Model(&ReturnMoneyRecord{}).Where("state = ?", 1).Where("autoUpdate < ?", "2024-10-16").Count(&total)
+	AnalysisDB.Debug().Model(&ReturnMoneyRecord{}).Where("state = ?", 1).Where("autoUpdate < ?", endTime).Count(&total)
 	log.Info("allDataReturnMoneyRecordForAccountReturn", zap.Any("RefundRecord 退款表 总数是", total))
 
 	rowsPerPage := 100                           // 每页的数量
 	totalPages := (int(total) / rowsPerPage) + 1 //总页数
 	for page := 0; page < totalPages; page++ {
 		offset := page * rowsPerPage
-		rows, err := AnalysisDB.Debug().Model(&ReturnMoneyRecord{}).Where("state = ?", 1).Where("autoUpdate < ?", "2024-10-16").Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
+		rows, err := AnalysisDB.Debug().Model(&ReturnMoneyRecord{}).Where("state = ?", 1).Where("autoUpdate < ?", endTime).Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
 		defer func() {
 			err = rows.Close()
 			if err != nil {
@@ -45,6 +45,7 @@ func allDataReturnMoneyRecordForAccountReturn() {
 			var account_return DwdFAccountReturn      //归集后的剑鱼回款表
 			var return_money_record ReturnMoneyRecord //原来回款表
 			var order_sale_record OrderSaleRecord     // 原来业绩表更
+			var order_sale_record2 OrderSaleRecord    // 原来业绩表更2
 			//var account_order_change DwdFAccountOrderChange // 归集后的业绩变更表
 			var data_exOrder DataexportOrder //原来的订单表
 			//var account_order DwdFAccountOrder              //归集后的订单表
@@ -118,7 +119,17 @@ func allDataReturnMoneyRecordForAccountReturn() {
 					account_return.ReturnType = "对公转账"
 				}
 				//业绩统计日期
-				account_return.SaleTime = data_exOrder.SaleTime              //业绩统计日期
+				account_return.SaleTime = data_exOrder.SaleTime //业绩统计日期
+				//归集后的回款表-业绩统计日期。如果订单表的sale_time 值为空，就用order_sale_record.
+				//业绩统计时间,去时间最早的一条并且state!=2和-2；
+				if data_exOrder.SaleTime == nil {
+					//3.业绩变更表- order_sale_record
+					err = JianyuDB.Where("ordercode =? AND state !=2 AND state !=-2 ", return_money_record.OrderCode).Order("id asc").First(&order_sale_record2).Error
+					if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+						log.Info("allDataRefundRecordForAccountOrderChange", zap.Error(err))
+					}
+					account_return.SaleTime = order_sale_record2.StatisticsTime //业绩统计日期
+				}
 				account_return.ReturnTime = return_money_record.ReturnTime   //回款日期
 				account_return.ReturnMoney = return_money_record.ReturnMoney //回款金额
 				//签约主体
@@ -173,11 +184,11 @@ func allDataReturnMoneyRecordForAccountReturn() {
 				//退款状态
 				switch data_exOrder.RefundStatus {
 				case 1:
-					account_return.RefundStatus = "全额回款"
+					account_return.RefundStatus = "全额退款"
 				case 2:
-					account_return.RefundStatus = "部分回款"
+					account_return.RefundStatus = "部分退款"
 				case 0:
-					account_return.RefundStatus = "未回款"
+					account_return.RefundStatus = "未退款"
 				}
 				//关联方式,operate_type;1-手动回款 2-自动回款
 				if return_money_record.OperateType == 1 {
@@ -200,6 +211,7 @@ func allDataReturnMoneyRecordForAccountReturn() {
 		}
 	}
 	log.Info("allDataReturnMoneyRecordForAccountReturn", zap.Any("return_money_record 退款表 迭代结束，总数是", total))
+	log.Info("allDataReturnMoneyRecordForAccountReturn", zap.Any("处理时长", time.Since(now).Minutes()))
 }
 
 // allDataDataexportOrderForAccountReturn 订单表数据处理-dataexport_order
@@ -214,15 +226,16 @@ func allDataDataexportOrderForAccountReturn() {
 		}
 		log.Info("aaaaaaaaaaaa")
 	}()
+	now := time.Now()
 	var total int64
-	AnalysisDB.Debug().Model(&DataexportOrder{}).Where("order_status =? AND is_backstage_order =? AND NOT (pay_way =? AND product_type =?)", 1, 0, "transferAccounts", "VIP订阅").Where("autoUpdate < ?", "2024-10-16").Count(&total)
+	AnalysisDB.Debug().Model(&DataexportOrder{}).Where("order_status =? AND is_backstage_order =? AND NOT (pay_way =? AND product_type =?)", 1, 0, "transferAccounts", "VIP订阅").Where("autoUpdate < ?", endTime).Count(&total)
 	log.Info("allDataDataexportOrderForAccountReturn", zap.Any("dataexport_order 订单表 ，order_status=1 and is_backstage_order=0 ，\n\t并排除pay_way=’transferAccounts’ and product_type=’VIP订阅’，总数是", total))
 
 	rowsPerPage := 100                           // 每页的数量
 	totalPages := (int(total) / rowsPerPage) + 1 //总页数
 	for page := 0; page < totalPages; page++ {
 		offset := page * rowsPerPage
-		rows, err := AnalysisDB.Debug().Model(&DataexportOrder{}).Where("order_status =? AND is_backstage_order =? AND NOT (pay_way =? AND product_type =?)", 1, 0, "transferAccounts", "VIP订阅").Where("autoUpdate < ?", "2024-10-16").Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
+		rows, err := AnalysisDB.Debug().Model(&DataexportOrder{}).Where("order_status =? AND is_backstage_order =? AND NOT (pay_way =? AND product_type =?)", 1, 0, "transferAccounts", "VIP订阅").Where("autoUpdate < ?", endTime).Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
 		defer func() {
 			err = rows.Close()
 			if err != nil {
@@ -234,7 +247,8 @@ func allDataDataexportOrderForAccountReturn() {
 			var data_exOrder DataexportOrder     //原来的订单表
 			var account_return DwdFAccountReturn //归集后的剑鱼回款表
 			//var return_money_record ReturnMoneyRecord //原来回款表
-			var order_sale_record OrderSaleRecord // 原来业绩表更
+			var order_sale_record OrderSaleRecord  // 原来业绩表更
+			var order_sale_record2 OrderSaleRecord // 原来业绩表更
 			// ScanRows 方法用于将一行记录扫描至结构体
 			err = AnalysisDB.ScanRows(rows, &data_exOrder)
 			if err != nil {
@@ -299,6 +313,16 @@ func allDataDataexportOrderForAccountReturn() {
 				//
 
 				account_return.SaleTime = data_exOrder.SaleTime
+				//归集后的回款表-业绩统计日期。如果订单表的sale_time 值为空，就用order_sale_record.
+				//业绩统计时间,去时间最早的一条并且state!=2和-2；
+				if data_exOrder.SaleTime == nil {
+					//3.业绩变更表- order_sale_record
+					err = JianyuDB.Where("ordercode =? AND state !=2 AND state !=-2 ", data_exOrder.OrderCode).Order("id asc").First(&order_sale_record2).Error
+					if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+						log.Info("allDataRefundRecordForAccountOrderChange", zap.Error(err))
+					}
+					account_return.SaleTime = order_sale_record2.StatisticsTime //业绩统计日期
+				}
 				account_return.ReturnTime = data_exOrder.PayTime
 				account_return.ReturnMoney = util.IntAll(filterMap["originalAmount"]) //回款金额
 				//签约主体
@@ -339,17 +363,17 @@ func allDataDataexportOrderForAccountReturn() {
 					}
 				}
 				//下单渠道
-				account_return.OrderCode = data_exOrder.OrderChannel
+				account_return.OrderChannel = data_exOrder.OrderChannel
 				//销售渠道-distribution_channel
 				account_return.DistributionChannel = data_exOrder.DistributionChannel
 				//退款状态-refund_status
 				switch data_exOrder.RefundStatus {
 				case 1:
-					account_return.RefundStatus = "全额回款"
+					account_return.RefundStatus = "全额退款"
 				case 2:
-					account_return.RefundStatus = "部分回款"
+					account_return.RefundStatus = "部分退款"
 				case 0:
-					account_return.RefundStatus = "未回款"
+					account_return.RefundStatus = "未退款"
 				}
 				account_return.OperateType = "自动回款"           //关联方式
 				account_return.OperatePerson = "jianyu@admin" //创建人
@@ -365,4 +389,7 @@ func allDataDataexportOrderForAccountReturn() {
 
 	}
 
+	log.Info("allDataDataexportOrderForAccountReturn 迭代结束", zap.Int64("数据总量", total))
+	log.Info("allDataDataexportOrderForAccountReturn", zap.Any("处理时长", time.Since(now).Minutes()))
+
 }

account_order/account_return_change.go

@@ -29,15 +29,16 @@ func allDataOrderSaleRecordForAccountReturnChange() {
 		}
 		log.Info("allDataOrderSaleRecordForAccountReturnChange,数据处理完毕")
 	}()
+	now := time.Now()
 	var total int64
-	JianyuDB.Debug().Model(&OrderSaleRecord{}).Where("autoUpdate < ?", "2024-10-16").Where("change_reason IN ? AND state IN ? ", []string{"业绩红冲", "销售撞单", "补录销售信息"}, []int{-1, 1}).Count(&total)
+	JianyuDB.Debug().Model(&OrderSaleRecord{}).Where("autoUpdate < ?", endTime).Where("change_reason IN ? AND state IN ? ", []string{"业绩红冲", "销售撞单", "补录销售信息"}, []int{-1, 1}).Count(&total)
 	log.Info("allDataOrderSaleRecordForAccountReturnChange", zap.Any("业绩变更表中取变更类型change_reason是：业绩红冲、销售撞单、补录销售信息，且state in (-1,1)，总数是", total))
 
 	rowsPerPage := 100                           //每页的数量
 	totalPages := (int(total) / rowsPerPage) + 1 //总页数
 	for page := 0; page < totalPages; page++ {
 		offset := page * rowsPerPage
-		rows, err := JianyuDB.Debug().Model(&OrderSaleRecord{}).Where("autoUpdate < ?", "2024-10-16").Where("change_reason IN ? AND state IN ? ", []string{"业绩红冲", "销售撞单", "补录销售信息"}, []int{-1, 1}).Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
+		rows, err := JianyuDB.Debug().Model(&OrderSaleRecord{}).Where("autoUpdate < ?", endTime).Where("change_reason IN ? AND state IN ? ", []string{"业绩红冲", "销售撞单", "补录销售信息"}, []int{-1, 1}).Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
 		defer func() {
 			err = rows.Close()
 			if err != nil {
@@ -209,6 +210,7 @@ func allDataOrderSaleRecordForAccountReturnChange() {
 	}
 
 	log.Info("allDataOrderSaleRecordForAccountReturnChange", zap.Any("数据处理完毕", total))
+	log.Info("allDataOrderSaleRecordForAccountReturnChange", zap.Any("处理时长", time.Since(now).Minutes()))
 }
 
 // allDataRefundRecordForAccountReturnChange 退款表数据处理-data_analysis.refund_record
@@ -219,8 +221,9 @@ func allDataRefundRecordForAccountReturnChange() {
 		}
 		log.Info("allDataRefundRecordForAccountReturnChange", zap.String("数据处理完毕", ""))
 	}()
+	now := time.Now()
 	var total int64
-	AnalysisDB.Debug().Model(&RefundRecord{}).Where("refund_time < ?", "2024-10-16").Count(&total)
+	AnalysisDB.Debug().Model(&RefundRecord{}).Where("refund_time < ?", endTime).Count(&total)
 	log.Info("allDataRefundRecordForAccountReturnChange", zap.Any("总数是", total))
 
 	rowsPerPage := 100                           //每页的数量
@@ -228,7 +231,7 @@ func allDataRefundRecordForAccountReturnChange() {
 
 	for page := 0; page < totalPages; page++ {
 		offset := page * rowsPerPage
-		rows, err := AnalysisDB.Debug().Model(&RefundRecord{}).Where("refund_time < ?", "2024-10-16").Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
+		rows, err := AnalysisDB.Debug().Model(&RefundRecord{}).Where("refund_time < ?", endTime).Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
 		defer func() {
 			err = rows.Close()
 			if err != nil {
@@ -391,4 +394,6 @@ func allDataRefundRecordForAccountReturnChange() {
 	}
 
 	log.Info("allDataRefundRecordForAccountReturnChange", zap.Any("数据处理完毕", total))
+	log.Info("allDataRefundRecordForAccountReturnChange", zap.Any("处理时长", time.Since(now).Minutes()))
+
 }

account_order/account_test.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"fmt"
+	"go.uber.org/zap"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"testing"
+)
+
+func TestOrder(T *testing.T) {
+	username := "root"
+	password := "123"
+	host := "127.0.0.1:3306" // 本地
+	//host := "172.17.162.27:14000" //线上
+	database := "test"
+	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", username, password, host, database)
+	// 连接到数据库
+	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Error), //不打印日志
+	})
+	if err != nil {
+		log.Info("dealAllDataAccountOrder, Rows Error", zap.Error(err))
+		return
+	}
+	rows, err := db.Debug().Model(&DataexportOrder{}).Where("autoUpdate < ?", endTime).
+		Order("id desc").Offset(0).Limit(100).Rows()
+	for rows.Next() {
+		var dataExOrder DataexportOrder //原来的订单表
+		// ScanRows 方法用于将一行记录扫描至结构体
+		err = db.ScanRows(rows, &dataExOrder)
+		if err != nil {
+			log.Info("dealAllDataAccountOrder,ScanRows err ", zap.Error(err))
+		}
+		if dataExOrder.OrderCode != "" {
+			year := dataExOrder.VipStarttime.Year()
+			log.Info("year", zap.Any("aa", year))
+			log.Info("aaa", zap.Any("viptime", dataExOrder.VipStarttime))
+		}
+
+	}
+}

account_order/config.go

@@ -52,6 +52,9 @@ type MysqlConf struct {
 	Table5   string
 	Table6   string
 	Table7   string
+	Stime    string
+	Etime    string
+	Test     bool
 }
 
 type Log struct {
@@ -188,7 +191,11 @@ type DwdFAccountOrder struct {
 }
 
 func (DwdFAccountOrder) TableName() string {
-	return "dwd_f_account_order"
+	if GF.Mysql.Test {
+		return "dwd_f_account_order_test"
+	} else {
+		return "dwd_f_account_order"
+	}
 }
 
 // OrderSaleRecord  原来的业绩变更表
@@ -375,7 +382,11 @@ type DwdFAccountOrderChange struct {
 }
 
 func (DwdFAccountOrderChange) TableName() string {
-	return "dwd_f_account_order_change"
+	if GF.Mysql.Test {
+		return "dwd_f_account_order_change_test"
+	} else {
+		return "dwd_f_account_order_change"
+	}
 }
 
 // RefundRecord 退款表结构体
@@ -425,7 +436,11 @@ type DwdFAccountReturn struct {
 }
 
 func (DwdFAccountReturn) TableName() string {
-	return "dwd_f_account_return"
+	if GF.Mysql.Test {
+		return "dwd_f_account_return_test"
+	} else {
+		return "dwd_f_account_return"
+	}
 }
 
 // AliPay 结构体表示支付宝支付记录表
@@ -498,5 +513,9 @@ type DwdFAccountReturnChange struct {
 
 // TableName 返回表名
 func (DwdFAccountReturnChange) TableName() string {
-	return "dwd_f_account_return_change"
+	if GF.Mysql.Test {
+		return "dwd_f_account_return_change_test"
+	} else {
+		return "dwd_f_account_return_change"
+	}
 }

account_order/config.toml

@@ -20,10 +20,10 @@
 
 [cron] ## 定时任务
 #    spec = "0 */1 * * * *"  ## 5分钟执行一次
-    spec = "0 00 20 * * *"   ## 每天20点执行
-    start = -1               ## 表示开始时间昨天凌晨
-    end = 0                  ## 表示截止时间到今天凌晨
-    delete = -30              ## 删除30天之前的数据
+    spec = "0 00 23 * * *"   ## 每天23点执行
+#    start = -1               ## 表示开始时间昨天凌晨
+#    end = 0                  ## 表示截止时间到今天凌晨
+#    delete = -30             ## 删除30天之前的数据
 
 
 [mysql]
@@ -41,6 +41,8 @@
     table5 = "order_sale_record" ## 业绩变更表
     table6 = "dwd_f_userbase_baseinfo" ## 用户归集表
     table7 = "dwd_f_userbase_id_mapping" ## 用户归集表
+    etime = ""  ## 数据截止时间，"2024-11-02"
+    test = true
 
 
 
@@ -51,6 +53,10 @@
 #    password= "=PDT49#80Z!RVv52_z"
 #    table = "ods_datamonitoring_bidding"    ## 存储结果数据表
 
+[env]
+
+    spec = "0 00 01 * * *"  ## 定时任务，每天凌晨1点执行
+
 [email]
     api = "http://172.17.162.36:19281/_send/_mail"
     to = "wangchengcheng@topnet.net.cn"

account_order/go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect

account_order/go.sum

@@ -96,6 +96,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=

account_order/main.go

@@ -4,28 +4,57 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/robfig/cron/v3"
 	"go.uber.org/zap"
 	"gorm.io/gorm"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"sync"
 	"time"
 )
 
 var (
-	Mgo             *mongodb.MongodbSim
-	MgoP            *mongodb.MongodbSim
-	MgoSpider       *mongodb.MongodbSim
-	AnalysisDB      *gorm.DB
-	JianyuDB        *gorm.DB
-	JianyuSubjectDB *gorm.DB
+	Mgo              *mongodb.MongodbSim
+	MgoP             *mongodb.MongodbSim
+	MgoSpider        *mongodb.MongodbSim
+	AnalysisDB       *gorm.DB
+	JianyuDB         *gorm.DB
+	JianyuSubjectDB  *gorm.DB
+	endTime          string //取数据，时间范围的截止时间；如果为空，就取当前时间
+	accountOrderPool = make(chan DwdFAccountOrder, 5000)
 )
 
+// var endTime = "2024-11-02"
+
 func main() {
-	//dealAllDataAccountOrder() // 1.处理归集后-存量剑鱼订单表-dwd_f_account_order
-	//dealAllDataAccountOrderChangeRecord() //2.处理归集后-存量业绩表更表-dwd_f_account_order_change
-	//dealAllDataAccountReturn() //3.处理归集后-存量剑鱼回款表-dwd_f_account_return
-	dealAllDataAccountReturnChange() //4.处理归集后-剑鱼回款变更表-dwd_f_account_return_change
+	local, _ := time.LoadLocation("Asia/Shanghai")
+	c := cron.New(cron.WithLocation(local), cron.WithSeconds())
+	eid, err := c.AddFunc(GF.Cron.Spec, dealData)
+	if err != nil {
+		log.Info("main", zap.Any("AddFunc err", err))
+	}
+	log.Info("main", zap.Any("eid", eid))
+
+	c.Start()
+	defer c.Stop()
+
+	select {}
+}
+
+// dealData 处理订单相关数据
+func dealData() {
+	//1.清空数据表
+	truncateTables() //清空数据表
+	endTime = time.Now().Format("2006-01-02 15:04:05")
+	log.Info("配置取值时间范围为空", zap.String("默认取值截止时间为：", endTime))
+
+	go saveAccountOrder() //保存订单数据
+	go dealAllDataAccountOrder2()
+	//dealAllDataAccountOrder()              //1.处理归集后-存量剑鱼订单表-dwd_f_account_order
+	go dealAllDataAccountOrderChangeRecord() //2.处理归集后-存量业绩表更表-dwd_f_account_order_change
+	go dealAllDataAccountReturn()            //3.处理归集后-存量剑鱼回款表-dwd_f_account_return
+	go dealAllDataAccountReturnChange()      //4.处理归集后-剑鱼回款变更表-dwd_f_account_return_change
 }
 
 // dealAllDataAccountOrder 处理归集后-存量剑鱼订单表-dwd_f_account_order
@@ -36,9 +65,9 @@ func dealAllDataAccountOrder() {
 		}
 		log.Info("aaaaaaaaaaaa")
 	}()
-
+	now := time.Now()
 	var total int64
-	AnalysisDB.Debug().Model(&DataexportOrder{}).Where("autoUpdate < ?", "2024-10-16").Count(&total)
+	AnalysisDB.Debug().Model(&DataexportOrder{}).Where("autoUpdate < ?", endTime).Count(&total)
 	log.Info("dealAllDataAccountOrder", zap.Any("总数是", total))
 
 	rowsPerPage := 100                           // 每页的数量
@@ -46,7 +75,7 @@ func dealAllDataAccountOrder() {
 
 	for page := 0; page < totalPages; page++ {
 		offset := page * rowsPerPage
-		rows, err := AnalysisDB.Debug().Model(&DataexportOrder{}).Where("autoUpdate < ?", "2024-10-16").Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
+		rows, err := AnalysisDB.Debug().Model(&DataexportOrder{}).Where("autoUpdate < ?", endTime).Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
 		defer func() {
 			err = rows.Close()
 			if err != nil {
@@ -234,8 +263,10 @@ func dealAllDataAccountOrder() {
 				//
 				accountOrder.UserPhone = dataExOrder.UserPhone
 				accountOrder.UserID = dataExOrder.UserID
-				accountOrder.VipStarttime = dataExOrder.VipStarttime
-				accountOrder.VipEndtime = dataExOrder.VipEndtime
+				if dataExOrder.VipStarttime == nil || dataExOrder.VipStarttime.Year() > 1000 {
+					accountOrder.VipStarttime = dataExOrder.VipStarttime
+					accountOrder.VipEndtime = dataExOrder.VipEndtime
+				}
 				// 合同状态
 				err = AnalysisDB.Where("order_code = ? ", dataExOrder.OrderCode).Find(&contract).Error
 				if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
@@ -281,5 +312,353 @@ func dealAllDataAccountOrder() {
 	}
 
 	log.Info("dealAllDataAccountOrder 迭代结束", zap.Int64("数据总量", total))
+	log.Info("dealAllDataAccountOrder", zap.Any("处理时长", time.Since(now).Minutes()))
+
+}
+
+func dealAllDataAccountOrder2() {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Info("dealAllDataAccountOrder2 Panic recovered", zap.Any("reason", r))
+		}
+		log.Info("dealAllDataAccountOrder2 over")
+	}()
+	now := time.Now()
+	var total int64
+	AnalysisDB.Debug().Model(&DataexportOrder{}).Where("autoUpdate < ?", endTime).Count(&total)
+	log.Info("dealAllDataAccountOrder", zap.Any("总数是", total))
+
+	rowsPerPage := 100                           // 每页的数量
+	totalPages := (int(total) / rowsPerPage) + 1 //总页数
+
+	// 控制并发数的 channel，限制最大并发数量为 10
+	concurrencyLimit := 5
+	sem := make(chan struct{}, concurrencyLimit)
+	var wg sync.WaitGroup // 等待所有协程完成
+
+	// 启动多协程处理
+	for i := 0; i < totalPages; i++ {
+		wg.Add(1) // 增加一个 WaitGroup 计数
+		go func(page int) {
+			defer wg.Done() // 完成时减去一个计数
+			// 获取并发信号，限制同时处理的协程数
+			sem <- struct{}{}        // 向 channel 中发送一个空结构体，占用一个槽位
+			defer func() { <-sem }() // 在协程结束时从 channel 中取出，释放一个槽位
+
+			offset := page * rowsPerPage
+			rows, err := AnalysisDB.Debug().Model(&DataexportOrder{}).Where("autoUpdate < ?", endTime).
+				Order("id desc").Offset(offset).Limit(rowsPerPage).Rows()
+			if err != nil {
+				log.Info("dealAllDataAccountOrder, Rows Error", zap.Error(err))
+				return
+			}
+			defer func() {
+				err = rows.Close()
+				if err != nil {
+					log.Info("Err rows.Close", zap.Error(err))
+				}
+			}()
+
+			log.Info("dealAllDataAccountOrder ", zap.Int("current page", page))
+
+			// 处理每一行数据
+			for rows.Next() {
+				var dataExOrder DataexportOrder       //原来的订单表
+				var accountOrder DwdFAccountOrder     //归集后的订单表
+				var user1 DwdFUserbaseBaseinfo        //个人用户表
+				var returnRecord ReturnMoneyRecord    //原来的回款表
+				var returnRecords []ReturnMoneyRecord //原来的回款表
+				var contract Contract                 //合同表
+				// ScanRows 方法用于将一行记录扫描至结构体
+				err = AnalysisDB.ScanRows(rows, &dataExOrder)
+				if err != nil {
+					log.Info("dealAllDataAccountOrder,ScanRows err ", zap.Error(err))
+				}
+
+				filter := dataExOrder.Filter
+				filterMap := make(map[string]interface{})
+				err := json.Unmarshal([]byte(filter), &filterMap)
+				if err != nil {
+					log.Info("dealAllDataAccountOrder, filter.json.Unmarshal", zap.Error(err))
+				}
+				//订单编号不为空
+				if dataExOrder.OrderCode != "" {
+					// 业务逻辑...
+					accountOrder.OrderCode = dataExOrder.OrderCode
+					var osr OrderSaleRecord //原来的业绩变更表
+					err = JianyuDB.Order("id asc").Where("ordercode = ? ", dataExOrder.OrderCode).First(&osr).Error
+					if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+						log.Info("dealAllDataAccountOrder", zap.Error(err))
+					}
+					//有业绩表更表记录
+					if osr.ID > 0 {
+						if osr.SalerName == "-" {
+							accountOrder.SalerName = "运营"
+						} else {
+							accountOrder.SalerName = osr.SalerName
+						}
+						accountOrder.SalerDept = osr.SalerDept
+					}
+					accountOrder.CompanyName = dataExOrder.CompanyName
+					if len(dataExOrder.UserID) > 20 { //个人身份
+						err = JianyuSubjectDB.Where("userid = ? ", dataExOrder.UserID).First(&user1).Error
+						if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+							log.Info("dealAllDataAccountOrder", zap.Error(err))
+						}
+						accountOrder.UserRegtime = user1.LRegistedate
+					} else {
+						//企业身份
+						rs := new(struct {
+							PositionId   int        `gorm:"column:position_id"`
+							LRegistedate *time.Time `gorm:"column:l_registedate"`
+						})
+						if dataExOrder.UserID != "" {
+							sql := fmt.Sprintf(`SELECT im.position_id,ub.l_registedate FROM
+				(SELECT userid,position_id FROM Jianyu_subjectdb.dwd_f_userbase_id_mapping WHERE position_id = %s) im
+				LEFT JOIN Jianyu_subjectdb.dwd_f_userbase_baseinfo ub
+					ON im.userid=ub.userid`, dataExOrder.UserID)
+							err = JianyuSubjectDB.Raw(sql).Scan(&rs).Error
+							if err != nil {
+								log.Info("处理用户订单表数据", zap.String("查询企业用户注册时间失败", dataExOrder.UserID))
+							}
+							accountOrder.UserRegtime = rs.LRegistedate
+						}
+					}
+
+					accountOrder.CreateTime = dataExOrder.CreateTime
+					if dataExOrder.IsBackstageOrder == 0 {
+						accountOrder.ReturnTime = dataExOrder.PayTime
+						accountOrder.TotalReceived = util.IntAll(filterMap["originalAmount"]) //累计已收
+					} else if dataExOrder.IsBackstageOrder == 1 {
+						err = AnalysisDB.Where("order_code = ? ", dataExOrder.OrderCode).Where("state = 1").Order("return_time asc").First(&returnRecord).Error
+						if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+							log.Info("dealAllDataAccountOrder", zap.Error(err))
+						}
+						accountOrder.ReturnTime = returnRecord.ReturnTime
+						err = AnalysisDB.Where("order_code = ? ", dataExOrder.OrderCode).Find(&returnRecords).Error
+						if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+							log.Info("dealAllDataAccountOrder", zap.Error(err))
+						}
+						totalMoney := 0
+						if len(returnRecords) > 0 {
+							for _, v := range returnRecords {
+								totalMoney += v.ReturnMoney
+							}
+						}
+						accountOrder.TotalReceived = totalMoney
+					}
+					accountOrder.SaleTime = dataExOrder.SaleTime
+					accountOrder.OriginalPrice = dataExOrder.OriginalPrice
+					accountOrder.ContractMoney = util.IntAll(filterMap["originalAmount"])
+					accountOrder.Commission = dataExOrder.Commission
+					accountOrder.ProceduresMoney = dataExOrder.ProceduresMoney
+					accountOrder.ReceivableAmount = accountOrder.ContractMoney - accountOrder.Commission - accountOrder.ProceduresMoney
+					accountOrder.ProductType = dataExOrder.ProductType
+					if dataExOrder.ProductType == "大会员" || dataExOrder.ProductType == "大会员-子账号" || dataExOrder.ProductType == "大会员-补充包" {
+						switch util.IntAll(filterMap["level"]) {
+						case 1:
+							accountOrder.DataSpec = "专家版"
+						case 2:
+							accountOrder.DataSpec = "智慧版"
+						case 3:
+							accountOrder.DataSpec = "商机版"
+						case 4:
+							accountOrder.DataSpec = "试用版"
+						case 5:
+							accountOrder.DataSpec = "定制版"
+						case 6:
+							accountOrder.DataSpec = "商机版2.0"
+						case 7:
+							accountOrder.DataSpec = "专家版2.0"
+						}
+						//付费类型  VipType
+						switch util.IntAll(filterMap["recordPayType"]) {
+						case 1:
+							accountOrder.VipType = "购买"
+						case 2:
+							accountOrder.VipType = "续费"
+						case 3:
+							accountOrder.VipType = "升级"
+						case 4:
+							accountOrder.VipType = "试用"
+						}
+					} else {
+						accountOrder.DataSpec = dataExOrder.DataSpec
+						////付费类型  VipType
+						switch dataExOrder.VipType {
+						case 0:
+							accountOrder.VipType = "购买"
+						case 1:
+							accountOrder.VipType = "续费"
+						case 2:
+							accountOrder.VipType = "升级"
+						case 3:
+							accountOrder.VipType = "试用"
+						}
+					}
+					//OrderStatus 订单状态
+					if dataExOrder.RefundStatus == 1 || dataExOrder.RefundStatus == 2 {
+						accountOrder.OrderStatus = "已完成"
+					} else {
+						switch dataExOrder.OrderStatus {
+						case 1:
+							accountOrder.OrderStatus = "已完成"
+						case -1:
+							accountOrder.OrderStatus = "逻辑删除"
+						case -2:
+							accountOrder.OrderStatus = "已取消"
+						case -3:
+							accountOrder.OrderStatus = "已取消"
+						case 0:
+							accountOrder.OrderStatus = "未完成"
+						}
+					}
+					//回款状态
+					if dataExOrder.IsBackstageOrder == 0 {
+						if accountOrder.OrderStatus == "已支付" {
+							accountOrder.ReturnStatus = "全额回款"
+						} else {
+							accountOrder.ReturnStatus = "未支付"
+						}
+					} else if dataExOrder.IsBackstageOrder == 1 {
+						switch dataExOrder.ReturnStatus {
+						case 1:
+							accountOrder.ReturnStatus = "全额回款"
+						case 2:
+							accountOrder.ReturnStatus = "部分回款"
+						case 0:
+							accountOrder.ReturnStatus = "未回款"
+						}
+					}
+					//refund_status 回款状态
+					switch dataExOrder.RefundStatus {
+					case 0:
+						accountOrder.RefundStatus = "未退款"
+					case 1:
+						accountOrder.RefundStatus = "全额退款"
+					case 2:
+						accountOrder.RefundStatus = "部分退款"
+					}
+					//
+					accountOrder.UserPhone = dataExOrder.UserPhone
+					accountOrder.UserID = dataExOrder.UserID
+					if dataExOrder.VipStarttime == nil || dataExOrder.VipStarttime.Year() > 1000 {
+						accountOrder.VipStarttime = dataExOrder.VipStarttime
+						accountOrder.VipEndtime = dataExOrder.VipEndtime
+					}
+					// 合同状态
+					err = AnalysisDB.Where("order_code = ? ", dataExOrder.OrderCode).Find(&contract).Error
+					if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+						log.Info("dealAllDataAccountOrder", zap.Error(err))
+					}
+					if contract.ContractStatus == 1 {
+						accountOrder.ContractStatus = "已签协议"
+					} else if contract.ContractStatus == 0 {
+						accountOrder.ContractStatus = "未签协议"
+					}
+					accountOrder.ContractCode = contract.ContractCode
+					accountOrder.ContractTime = contract.ContractTime
+					if dataExOrder.SigningSubject == "h01" {
+						accountOrder.SigningSubject = "北京剑鱼信息技术有限公司"
+					} else if dataExOrder.SigningSubject == "h02" {
+						accountOrder.SigningSubject = "北京拓普丰联信息科技股份有限公司"
+					}
+					accountOrder.OrderChannel = dataExOrder.OrderChannel
+					accountOrder.DistributionChannel = dataExOrder.DistributionChannel
+					if dataExOrder.IsBackstageOrder == 1 {
+						accountOrder.IsBackstageOrder = "是"
+					} else {
+						accountOrder.IsBackstageOrder = "否"
+					}
+					switch dataExOrder.PayWay {
+					case "ali", "ali_app", "ali_pc": //ali、ali_app、ali_pc 处理为“支付宝”
+						accountOrder.PayWay = "支付宝"
+					case "wx", "wx_app", "wx_js", "wx_pc": // wx、wx_app、wx_js、wx_pc 处理为“微信”
+						accountOrder.PayWay = "微信"
+					case "transferAccounts":
+						accountOrder.PayWay = "对公转账"
+					default:
+						accountOrder.PayWay = dataExOrder.PayWay
+					}
+					//生成订单数据
+					accountOrderPool <- accountOrder
+					//err = AnalysisDB.Create(&accountOrder).Error
+					//if err != nil {
+					//	log.Info("dealAllDataAccountOrder Create ", zap.Error(err))
+					//}
+				}
+			}
+		}(i)
+	}
+
+	// 等待所有协程处理完成
+	wg.Wait()
+
+	log.Info("dealAllDataAccountOrder 迭代结束", zap.Int64("数据总量", total))
+	log.Info("dealAllDataAccountOrder", zap.Any("处理时长", time.Since(now).Minutes()))
+}
+
+// saveAccountOrder 保存归集订单表
+func saveAccountOrder() {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Info("saveAccountOrder Panic recovered", zap.Any("reason", r))
+		}
+		log.Info("saveAccountOrder over")
+	}()
+
+	for {
+		select {
+		case accountOrder, ok := <-accountOrderPool:
+			if !ok {
+				// 通道已关闭，退出循环
+				return
+			}
+			err := AnalysisDB.Create(&accountOrder).Error
+			if err != nil {
+				log.Info("dealAllDataAccountOrder Create ", zap.Error(err), zap.String("order_code", accountOrder.OrderCode))
+			}
+		}
+	}
+}
+
+// truncateTables 清空订单相关数据表
+func truncateTables() {
+	var (
+		account_order         DwdFAccountOrder
+		account_order_change  DwdFAccountOrderChange
+		account_return        DwdFAccountReturn
+		account_return_change DwdFAccountReturnChange
+	)
+
+	table1 := account_order.TableName()
+	table2 := account_order_change.TableName()
+	table3 := account_return.TableName()
+	table4 := account_return_change.TableName()
+
+	// 清空表 1
+	db, err := AnalysisDB.DB()
+	if err != nil {
+		panic("获取数据库连接对象失败：" + err.Error())
+	}
+	_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", table1))
+	if err != nil {
+		log.Info("清空失败", zap.String("数据表", table1))
+	}
+	_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", table2))
+	if err != nil {
+		log.Info("清空失败", zap.String("数据表", table2))
+	}
+
+	_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", table3))
+	if err != nil {
+		log.Info("清空失败", zap.String("数据表", table3))
+	}
+
+	_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", table4))
+	if err != nil {
+		log.Info("清空失败", zap.String("数据表", table4))
+	}
 
+	log.Info("所有数据表清空完毕", zap.String("数据表是：", fmt.Sprintf("%s,%s,%s,%s", table1, table2, table3, table4)))
 }

account_order/readme.md

@@ -21,7 +21,9 @@
 
 
 ##  _存量数据_ 范围
-autoUpdate < 2024-10-16 
+autoUpdate < etime;大部分数据表取数据，都是按照 该字段取值，部分如：退款表，取值是按照`refund_time` 字段取值。
+如果配置文件没有配置 `etime`或者为空，就取程序运行时间为截止时间。
+
 
 
 

bidding_tags/config.toml

@@ -68,4 +68,4 @@ format = "json"
     ## 情报_安防
     rulee ="A02091107\nA023704\nA023705\nA023709\nB0510\nB0601\nB060101\nB060102\nB060103\nB060104\nB060199\nB0602\nB060201\nB060202\nB060203\nB060204\nB060205\nB060206\nB060207\nB060208\nB060299\nC160201\nC160202\nC160204\nC160701"
     ## 情报_法务;使用中标单位,标题匹配
-    rulef = "律师\n事务所\n法律服务\n法律顾问\n法律咨询\n法律尽职调查\n法律援助"
+    rulef = "律师\n事务所\n法律服务\n法律顾问\n法律咨询\n法律尽职调查\n法律援助\n调解服务\n诉讼服务\n法律指导\n法治宣传\n法治服务\n律师服务\n诉讼服务\n法治宣传\n律师服务\n司法服务\n法治服务\n仲裁服务\n法律援助\n法律需求\n知识产权服务\n法律指导"

bidding_tags/main.go

@@ -146,9 +146,15 @@ func main() {
 	//dealAllAi() //大模型
 
 	//dealInc()
-
-	//dealTopInformationAi7(nil)
-
+	//------------------------------------------------//
+	//where := map[string]interface{}{
+	//	"_id": map[string]interface{}{
+	//		//"$lt": mongodb.StringTOBsonId("6715b7e8b25c3e1debabb8eb"),
+	//		"$lt": mongodb.StringTOBsonId("672ce44bb25c3e1deb1109d7"), //2024-11-08;调用ai8
+	//	},
+	//}
+	//dealTopInformationAi7(where)//处理存量数据
+	//------------------------------------------------//
 	local, _ := time.LoadLocation("Asia/Shanghai")
 	c := cron.New(cron.WithLocation(local), cron.WithSeconds())
 	_, err := c.AddFunc(GF.Env.Spec, dealInc)
@@ -745,7 +751,7 @@ func dealTopInformationAi7(where interface{}) {
 	sess := Mgo.GetMgoConn()
 	defer Mgo.DestoryMongoConn(sess)
 	count := 0
-	ch := make(chan bool, 50)
+	ch := make(chan bool, 100)
 	wg := &sync.WaitGroup{}
 	var lines = make([]interface{}, 0)
 	it := sess.DB(GF.Mongo.DB).C(GF.Mongo.Coll).Find(where).Select(nil).Sort("-_id").Iter()
@@ -807,7 +813,8 @@ func dealTopInformationAi7(where interface{}) {
 					<-ch
 					wg.Done()
 				}()
-				resu, err := ZpAI7(key, model, lines)
+				//resu, err := ZpAI7(key, model, lines)
+				resu, err := ZpAI8(key, model, lines)
 				if err != nil {
 					jlog.Info("dealTopInformationAi7,第一次请求失败", zap.Error(err))
 					resu, err = ZpAI7(key, model, lines)

bidding_tags/tools.go

@@ -565,3 +565,95 @@ C、修正结果，输出最终答案
 	}
 	return
 }
+
+// ZpAI8 经过刘康佳优化
+func ZpAI8(apiKey, model string, data interface{}) (rest map[string][]map[string]interface{}, err error) {
+	expireAtTime := int64(1735660800) // token 过期时间
+	text := fmt.Sprintf(`
+"""
+你是一个全行业领域的专家，对招投标行业领域分类了如指掌
+
+任务如下：
+# 输入内容
+公告start
+%s
+
+公告end
+
+# 流程：
+流程1:你需要分析提取出对应title中采购的标的物信息（去掉机构、公告类型、地点、网上超市、合同等无意义内容）
+流程2: 根据流程1和流程2的结果，通过理解每个领域的定义将标的物映射到下面合理的领域名称。
+
+# 限制
+    - ‘其他领域’标签只能单独出现，不可以和出现在别的领域标签后面
+    - 如果没有标的物或者下面领域标签没有适合的结果，则领域标签只输出‘其他领域’
+    - 不可以随意捏造并使用不存在的领域名称
+# 领域名称定义：
+    - 其他领域：凡是不存在下面领域定义的标的物,包括但不限于农副产品、物资、实验设备、监理服务、采购意向等，直接输出"其他领域"
+    - 车辆领域：涉及所有与车辆相关的业务和服务，包括但不限于新车购买、二手车交易、车辆租赁、汽车维修保养、汽车配件销售、车辆保险、驾驶培训等；
+    - 安防领域：安防领域通常包括但不限于视频监控、入侵检测、安全警报、门禁系统、防盗报警、周界防护、智能安防解决方案、火灾报警与应急处理、人员识别与追踪、安全巡逻、数据分析与风险评估等，例如：安防监控、安防产品、周界防范、安防监控系统、信息安全、数据安全等；
+    - 印务领域：跟印刷或出版相关的行业和活动，包括印刷技术、印刷材料、印刷设备与维护、广告服务、出版服务、包装设计等，还包括打印机、印刷机、油墨、打印纸张等印刷相关材料和配件的内容，如印刷、打印复印、打印设备、云打印、印刷服务等。
+    - 环境领域：跟环境保护或环境治理相关的内容，如电子废物处理、垃圾处理、生态污水处理、生态修复、环境监管体系、废物管理与资源化等
+    - 家具领域：跟家具相关的内容，包括家用家具、办公家具、教学家具、医院家具、宿舍家具、定制家具、家具采购、家具设计与安装、家具维修与翻新等，如办公桌、电脑桌、文件柜、人体工学椅、欧式家具、木质家具、木地板、家具饰品等；
+    - 财务审计领域：涉及财务和审计相关的业务和服务，包括但不限于资产评估、财务报表审计、内部控制审计、税务审计、经济责任审计、财务咨询、会计服务、资产清查、纳税申报、破产清算等。
+    - 管理咨询领域：包括但不限于工程咨询管理、人力资源管理、市场营销策划、业务流程优化、企业管理咨询、管理培训、战略顾问、人力资源咨询、相关培训项目业务、咨询服务相关业务等，
+    - 保险领域：涉及保险产品的销售、服务及管理，包括但不限于人寿保险、财产保险、健康保险、意外伤害保险、责任保险、再保险、保险理赔、风险管理、保险产品设计与开发等，如保险代理、保险经纪、保险咨询服务、保险解决方案等。
+    - 工程设计咨询领域：注意：该领域只针对服务类型，指涉及工程设计、设计施工承包、工程造价咨询、工程技术可行性、环境影响研究、经济型评估等设计前的咨询服务以及勘测相关技术服务、建筑设计等
+    - 招标代理领域：仅限于标的物主干部分是招投标代理服务等，
+
+A、依次获取公告title中采购的标的物判断对应领域名称，严格按照下面JSON格式输出
+
+{
+  "result":[
+      {"title":"title（原title文本）","class":"领域名称"(如果是单领域则只输出一个领域，如果公告属于多个领域请用','分割)},
+    ]
+}
+
+B、审查结果
+（1）审查A的结果输出，判断输出领域与公告采购标的物是否强相关，否则输出"其他领域"
+
+
+C、修正结果，输出最终答案
+
+答（干净利落，不解释,只要C结果,严格按照JSON格式返回数据，不需要多余的解释说明）
+"""
+
+`, data)
+
+	originalValue := 0.01
+	originalValue2 := 0.7
+	mssage := zhipu.PostParams{
+		Model: model,
+		Messages: []zhipu.Message{
+			{
+				Role:    "user", // 消息的角色信息 详见文档
+				Content: text,
+			},
+		},
+		Temperature: &originalValue,
+		TopP:        &originalValue2,
+	}
+
+	postResponse, err := zhipu.BeCommonModel(expireAtTime, mssage, apiKey, time.Minute*10)
+	if err != nil {
+		log.Println("请求智普失败", err)
+		return
+	}
+
+	if choices, ok := postResponse["choices"].([]interface{}); ok {
+		if choice, ok2 := choices[0].(map[string]interface{}); ok2 {
+			if message, ok3 := choice["message"].(map[string]interface{}); ok3 {
+				if content, ok4 := message["content"].(string); ok4 {
+					content = strings.ReplaceAll(content, "\n", "")
+					content = strings.ReplaceAll(content, "json", "")
+					content = strings.ReplaceAll(content, "`", "")
+					err = json.Unmarshal([]byte(content), &rest)
+					//if err != nil {
+					//	log.Println("Unmarshal err", err, "content:", content)
+					//}
+				}
+			}
+		}
+	}
+	return
+}