更新程序

account_order/clickhouse.go

@@ -10,45 +10,55 @@ import (
 // truncateClickhouse 清空Clickhouse数据表
 func truncateClickhouse() {
 	var (
-		f_order DwdFOrder
-		//account_order_change  DwdFAccountOrderChange
-		//account_return        DwdFAccountReturn
-		//account_return_change DwdFAccountReturnChange
+		f_order         DwdFOrder
+		f_order_change  DwdFOrderChange
+		f_return        DwdFReturn
+		f_return_change DwdFReturnChange
 	)
 
 	table1 := f_order.TableName()
-	//table2 := account_order_change.TableName()
-	//table3 := account_return.TableName()
-	//table4 := account_return_change.TableName()
+	table2 := f_order_change.TableName()
+	table3 := f_return.TableName()
+	table4 := f_return_change.TableName()
 
 	// 清空表 1
 	db, err := ClickhouseDB.DB()
 	if err != nil {
 		panic("获取数据库连接对象失败：" + err.Error())
 	}
-	_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s.%s", GF.Clickhouse.DB, table1))
+	s1 := fmt.Sprintf("TRUNCATE TABLE %s.%s", GF.Clickhouse.DB, table1)
+	_, err = db.Exec(s1)
 	if err != nil {
-		log.Info("清空失败", zap.String("数据表", table1))
+		log.Info("清空失败", zap.String("数据表", s1))
 	} else {
-		log.Info("清空成功", zap.String("数据表", table1))
+		log.Info("清空成功", zap.String("数据表", s1))
+	}
+	//清空表 2
+	s2 := fmt.Sprintf("TRUNCATE TABLE %s.%s", GF.Clickhouse.DB, table2)
+	_, err = db.Exec(s2)
+	if err != nil {
+		log.Info("清空失败", zap.String("数据表", s2))
+	} else {
+		log.Info("清空成功", zap.String("数据表", s2))
 	}
 
-	//_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", table2))
-	//if err != nil {
-	//	log.Info("清空失败", zap.String("数据表", table2))
-	//}
-	//
-	//_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", table3))
-	//if err != nil {
-	//	log.Info("清空失败", zap.String("数据表", table3))
-	//}
-	//
-	//_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", table4))
-	//if err != nil {
-	//	log.Info("清空失败", zap.String("数据表", table4))
-	//}
+	//清空表 3
+	s3 := fmt.Sprintf("TRUNCATE TABLE %s.%s", GF.Clickhouse.DB, table3)
+	_, err = db.Exec(s3)
+	if err != nil {
+		log.Info("清空失败", zap.String("数据表", s3))
+	} else {
+		log.Info("清空成功", zap.String("数据表", s3))
+	}
+
+	//清空表 4
+	s4 := fmt.Sprintf("TRUNCATE TABLE %s.%s", GF.Clickhouse.DB, table4)
+	_, err = db.Exec(fmt.Sprintf("TRUNCATE TABLE %s", s4))
+	if err != nil {
+		log.Info("清空失败", zap.String("数据表", s4))
+	}
 	//
-	//log.Info("所有数据表清空完毕", zap.String("数据表是：", fmt.Sprintf("%s,%s,%s,%s", table1, table2, table3, table4)))
+	log.Info("所有数据表清空完毕", zap.String("数据表是：", fmt.Sprintf("%s,%s,%s,%s", table1, table2, table3, table4)))
 }
 
 // DwdFOrder 订单表-Clickhouse

account_order/main.go

@@ -32,8 +32,8 @@ var (
 func main() {
 	local, _ := time.LoadLocation("Asia/Shanghai")
 	c := cron.New(cron.WithLocation(local), cron.WithSeconds())
-	//eid, err := c.AddFunc(GF.Cron.Spec, dealData)		//tidb 数据库
-	eid, err := c.AddFunc(GF.Cron.Spec, dealDataForClickhouse) //Clickhouse数据库
+	eid, err := c.AddFunc(GF.Cron.Spec, dealData) //tidb 数据库
+	//eid, err := c.AddFunc(GF.Cron.Spec, dealDataForClickhouse) //Clickhouse数据库
 	if err != nil {
 		log.Info("main", zap.Any("AddFunc err", err))
 	}
@@ -554,6 +554,9 @@ func saveDataClickhouse() {
 		batchSize := 100 // 设置批量插入大小
 		var batch []DwdFAccountOrder
 		for accountOrder := range accountOrderPool {
+			accountOrder.SaleTime = validateAndFormatDate(accountOrder.SaleTime)
+			accountOrder.ReturnTime = validateAndFormatDate(accountOrder.ReturnTime)
+			accountOrder.RefundTime = validateAndFormatDate(accountOrder.RefundTime)
 			batch = append(batch, accountOrder)
 			if len(batch) >= batchSize {
 				err := ClickhouseDB.Model(DwdFOrder{}).CreateInBatches(batch, batchSize).Error
@@ -579,6 +582,8 @@ func saveDataClickhouse() {
 		batchSize := 100 // 设置批量插入大小
 		var batch []DwdFAccountOrderChange
 		for accountOrderChange := range accountOrderChangePool {
+			accountOrderChange.SaleTime = validateAndFormatDate(accountOrderChange.SaleTime)
+			accountOrderChange.ContractTime = validateAndFormatDate(accountOrderChange.ContractTime)
 			batch = append(batch, accountOrderChange)
 			if len(batch) >= batchSize {
 				err := ClickhouseDB.Model(DwdFOrderChange{}).CreateInBatches(batch, batchSize).Error
@@ -602,6 +607,8 @@ func saveDataClickhouse() {
 		batchSize := 100 // 设置批量插入大小
 		var batch []DwdFAccountReturn
 		for accountReturn := range accountReturnPool {
+			accountReturn.SaleTime = validateAndFormatDate(accountReturn.SaleTime)
+			accountReturn.ReturnTime = validateAndFormatDate(accountReturn.ReturnTime)
 			batch = append(batch, accountReturn)
 			if len(batch) >= batchSize {
 				err := ClickhouseDB.Model(DwdFReturn{}).CreateInBatches(batch, batchSize).Error
@@ -626,6 +633,8 @@ func saveDataClickhouse() {
 		batchSize := 100 // 设置批量插入大小
 		var batch []DwdFAccountReturnChange
 		for accountReturnChange := range accountReturnChangePool {
+			accountReturnChange.SaleTime = validateAndFormatDate(accountReturnChange.SaleTime)
+			accountReturnChange.ReturnTime = validateAndFormatDate(accountReturnChange.ReturnTime)
 			batch = append(batch, accountReturnChange)
 			if len(batch) >= batchSize {
 				err := ClickhouseDB.Model(DwdFReturnChange{}).CreateInBatches(batch, batchSize).Error

account_order/tools.go

@@ -46,3 +46,15 @@ func HTTPRequest(method, url string, headers map[string]string, body []byte) (st
 
 	return string(respBody), nil
 }
+
+// validateAndFormatDate 验证时间字段
+func validateAndFormatDate(dateTime *time.Time) *time.Time {
+	if dateTime == nil {
+		return nil // 如果是空，返回 nil
+	}
+
+	if dateTime.Before(time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)) || dateTime.After(time.Date(2105, 12, 31, 23, 59, 59, 0, time.UTC)) {
+		return nil
+	}
+	return dateTime
+}

project_portrait/main.go

@@ -27,7 +27,7 @@ var (
 	portraitMgo   = ""                // MongoDB 的表名
 	GF            GlobalConf
 	// 情报分类一级标签
-	topInfos = []string{"情报_物业", "情报_环境采购", "情报_印务商机", "情报_家具招投标", "情报_车辆租赁", "情报_安防"}
+	topInfos = []string{"情报_物业", "情报_环境采购", "情报_印务商机", "情报_家具招投标", "情报_车辆租赁", "情报_安防", "情报_法务", "情报_财务审计", "情报_招标代理", "情报_管理咨询", "情报_保险", "情报_工程设计咨询"}
 )
 
 type PortraitData struct {
@@ -470,7 +470,7 @@ func processHit(hit *elastic.SearchHit, client *elastic.Client, MgoB *mongodb.Mo
 
 		searchResult, err := searchService2.Do(context.Background())
 		if err != nil {
-			log.Fatalf("Error getting response: %s", err)
+			log.Printf("Error getting response: %s", err)
 		}
 
 		if searchResult.Hits.TotalHits.Value > 0 {