update

zhaiquan/config.go

@@ -289,3 +289,26 @@ func (ProjectBondRelation) TableName() string {
 	}
 
 }
+
+// ProjectListInfo ProjectListInfo
+type ProjectListInfo struct {
+	ID                 uint    `gorm:"primaryKey;autoIncrement;column:id" json:"id"`
+	ProjectName        string  `gorm:"column:project_name" json:"project_name"`
+	TotalInvestment    float64 `gorm:"column:total_investment" json:"total_investment"`         // 总投资（亿元）
+	Area               string  `gorm:"column:area" json:"area"`                                 // 省份
+	City               string  `gorm:"column:city" json:"city"`                                 // 城市
+	PresentIssueAmount float64 `gorm:"column:present_issue_amount" json:"present_issue_amount"` // 累计发行额（亿元）
+	IssueTerm          string  `gorm:"column:issue_term" json:"issue_term"`                     // 发行期限（年）
+	IssueInterestRate  string  `gorm:"column:issue_interest_rate" json:"issue_interest_rate"`   // 发行利率（%）
+	IssueDate          string  `gorm:"column:issue_date" json:"issue_date"`                     // 发行时间
+	District           string  `gorm:"column:district" json:"district"`                         // 区县
+}
+
+func (ProjectListInfo) TableName() string {
+
+	if GF.Mysql.Prefix != "" {
+		return GF.Mysql.Prefix + "_project_list_info"
+	} else {
+		return "project_list_info"
+	}
+}

zhaiquan/config.toml

@@ -6,8 +6,10 @@
     username = ""
     password = ""
     direct = true
-    list ="specialbond_list,specialbond_list_history,specialbond_list_202502"    ## 列表数据表
-    detail = "specialbond_detail,specialbond_detail_history,specialbond_detail_202502" ## 详情数据表
+    list = "specialbond_list_weekly"
+    detail = "specialbond_detail_weekly"
+#    list ="specialbond_list,specialbond_list_history,specialbond_list_202502"    ## 列表数据表
+#    detail = "specialbond_detail,specialbond_detail_history,specialbond_detail_202502" ## 详情数据表
 
 
 

zhaiquan/export.go

@@ -14,11 +14,12 @@ import (
 // exportData 导出专项债数据
 func exportData() {
 	//查询条件
-	area := "安徽省"         //省份
-	start := "2025-01-01" // 发售时间
+	area := "湖北省" //省份
+	area_list := "湖北"
+	start := "2025" // 发售时间
 
 	//
-	file := time.Now().Format("20060102") + "专项债数据.xlsx"
+	file := area + time.Now().Format("20060102") + "专项债数据.xlsx"
 	currentPwd, _ := os.Getwd()
 	exportFile := fmt.Sprintf("%s/%s", currentPwd, file)
 	xlsx := excelize.NewFile(excelize.Options{ShortDatePattern: "yyyy/m/dd"})
@@ -113,10 +114,84 @@ func exportData() {
 		if err != nil {
 			log.Error("exportData；Error ", zap.Error(err))
 		}
-		_ = xlsx.SetCellStyle(sheet2, fmt.Sprintf("%s%d", "A", line2), "BA"+strconv.Itoa(line), styleOne)
+		_ = xlsx.SetCellStyle(sheet2, fmt.Sprintf("%s%d", "A", line2), "BA"+strconv.Itoa(line2), styleOne)
 	}
 
+	log.Info("exportData", zap.Any("债券数据处理完毕,总数是:", len(bonds)))
+
+	//3.项目列表
+	sheet3 := "项目列表"
+	xlsx.NewSheet(sheet3)
+	line3 := 0
+	project_lists := make([]map[string]interface{}, 0)
+	err = MysqlDB.Model(&ProjectListInfo{}).Where(" area = ?  AND  issue_date >= ? ", area_list, start).Find(&project_lists).Error
+	if err != nil {
+		log.Error("exportData；Error checking for existing project", zap.Error(err))
+	}
+	keys3 := []string{"project_name", "total_investment", "area", "city", "district", "present_issue_amount", "issue_term", "issue_interest_rate", "issue_date"}
+	titles3 := []string{"项目名称", "总投资（万元）", "省份", "地市", "区县", "累计发行额（亿元）", "发行期限（年）", "发行利率（%）", "发行时间"}
+
+	line3++
+	//设置第一行title
+	_ = xlsx.SetSheetRow(sheet3, fmt.Sprintf("%s%d", "A", line3), &titles3)
+	//
+	for _, v := range project_lists {
+		//处理数据到文件
+		line3++
+		val := []interface{}{}
+		for _, vv := range keys3 {
+			val = append(val, v[vv])
+		}
+		err := xlsx.SetSheetRow(sheet3, fmt.Sprintf("%s%d", "A", line3), &val)
+		if err != nil {
+			log.Error("exportData；Error ", zap.Error(err))
+			return
+		}
+		_ = xlsx.SetCellStyle(sheet3, fmt.Sprintf("%s%d", "A", line3), "BA"+strconv.Itoa(line3), styleOne)
+	}
+	log.Info("exportData", zap.Any("项目列表数据处理完毕,总数是:", len(project_lists)))
+
+	//4. 项目债券关系
+	/**
+	SELECT d.*
+	FROM zxz_project_issue_details d
+	JOIN zxz_project_base_info p ON d.project_name = p.project_name
+	WHERE p.area = '天津市';
+	*/
+	sheet4 := "项目债券关系"
+	xlsx.NewSheet(sheet4)
+	line4 := 0
+	keys4 := []string{"project_name", "project_bach_name", "bond_name", "first_publish_date", "batch_num", "present_issue_amount", "issue_interest_rate", "total_issue_amount"}
+	titles4 := []string{"项目名称", "项目批次名称", "债券名称", "发布时间", "批次", "发行额", "发行利率", "累计发行金额"}
+
+	line4++
+	//设置第一行title
+	_ = xlsx.SetSheetRow(sheet4, fmt.Sprintf("%s%d", "A", line4), &titles4)
+	// 查询天津市的债券数据
+	var issues = make([]map[string]interface{}, 0)
+	err = MysqlDB.Table("zxz_project_issue_details d").
+		Select("d.*").
+		Joins("JOIN zxz_project_base_info p ON d.project_name = p.project_name ").
+		Where("p.area =  ?  AND p.create_date >= ? ", area, start).
+		Find(&issues).Error
+	for _, v := range issues {
+		//处理数据到文件
+		line4++
+		val := []interface{}{}
+		for _, vv := range keys4 {
+			val = append(val, v[vv])
+		}
+		err := xlsx.SetSheetRow(sheet4, fmt.Sprintf("%s%d", "A", line4), &val)
+		if err != nil {
+			log.Error("exportData；Error ", zap.Error(err))
+			return
+		}
+		_ = xlsx.SetCellStyle(sheet4, fmt.Sprintf("%s%d", "A", line4), "BA"+strconv.Itoa(line4), styleOne)
+	}
+
+	//
+	//保存文件
 	xlsx.Path = exportFile
 	xlsx.Save()
-	log.Info("exportData", zap.Any("债券数据处理完毕,总数是:", len(bonds)))
+	log.Info("exportData", zap.Any("项目债券关系处理完毕,总数是:", len(issues)))
 }

zhaiquan/go.mod

@@ -3,6 +3,7 @@ module zhaiquan
 go 1.22.6
 
 require (
+	github.com/robfig/cron/v3 v3.0.1
 	github.com/spf13/viper v1.19.0
 	github.com/xuri/excelize/v2 v2.9.0
 	go.uber.org/zap v1.22.0

zhaiquan/go.sum

@@ -104,6 +104,8 @@ github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7
 github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
 github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00=
 github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=

zhaiquan/main.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"github.com/robfig/cron/v3"
 	"go.uber.org/zap"
 	"gorm.io/gorm"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"strings"
+	"time"
 )
 
 var (
@@ -16,10 +18,34 @@ var (
 
 func main() {
 	Init()
+	local, _ := time.LoadLocation("Asia/Shanghai")
+	c := cron.New(cron.WithLocation(local), cron.WithSeconds())
+	eid, err := c.AddFunc(GF.Cron.Spec, importData) // 处理增量专项债
+	if err != nil {
+		log.Info("main", zap.Any("AddFunc err", err))
+	}
+	log.Info("main", zap.Any("eid", eid))
+
+	c.Start()
+	defer c.Stop()
+
+	select {}
+	//importData()	//导入专项债数据
 	//dealProject()             // 处理项目、债券数据
 	//dealProjectBondRelation() //更新项目和债券的关联关系
+
+	//dealProjectListAll() //处理存量项目列表数据;补充列表数据使用
 	/*导出数据*/
-	exportData()
+	//exportData()
+}
+
+// importData 导入数据
+func importData() {
+	dealProject()             // 处理项目、债券数据
+	dealProjectBondRelation() //更新项目和债券的关联关系
+
+	log.Info("importData:", zap.String("count", "数据处理完毕"))
+
 }
 
 // dealProject 处理项目相关数据
@@ -52,6 +78,8 @@ func dealProject() {
 			}
 
 			projectName := util.ObjToString(tmp["projectName"])
+			//0.项目列表数据
+			dealProjectList(tmp)
 			//
 			where := map[string]interface{}{
 				"project.jcxx.projectName": projectName,
@@ -60,6 +88,7 @@ func dealProject() {
 			if len(*detail) == 0 {
 				continue
 			}
+
 			//1.项目数据
 			if project, ok := (*detail)["project"].(map[string]interface{}); ok {
 				//1.基础信息
@@ -470,3 +499,105 @@ func dealProjectBondRelation() {
 		log.Info("dealProjectBondRelation", zap.Any("数据处理完毕", count))
 	}
 }
+
+// dealProjectList 处理项目列表数据
+func dealProjectList(tmp map[string]interface{}) {
+	projectName := util.ObjToString(tmp["projectName"])
+	project_list_exist := ProjectListInfo{}
+	err := MysqlDB.Where(&ProjectListInfo{ProjectName: projectName}).First(&project_list_exist).Error
+	if err != nil && err != gorm.ErrRecordNotFound {
+		// 处理查询错误
+		log.Error("dealProjectList；Error checking for existing project", zap.Error(err))
+	}
+
+	if project_list_exist.ID > 0 {
+		project_list_exist.Area = util.ObjToString(tmp["regionName"])
+		project_list_exist.City = util.ObjToString(tmp["cityName"])
+		project_list_exist.District = util.ObjToString(tmp["countyName"])
+		project_list_exist.IssueTerm = util.ObjToString(tmp["issueTerm"])
+		project_list_exist.IssueDate = util.ObjToString(tmp["issueDate"])
+		project_list_exist.IssueInterestRate = util.ObjToString(tmp["issueInterestRate"])
+		project_list_exist.TotalInvestment = util.Float64All(tmp["totalInvestment"])
+		project_list_exist.PresentIssueAmount = util.Float64All(tmp["presentIssueAmount"])
+		err = MysqlDB.Save(&project_list_exist).Error
+		if err != nil {
+			log.Info("dealProjectList Save (Update) ", zap.Error(err))
+		}
+	} else {
+		project_list := ProjectListInfo{
+			ProjectName:        projectName,
+			Area:               util.ObjToString(tmp["regionName"]),
+			City:               util.ObjToString(tmp["cityName"]),
+			District:           util.ObjToString(tmp["countyName"]),
+			TotalInvestment:    util.Float64All(tmp["totalInvestment"]),
+			PresentIssueAmount: util.Float64All(tmp["presentIssueAmount"]),
+			IssueTerm:          util.ObjToString(tmp["issueTerm"]),
+			IssueDate:          util.ObjToString(tmp["issueDate"]),
+			IssueInterestRate:  util.ObjToString(tmp["issueInterestRate"]),
+		}
+
+		err = MysqlDB.Create(&project_list).Error
+		if err != nil {
+			log.Info("dealProjectList；dealProject Create ", zap.Error(err), zap.String("project", projectName))
+		}
+	}
+}
+
+// dealProjectList 处理项目列表存量数据
+func dealProjectListAll() {
+	tables := strings.Split(GF.Mongob.List, ",")
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	for _, table := range tables {
+		query := sess.DB("py_theme").C(table).Find(nil).Select(nil).Iter()
+		count := 0
+		for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+			if count%100 == 0 {
+				log.Info("current:", zap.Int("count", count), zap.Any("projectName", tmp["projectName"]), zap.String("当前数据表：", table))
+			}
+
+			projectName := util.ObjToString(tmp["projectName"])
+			project_list_exist := ProjectListInfo{}
+			err := MysqlDB.Where(&ProjectListInfo{ProjectName: projectName}).First(&project_list_exist).Error
+			if err != nil && err != gorm.ErrRecordNotFound {
+				// 处理查询错误
+				log.Error("dealProjectList；Error checking for existing project", zap.Error(err))
+			}
+
+			if project_list_exist.ID > 0 {
+				project_list_exist.Area = util.ObjToString(tmp["regionName"])
+				project_list_exist.City = util.ObjToString(tmp["cityName"])
+				project_list_exist.District = util.ObjToString(tmp["countyName"])
+				project_list_exist.IssueTerm = util.ObjToString(tmp["issueTerm"])
+				project_list_exist.IssueDate = util.ObjToString(tmp["issueDate"])
+				project_list_exist.IssueInterestRate = util.ObjToString(tmp["issueInterestRate"])
+				project_list_exist.TotalInvestment = util.Float64All(tmp["totalInvestment"])
+				project_list_exist.PresentIssueAmount = util.Float64All(tmp["presentIssueAmount"])
+				err = MysqlDB.Save(&project_list_exist).Error
+				if err != nil {
+					log.Info("dealProjectList Save (Update) ", zap.Error(err))
+				}
+			} else {
+				project_list := ProjectListInfo{
+					ProjectName:        projectName,
+					Area:               util.ObjToString(tmp["regionName"]),
+					City:               util.ObjToString(tmp["cityName"]),
+					District:           util.ObjToString(tmp["countyName"]),
+					TotalInvestment:    util.Float64All(tmp["totalInvestment"]),
+					PresentIssueAmount: util.Float64All(tmp["presentIssueAmount"]),
+					IssueTerm:          util.ObjToString(tmp["issueTerm"]),
+					IssueDate:          util.ObjToString(tmp["issueDate"]),
+					IssueInterestRate:  util.ObjToString(tmp["issueInterestRate"]),
+				}
+
+				err = MysqlDB.Create(&project_list).Error
+				if err != nil {
+					log.Info("dealProjectList；dealProject Create ", zap.Error(err), zap.String("project", projectName))
+				}
+			}
+		}
+	}
+
+	log.Info("dealProjectList", zap.String("数据处理完毕", "!!!!!"))
+}