法人库tidb 企业历史数据处理

ent_info/config.toml

@@ -53,7 +53,7 @@
     db = "information"
 
 [mysql]  ## mysql
-    host = "192.168.3.14:4000"
+    host = "172.20.45.129:4000"
     username = "root"
     password = "=PDT49#80Z!RVv52_z"
     db = "global_common_data"

ent_info/go.sum

@@ -122,8 +122,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
-github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=

ent_info/increment.go

@@ -31,7 +31,7 @@ func dealIncQyData() {
 
 	//1、处理qyxy_std 凭安增量数据
 	count := 0
-	it := sess.DB(GF.Mongo.DB).C(GF.Mongo.Coll).Find(where).Limit(100).Select(nil).Iter()
+	it := sess.DB(GF.Mongo.DB).C(GF.Mongo.Coll).Find(where).Select(nil).Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
 		if count%1000 == 0 {
 			log.Info("dealIncQyData", zap.Int("current:", count), zap.Any("company_name", tmp["company_name"]))

escheck/go.sum

@@ -1,5 +1,3 @@
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230510082025-6d175201278b h1:LuxfO97Gi4BzZlhNcZC9S3tk+5Z/0IMXpiyHMY4RcGk=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230510082025-6d175201278b/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=

escheck/main.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	es "github.com/olivere/elastic/v7"
 	"github.com/robfig/cron/v3"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
+	"io"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
@@ -124,11 +126,11 @@ func (t *T) task() {
 	//竞品网站总量
 	count1Query := es.NewBoolQuery().Must(rangeQuery).Filter(termsQuery)
 	//元博网（采购与招标网）,竞品网站数量统计
-	countComptetQuery1 := es.NewBoolQuery().Must(rangeQuery).Filter(es.NewTermQuery("site", "元博网（采购与招标网）"))
-	countComptetQuery2 := es.NewBoolQuery().Must(rangeQuery).Filter(es.NewTermQuery("site", "中国招标与采购网"))
-	countComptetQuery3 := es.NewBoolQuery().Must(rangeQuery).Filter(es.NewTermQuery("site", "北京隆道网络科技有限公司"))
-	countComptetQuery4 := es.NewBoolQuery().Must(rangeQuery).Filter(es.NewTermQuery("site", "友云采"))
-	countComptetQuery5 := es.NewBoolQuery().Must(rangeQuery).Filter(es.NewTermQuery("site", "标800"))
+	//countComptetQuery1 := es.NewBoolQuery().Must(rangeQuery).Filter(es.NewTermQuery("site", "元博网（采购与招标网）"))
+	//countComptetQuery2 := es.NewBoolQuery().Must(rangeQuery).Filter(es.NewTermQuery("site", "中国招标与采购网"))
+	//countComptetQuery3 := es.NewBoolQuery().Must(rangeQuery).Filter(es.NewTermQuery("site", "北京隆道网络科技有限公司"))
+	//countComptetQuery4 := es.NewBoolQuery().Must(rangeQuery).Filter(es.NewTermQuery("site", "友云采"))
+	//countComptetQuery5 := es.NewBoolQuery().Must(rangeQuery).Filter(es.NewTermQuery("site", "标800"))
 
 	es2 := elastic.Elastic{S_esurl: esAddr2, I_size: 2, Username: username, Password: password}
 	es2.InitElasticSize()
@@ -138,18 +140,19 @@ func (t *T) task() {
 
 	count := int(es2.Count(esIndex2, countQuery))  //公司 es集群 数量统计
 	count1 := int(es2.Count(esIndex, count1Query)) //竞品网站 数量统计
-	//元博网（采购与招标网）
-	countComptet1 := int(es2.Count(esIndex, countComptetQuery1))
-	//中国招标与采购网
-	countComptet2 := int(es2.Count(esIndex, countComptetQuery2))
-	//北京隆道网络科技有限公司
-	countComptet3 := int(es2.Count(esIndex, countComptetQuery3))
-	//友云采
-	countComptet4 := int(es2.Count(esIndex, countComptetQuery4))
-	//标800
-	countComptet5 := int(es2.Count(esIndex, countComptetQuery5))
-
-	countNew := int(es3.Count(esIndex3, countQuery)) //华为云 新集群
+	////元博网（采购与招标网）
+	//countComptet1 := int(es2.Count(esIndex, countComptetQuery1))
+	////中国招标与采购网
+	//countComptet2 := int(es2.Count(esIndex, countComptetQuery2))
+	////北京隆道网络科技有限公司
+	//countComptet3 := int(es2.Count(esIndex, countComptetQuery3))
+	////友云采
+	//countComptet4 := int(es2.Count(esIndex, countComptetQuery4))
+	////标800
+	//countComptet5 := int(es2.Count(esIndex, countComptetQuery5))
+
+	countNew := int(es3.Count(esIndex3, countQuery))          //华为云 新集群
+	countNewTmp := int(es3.Count("bidding_temp", countQuery)) //华为云 新集群
 	log.Println("count", count)
 	log.Println("count1", count1)
 	log.Println("countNew", countNew)
@@ -369,13 +372,14 @@ func (t *T) task() {
 			reportBuilder.WriteString("|--------|--------|---------|------|--------------|\n")
 
 			addMarkdownRow(&reportBuilder, "阿里云es集群", count, count3, count3-count, count2)
-			addMarkdownRow(&reportBuilder, "竟品统计结果", count1, count5, count5-count1, count4)
-			addMarkdownRow(&reportBuilder, "元博网", countComptet1, competeReal1, competeReal1-countComptet1, competCount1)
-			addMarkdownRow(&reportBuilder, "中国招标与采购网", countComptet2, competeReal2, competeReal2-countComptet2, competCount2)
-			addMarkdownRow(&reportBuilder, "北京隆道网络科技有限公司", countComptet3, competeReal3, competeReal3-countComptet3, competCount3)
-			addMarkdownRow(&reportBuilder, "友云采", countComptet4, competeReal4, competeReal4-countComptet4, competCount4)
-			addMarkdownRow(&reportBuilder, "标800", countComptet5, competeReal5, competeReal5-countComptet5, competCount5)
+			//addMarkdownRow(&reportBuilder, "竟品统计结果", count1, count5, count5-count1, count4)
+			//addMarkdownRow(&reportBuilder, "元博网", countComptet1, competeReal1, competeReal1-countComptet1, competCount1)
+			//addMarkdownRow(&reportBuilder, "中国招标与采购网", countComptet2, competeReal2, competeReal2-countComptet2, competCount2)
+			//addMarkdownRow(&reportBuilder, "北京隆道网络科技有限公司", countComptet3, competeReal3, competeReal3-countComptet3, competCount3)
+			//addMarkdownRow(&reportBuilder, "友云采", countComptet4, competeReal4, competeReal4-countComptet4, competCount4)
+			//addMarkdownRow(&reportBuilder, "标800", countComptet5, competeReal5, competeReal5-countComptet5, competCount5)
 			addMarkdownRow(&reportBuilder, "华为云es集群", countNew, count3, count3-countNew, count2)
+			addMarkdownRow(&reportBuilder, "华为云es集群 bidding_temp", countNewTmp, count3, count3-countNewTmp, count2)
 
 			report = reportBuilder.String()
 
@@ -423,6 +427,121 @@ func (t *T) task() {
 			SendBot(wxurl, report)
 		}
 
+		//9008 bidding 数据和 9005 bidding 数据量不一致
+		if countNew != count {
+			client1, _ := es.NewClient(
+				es.SetURL(esAddr2),
+				es.SetBasicAuth(username, password),
+				es.SetSniff(false),
+			)
+			client2, _ := es.NewClient(
+				es.SetURL(esAddr3),
+				es.SetBasicAuth(username3, password3),
+				es.SetSniff(false),
+			)
+			// 9008 比 9005 的索引数据量多
+			if count > countNew {
+				ids1, err := fetchIDs(client1, "bidding", rangeQuery)
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("集群 9008 bidding 总数: %d\n", len(ids1))
+				ids2, err := fetchIDs(client2, "bidding", rangeQuery)
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("集群9005 bidding 总数: %d\n", len(ids2))
+
+				fmt.Println("差集 (集群9008 - 集群9005):")
+				diffIDS := make([]string, 0)
+				for id := range ids1 {
+					if _, ok := ids2[id]; !ok {
+						fmt.Println(id)
+						diffIDS = append(diffIDS, id)
+					}
+				}
+				text := fmt.Sprintf("集群9008 - 集群9005 bidding 的差集数据是：\n %v", diffIDS)
+				SendBot(wxurl, text)
+			} else {
+				ids1, err := fetchIDs(client1, "bidding", rangeQuery)
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("集群 9008 bidding 总数: %d\n", len(ids1))
+				ids2, err := fetchIDs(client2, "bidding", rangeQuery)
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("集群9005 bidding 总数: %d\n", len(ids2))
+				fmt.Println("差集 (集群9005 - 集群9008):")
+				diffIDS := make([]string, 0)
+				for id := range ids2 {
+					if _, ok := ids1[id]; !ok {
+						fmt.Println(id)
+					}
+				}
+				text := fmt.Sprintf("集群9005 - 集群9008 bidding 的差集数据是：\n %v", diffIDS)
+				SendBot(wxurl, text)
+			}
+		}
+
+		if countNewTmp != count {
+			client1, _ := es.NewClient(
+				es.SetURL(esAddr2),
+				es.SetBasicAuth(username, password),
+				es.SetSniff(false),
+			)
+			client2, _ := es.NewClient(
+				es.SetURL(esAddr3),
+				es.SetBasicAuth(username3, password3),
+				es.SetSniff(false),
+			)
+
+			if count > countNewTmp {
+				ids1, err := fetchIDs(client1, "bidding", rangeQuery)
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("集群 9008 bidding 总数: %d\n", len(ids1))
+				ids2, err := fetchIDs(client2, "bidding_temp", rangeQuery)
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("集群9005 bidding_temp 总数: %d\n", len(ids2))
+
+				fmt.Println("差集 (集群9008 bidding - 集群9005 bidding_temp ):")
+				diffIDS := make([]string, 0)
+				for id := range ids1 {
+					if _, ok := ids2[id]; !ok {
+						fmt.Println(id)
+						diffIDS = append(diffIDS, id)
+					}
+				}
+				text := fmt.Sprintf("集群9008 bidding - 集群9005 bidding_temp 的差集数据是：\n %v", diffIDS)
+				SendBot(wxurl, text)
+			} else {
+				ids1, err := fetchIDs(client1, "bidding", rangeQuery)
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("集群 9008 bidding 总数: %d\n", len(ids1))
+				ids2, err := fetchIDs(client2, "bidding_temp", rangeQuery)
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("集群9005 bidding_temp 总数: %d\n", len(ids2))
+				fmt.Println("差集 (集群9005 biding_temp - 集群9008 bidding ):")
+				diffIDS := make([]string, 0)
+				for id := range ids2 {
+					if _, ok := ids1[id]; !ok {
+						fmt.Println(id)
+					}
+				}
+				text := fmt.Sprintf("集群9005 biding_temp - 集群9008 bidding的差集数据是：\n %v", diffIDS)
+				SendBot(wxurl, text)
+			}
+		}
+
 	}
 	log.Println("task over:", t.Name, eq, count)
 }
@@ -475,3 +594,63 @@ func SendBot(webhookURL, msg string) (b bool) {
 	log.Println("send bot Status:", resp.Status)
 	return
 }
+
+func fetchIDs(client *es.Client, index string, query es.Query) (map[string]struct{}, error) {
+	ctx := context.Background()
+	ids := make(map[string]struct{})
+
+	scrollID := ""
+	scroll := "10m"
+
+	searchSource := es.NewSearchSource().
+		Query(query).
+		Size(10000).
+		Sort("_doc", true)
+
+	searchService := client.Scroll(index).
+		Size(10000).
+		Scroll(scroll).
+		SearchSource(searchSource)
+
+	res, err := searchService.Do(ctx)
+	if err != nil {
+		if err == io.EOF {
+			fmt.Println("没有数据")
+			return ids, nil
+		}
+		return nil, fmt.Errorf("初始化滚动搜索失败: %w", err)
+	}
+
+	fmt.Println("命中总数：", res.TotalHits())
+
+	total := 0
+	for len(res.Hits.Hits) > 0 {
+		for _, hit := range res.Hits.Hits {
+			// 直接收集 _id
+			ids[hit.Id] = struct{}{}
+		}
+
+		total += len(res.Hits.Hits)
+		scrollID = res.ScrollId
+
+		res, err = client.Scroll().
+			ScrollId(scrollID).
+			Scroll(scroll).
+			Do(ctx)
+		//log.Println("已获取数量:", total)
+
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return ids, fmt.Errorf("滚动搜索失败: %w", err)
+		}
+	}
+
+	_, err = client.ClearScroll().ScrollId(scrollID).Do(ctx)
+	if err != nil {
+		log.Printf("清理滚动搜索失败：%s", err)
+	}
+
+	return ids, nil
+}

faren_tidb/all.go

@@ -0,0 +1,263 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/RoaringBitmap/roaring"
+	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	jlog "jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"strings"
+	"time"
+)
+
+// dealAllData 处理存量数据
+func dealAllData() {
+	jlog.Info("dealAllData", zap.String("开始处理法人库存量数据", "-------------"))
+	ctx := context.Background()
+	// 批量处理数据
+	batchSize := 100 // 每批处理的数据量
+	offset := 0
+	count := 0
+
+	for {
+		query := fmt.Sprintf(`
+			SELECT company_id,company_name,company_code,credit_no,org_tags,bitmapToArray(company_label)  labels,org_code, tax_code,establish_date,
+			       
+			       FROM ent_info  LIMIT %d OFFSET %d
+		`, batchSize, offset)
+
+		rows, err := ClickHouseConn.Query(ctx, query)
+		if err != nil {
+			jlog.Info("allUpdateBitmap2", zap.Error(err))
+		}
+
+		if !rows.Next() {
+			break
+		}
+
+		for rows.Next() {
+			count++
+			var ent EntInfo
+			//var bitmapVals []uint64
+			var company_id, companyName, creditNo, orgTags string
+
+			var oldLabels = make([]uint64, 0)
+			if err = rows.Scan(&company_id, &companyName, &creditNo, &orgTags, &oldLabels); err != nil {
+				jlog.Info("dealIncEntInfo", zap.Error(err))
+			}
+
+			if count%10000 == 0 {
+				jlog.Info("allUpdateBitmap2:", zap.Int("current", offset), zap.String("name", ent.CompanyName))
+			}
+
+		}
+
+		offset += batchSize
+	}
+
+	jlog.Info("dealAllData", zap.Int("存量数据迭代完毕", offset))
+}
+
+// dealAllFromCompanyBase 从company_base 处理惬意数据存量
+func dealAllFromCompanyBase() {
+	jlog.Info("dealAllFromCompanyBase", zap.String("开始处理", "-------企业库存量数据"))
+	defer util.Catch()
+	sess := MgoQY.GetMgoConn()
+	defer MgoQY.DestoryMongoConn(sess)
+
+	where := map[string]interface{}{
+		"company_type": map[string]interface{}{
+			"$ne": "个体工商户",
+		},
+	}
+
+	count := 0
+	batchSize := 100
+	ents := make([]EntInfo, 0, batchSize)
+	it := sess.DB(GF.MongoQy.DB).C("company_base").Find(where).Select(nil).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			jlog.Info("dealAllFromCompanyBase", zap.Any("current:", count), zap.Any("company_name", tmp["company_name"]))
+		}
+
+		company_status := util.ObjToString(tmp["company_status"])
+		if strings.Contains(company_status, "注销") || strings.Contains(company_status, "吊销") {
+			continue
+		}
+		if util.IntAll(tmp["use_flag"]) > 0 {
+			continue
+		}
+
+		var ent EntInfo
+		ent.CompanyID = util.ObjToString(tmp["company_id"])
+		ent.CompanyName = util.ObjToString(tmp["company_name"])
+		ent.CompanyCode = util.ObjToString(tmp["company_code"])
+		ent.CreditNo = util.ObjToString(tmp["credit_no"])
+		ent.OrgCode = util.ObjToString(tmp["org_code"])
+		ent.TaxCode = util.ObjToString(tmp["tax_code"])
+		ent.EstablishDate = util.ObjToString(tmp["establish_date"])
+		ent.LegalPerson = util.ObjToString(tmp["legal_person"])
+		ent.LegalPersonCaption = util.ObjToString(tmp["legal_person_caption"])
+		ent.CompanyStatus = util.ObjToString(tmp["company_status"])
+		ent.CompanyType = util.ObjToString(tmp["company_type"])
+		ent.Authority = util.ObjToString(tmp["authority"])
+		ent.IssueDate = util.ObjToString(tmp["issue_date"])
+		ent.OperationStartDate = util.ObjToString(tmp["operation_startdate"])
+		ent.OperationEndDate = util.ObjToString(tmp["operation_enddate"])
+		ent.Capital = util.ObjToString(tmp["capital"])
+		ent.CompanyAddress = util.ObjToString(tmp["company_address"])
+		ent.BusinessScope = util.ObjToString(tmp["business_scope"])
+		ent.ComeInTime = time.Now().Unix()
+		ent.UpdateTime = time.Now().Unix()
+		ent.LegalPersonType = int8(util.IntAll(tmp["legal_person_type"]))
+		ent.RealCapital = util.ObjToString(tmp["real_capital"])
+		ent.EnName = util.ObjToString(tmp["en_name"])
+		ent.ListCode = util.ObjToString(tmp["list_code"])
+
+		//annual_reports
+		std := getQyxyStd(util.ObjToString(tmp["company_name"]))
+		if std != nil && len(std) > 0 {
+			// 取出 annual_reports 字段
+			reports, ok := std["annual_reports"].([]interface{})
+			if ok {
+				var maxYear float64
+				var employeeNo string
+				// 遍历 annual_reports 数组
+				for i, r := range reports {
+					if reportMap, ok := r.(map[string]interface{}); ok {
+						year := util.Float64All(reportMap["report_year"])
+						emp := util.ObjToString(reportMap["employee_no"])
+						if i == 0 || year > maxYear {
+							maxYear = year
+							employeeNo = emp
+						}
+					}
+				}
+				if maxYear > 0 {
+					ent.EmployeeNo = util.IntAll(employeeNo)
+				}
+			}
+		}
+		//
+		ent.Website = util.ObjToString(tmp["website_url"])
+		ent.CompanyPhone = util.ObjToString(tmp["company_phone"])
+		ent.CompanyEmail = util.ObjToString(tmp["company_email"])
+		//company_industry_tags
+		whereIndustry := map[string]interface{}{
+			"company_id": util.ObjToString(tmp["company_id"]),
+		}
+		indus, _ := MgoQY.FindOne("company_industry", whereIndustry)
+		ent.CompanyIndustryTags = "{}" // 先给个默认值
+		if indus != nil && len(*indus) > 0 {
+			name_path := make([]string, 0)
+			name_code := make([]string, 0)
+			name_path = append(name_path, util.ObjToString((*indus)["industry_l1_name"]))
+			name_path = append(name_path, util.ObjToString((*indus)["industry_l2_name"]))
+			name_path = append(name_path, util.ObjToString((*indus)["industry_l3_name"]))
+			name_path = append(name_path, util.ObjToString((*indus)["industry_l4_name"]))
+			//
+			name_code = append(name_code, util.ObjToString((*indus)["industry_l1_code"]))
+			name_code = append(name_code, util.ObjToString((*indus)["industry_l2_code"]))
+			name_code = append(name_code, util.ObjToString((*indus)["industry_l3_code"]))
+			name_code = append(name_code, util.ObjToString((*indus)["industry_l4_code"]))
+			industry := map[string]interface{}{
+				"name_path": name_path,
+				"code_path": name_code,
+			}
+			// map 转 JSON
+			jsonBytes, _ := json.Marshal(industry)
+			ent.CompanyIndustryTags = string(jsonBytes)
+		}
+		//
+		area, city, district := util.ObjToString((std)["company_area"]), util.ObjToString((std)["company_city"]), util.ObjToString((std)["company_district"])
+		area_code, city_code, district_code := CalculateRegionCode(area, city, district)
+		ent.JYAreaCode = area_code
+		ent.JYCityCode = city_code
+		ent.JYDistrictCode = district_code
+		//
+		query := `
+	SELECT bitmapToArray(company_label)
+	FROM ent_info
+	WHERE company_id = ?
+`
+
+		var oldLabels = make([]uint64, 0)
+		row := ClickHouseConn.QueryRow(context.Background(), query, ent.CompanyID)
+		err := row.Scan(&oldLabels)
+		if err != nil {
+			if errors.Is(err, sql.ErrNoRows) {
+				//jlog.Info("dealIncEntInfo: 没查到数据", zap.String("company_id", ent.CompanyID))
+			} else {
+				jlog.Info("dealIncEntInfo: 查询出错", zap.Error(err))
+			}
+		}
+
+		// 转 RoaringBitmap
+		rbm := roaring.NewBitmap()
+		for _, v := range oldLabels {
+			rbm.Add(uint32(v))
+		}
+		bin, _ := rbm.ToBytes()
+		ent.JYCompanyLabel = bin
+		ent.JYOrgTopType = "企业"
+		company_type := util.ObjToString(tmp["company_type"])
+		if info, ok := nameNorm[company_type]; ok {
+			ent.JYCompanyTypeOriginCode = info.Code
+			ent.JYCompanyTypeIsLeaf = 1
+			ent.JYCompanyTypeLeafCode = info.Code
+			ent.JYCompanyTypeLeafName = info.Name
+			ent.JYCompanyTypeLeafTag = info.Tag
+			ent.JYOrgPropertyOneTag = "工商"
+			ent.JYOrgPropertyTwoTag = "企业"
+		}
+		//保存tidb
+		//if err := MysqlDB.Create(&ent).Error; err != nil {
+		//	jlog.Info("insert failed: %v", zap.Error(err))
+		//}
+		ents = append(ents, ent)
+
+		if len(ents) >= batchSize {
+			if err := MysqlDB.CreateInBatches(ents, batchSize).Error; err != nil {
+				jlog.Error("批量插入失败", zap.Error(err))
+			}
+			ents = ents[:0] // 清空 slice
+		}
+	}
+	// 循环结束后如果还有数据
+	if len(ents) > 0 {
+		if err := MysqlDB.CreateInBatches(ents, batchSize).Error; err != nil {
+			jlog.Error("批量插入失败", zap.Error(err))
+		}
+	}
+}
+
+// get 通过companyID 获取法人库数据
+func get() {
+	// 2. 查询一条数据
+	var ent EntInfo
+	if err := MysqlDB.Where("company_id = ?", "001c2e9882ae982abf6e1e9ed06e2654").First(&ent).Error; err != nil {
+		panic(err)
+	}
+
+	// 3. 反序列化 RoaringBitmap
+	rbm := roaring.NewBitmap()
+	if len(ent.JYCompanyLabel) > 0 {
+		if err := rbm.UnmarshalBinary(ent.JYCompanyLabel); err != nil {
+			panic(err)
+		}
+	}
+
+	// 4. 转成 []uint64
+	ids := make([]uint64, 0, rbm.GetCardinality())
+	it := rbm.Iterator()
+	for it.HasNext() {
+		ids = append(ids, uint64(it.Next()))
+	}
+
+	fmt.Println("CompanyID:", ent.CompanyID)
+	fmt.Println("标签ID集合:", ids)
+}

faren_tidb/config.go

@@ -0,0 +1,76 @@
+package main
+
+type GlobalConf struct {
+	Mongo      MgoConf
+	MongoQy    MgoConf
+	Env        EnvConf
+	Esa        EsConf
+	Esb        EsConf
+	Clickhouse CkConf
+	Log        Log
+	Mysql      MysqlConf
+}
+
+type CkConf struct {
+	Host     string
+	Username string
+	Password string
+	DB       string
+}
+
+type MgoConf struct {
+	Host     string
+	DB       string
+	Coll     string // 查询表
+	Username string
+	Password string
+	Size     int
+	Direct   bool
+}
+
+type EnvConf struct {
+	PortraitIndex string
+	PortraitMgo   string
+	Start         int
+	End           int
+	Spec          string
+	Spec2         string
+	Isw           bool   //是否保存标签里的权重
+	Esindex       string //ent_info
+	Pre           string //数据表前缀
+}
+
+type EsConf struct {
+	URL      string
+	Username string
+	Password string
+	Index    string
+}
+
+// LabelData  标签配置
+type LabelData struct {
+	Name        string        // 标签名称
+	Field       []string      //识别字段
+	Sfield      string        //保存字段
+	Rule        []string      // 具体规则
+	RegRule     []interface{} //规则的DFA
+	Weight      []float64     // 权重
+	TotalWeight float64       // 最终合并后的权重
+}
+
+type Log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type MysqlConf struct {
+	Host     string
+	Username string
+	Password string
+	DB       string
+}

faren_tidb/config.toml

@@ -0,0 +1,78 @@
+[mongo]  ## 标讯地址
+    host = "127.0.0.1:27083"
+    #        host = "172.17.189.140:27080"
+    db = "mixdata"
+    coll = "qyxy_std"
+    username = "SJZY_RWbid_ES"
+    password = "SJZY@B4i4D5e6S"
+    direct = true  ## 本地代理时需要打开，
+
+## 测试环境
+#    host = "192.168.3.149:27102"
+#    db = "qfw_data"
+#    coll = "wcc_special_gov_unit"
+##    coll = "wcc_special_enterprise"
+#    username = "root"
+#    password = "root"
+
+
+[mongoqy]  ## 181 凭安数据
+#    host = "172.17.4.181:27001"
+    host = "127.0.0.1:27001"
+    db = "mixdata"
+    username = ""
+    password = ""
+    direct = true  ## 本地代理时需要打开，
+
+aaaaaaaaaaaaq
+
+
+[clickhouse] ## clickhouse 数据库
+#        host = "cc-2ze9tv451wov14w9e.clickhouse.ads.aliyuncs.com:9000"
+        host = "localhost:18129"
+        username = "biservice"
+        password = "Bi_top95215#"
+        db = "information"
+    ### 测试环境
+#    host = "172.20.45.129:18123"
+#    username = "jytop"
+#    password = "pwdTopJy123"
+#    db = "information"
+
+[mysql]  ## mysql
+#    host = "172.20.45.129:4000"
+#    host = "127.0.0.1:4001"
+#    username = "root"
+#    password = "=PDT49#80Z!RVv52_z"
+#    db = "global_common_data"
+
+    host = "172.20.45.129:4000"
+    username = "root"
+    password = "=PDT49#80Z!RVv52_z"
+    db = "global_common_data"
+
+[env]
+    start = -2 ## 开始取2天前的数据，
+    end = -1 ## 截止取1天前的数据，
+    spec = "0 50 23 * * *"  ## 定时任务，每天23点50执行;更新法人库Bitmap
+    spec2 = "0 0 18 * * 6"  ## 定时任务，没周六18点执行，读取更新凭安数据，
+#    pre = "dws_f_" ## 数据表 前缀
+
+
+
+[log]
+# 日志路径，为空将输出控制台
+    logpath = ""
+#logpath = "logs/log.out"
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "json"

faren_tidb/entity.go

@@ -0,0 +1,78 @@
+package main
+
+// EntInfo TiDB 结构体，匹配所有字段
+type EntInfo struct {
+	ID                      uint64 `gorm:"primaryKey;autoIncrement" json:"id"`
+	CompanyID               string `gorm:"column:company_id" json:"company_id"`
+	CompanyName             string `gorm:"column:company_name" json:"company_name"`
+	CompanyCode             string `gorm:"column:company_code" json:"company_code"`
+	CreditNo                string `gorm:"column:credit_no" json:"credit_no"`
+	OrgCode                 string `gorm:"column:org_code" json:"org_code"`
+	TaxCode                 string `gorm:"column:tax_code" json:"tax_code"`
+	EstablishDate           string `gorm:"column:establish_date" json:"establish_date"`
+	LegalPerson             string `gorm:"column:legal_person" json:"legal_person"`
+	LegalPersonCaption      string `gorm:"column:legal_person_caption" json:"legal_person_caption"`
+	CompanyStatus           string `gorm:"column:company_status" json:"company_status"`
+	CompanyType             string `gorm:"column:company_type" json:"company_type"`
+	Authority               string `gorm:"column:authority" json:"authority"`
+	IssueDate               string `gorm:"column:issue_date" json:"issue_date"`
+	OperationStartDate      string `gorm:"column:operation_startdate" json:"operation_startdate"`
+	OperationEndDate        string `gorm:"column:operation_enddate" json:"operation_enddate"`
+	Capital                 string `gorm:"column:capital" json:"capital"`
+	CompanyAddress          string `gorm:"column:company_address" json:"company_address"`
+	BusinessScope           string `gorm:"column:business_scope" json:"business_scope"`
+	ComeInTime              int64  `gorm:"column:comeintime" json:"comeintime"`
+	UpdateTime              int64  `gorm:"column:updatetime" json:"updatetime"`
+	LegalPersonType         int8   `gorm:"column:legal_person_type" json:"legal_person_type"`
+	RealCapital             string `gorm:"column:real_capital" json:"real_capital"`
+	EnName                  string `gorm:"column:en_name" json:"en_name"`
+	ListCode                string `gorm:"column:list_code" json:"list_code"`
+	EmployeeNo              int    `gorm:"column:employee_no" json:"employee_no"`
+	Website                 string `gorm:"column:website" json:"website"`
+	CompanyPhone            string `gorm:"column:company_phone" json:"company_phone"`
+	CompanyEmail            string `gorm:"column:company_email" json:"company_email"`
+	CompanyIndustryTags     string `gorm:"column:company_industry_tags" json:"company_industry_tags"`
+	JYLongitudeLatitude     string `gorm:"column:jy_longitude_latitude" json:"jy_longitude_latitude"`
+	JYAreaCode              string `gorm:"column:jy_area_code" json:"jy_area_code"`
+	JYCityCode              string `gorm:"column:jy_city_code" json:"jy_city_code"`
+	JYDistrictCode          string `gorm:"column:jy_district_code" json:"jy_district_code"`
+	JYAdminLevel            string `gorm:"column:jy_admin_level" json:"jy_admin_level"`
+	JYCompanyLabel          []byte `gorm:"column:jy_company_label" json:"-"` // 存储RoaringBitmap序列化结果
+	JYOrgTopType            string `gorm:"column:jy_org_toptype" json:"jy_org_toptype"`
+	JYOrgSubType            string `gorm:"column:jy_org_subtype" json:"jy_org_subtype"`
+	JYCompanyTypeOriginCode string `gorm:"column:jy_company_type_origin_code" json:"jy_company_type_origin_code"`
+	JYCompanyTypeIsLeaf     int8   `gorm:"column:jy_company_type_is_leaf" json:"jy_company_type_is_leaf"`
+	JYCompanyTypeLeafCode   string `gorm:"column:jy_company_type_leaf_code" json:"jy_company_type_leaf_code"`
+	JYCompanyTypeLeafName   string `gorm:"column:jy_company_type_leaf_name" json:"jy_company_type_leaf_name"`
+	JYCompanyTypeLeafTag    string `gorm:"column:jy_company_type_leaf_tag" json:"jy_company_type_leaf_tag"`
+	JYOrgPropertyOneTag     string `gorm:"column:jy_org_property_one_tag" json:"jy_org_property_one_tag"`
+	JYOrgPropertyTwoTag     string `gorm:"column:jy_org_property_two_tag" json:"jy_org_property_two_tag"`
+	JYOrgPropertyThreeTag   string `gorm:"column:jy_org_property_three_tag" json:"jy_org_property_three_tag"`
+}
+
+func (EntInfo) TableName() string {
+	if GF.Env.Pre != "" {
+		return GF.Env.Pre + "ent_info"
+	} else {
+		return "ent_info"
+	}
+}
+
+type EntInfoNorm struct {
+	ID      uint   `gorm:"primaryKey;autoIncrement;column:id" json:"id"`
+	Level   int    `gorm:"column:level" json:"level"`       // 层级
+	Code    string `gorm:"column:code" json:"code"`         // 代码
+	Name    string `gorm:"column:name" json:"name"`         // 类型名称(叶子节点类型)
+	AllName string `gorm:"column:all_name" json:"all_name"` // 类型全称
+	Tag     string `gorm:"column:tag" json:"tag"`           // 内外个私农合
+}
+
+// TableName 自定义表名
+func (EntInfoNorm) TableName() string {
+	if GF.Env.Pre != "" {
+		return GF.Env.Pre + "ent_info_norm"
+	} else {
+		return "ent_info_norm"
+	}
+
+}

faren_tidb/go.mod

@@ -0,0 +1,69 @@
+module faren_tidb
+
+go 1.23.10
+
+require (
+	github.com/ClickHouse/clickhouse-go/v2 v2.40.1
+	github.com/RoaringBitmap/roaring v1.9.1
+	github.com/spf13/viper v1.20.1
+	github.com/xuri/excelize/v2 v2.9.1
+	go.uber.org/zap v1.27.0
+	gorm.io/driver/mysql v1.6.0
+	gorm.io/gorm v1.30.1
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40
+)
+
+require (
+	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/ClickHouse/ch-go v0.67.0 // indirect
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/bits-and-blooms/bitset v1.12.0 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/go-sql-driver/mysql v1.8.1 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/mschoch/smat v0.2.0 // indirect
+	github.com/paulmach/orb v0.11.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/richardlehane/mscfb v1.0.4 // indirect
+	github.com/richardlehane/msoleps v1.0.4 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tiendc/go-deepcopy v1.6.0 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/xuri/efp v0.0.1 // indirect
+	github.com/xuri/nfp v0.0.1 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.11.4 // indirect
+	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/trace v1.37.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.40.0 // indirect
+	golang.org/x/net v0.42.0 // indirect
+	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sys v0.34.0 // indirect
+	golang.org/x/text v0.27.0 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

faren_tidb/go.sum

@@ -0,0 +1,315 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/ClickHouse/ch-go v0.67.0 h1:18MQF6vZHj+4/hTRaK7JbS/TIzn4I55wC+QzO24uiqc=
+github.com/ClickHouse/ch-go v0.67.0/go.mod h1:2MSAeyVmgt+9a2k2SQPPG1b4qbTPzdGDpf1+bcHh+18=
+github.com/ClickHouse/clickhouse-go/v2 v2.40.1 h1:PbwsHBgqXRydU7jKULD1C8CHmifczffvQqmFvltM2W4=
+github.com/ClickHouse/clickhouse-go/v2 v2.40.1/go.mod h1:GDzSBLVhladVm8V01aEB36IoBOVLLICfyeuiIp/8Ezc=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/RoaringBitmap/roaring v1.9.1 h1:LXcSqGGGMKm+KAzUyWn7ZeREqoOkoMX+KwLOK1thc4I=
+github.com/RoaringBitmap/roaring v1.9.1/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
+github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
+github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
+github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
+github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
+github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
+github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
+github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
+github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
+github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00=
+github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
+github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
+github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
+github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
+github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
+github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo=
+github.com/tiendc/go-deepcopy v1.6.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=
+github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
+github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw=
+github.com/xuri/excelize/v2 v2.9.1/go.mod h1:x7L6pKz2dvo9ejrRuD8Lnl98z4JLt0TGAwjhW+EiP8s=
+github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q=
+github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
+go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
+go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
+golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
+golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
+golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
+golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
+golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
+gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
+gorm.io/gorm v1.30.1 h1:lSHg33jJTBxs2mgJRfRZeLDG+WZaHYCk3Wtfl6Ngzo4=
+gorm.io/gorm v1.30.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40 h1:xTeRmpFgwOdu+NbWg/YntX3MnQpttm7jj33C1+JdBTk=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

faren_tidb/init.go

@@ -0,0 +1,221 @@
+package main
+
+import (
+	"fmt"
+	"github.com/xuri/excelize/v2"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"strings"
+
+	"github.com/spf13/viper"
+	"go.uber.org/zap"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"os"
+	"time"
+)
+
+func init() {
+	InitConfig()
+	InitLog()
+	InitClickhouse()
+	InitMgo()
+	InitMgoQY()
+	InitMysql()
+	InitAreaCode()
+	InitNameNorm() //读取叶子节点标准数据到内存
+
+	//InitNorm() //第一次；读取表格文件导入到tidb ；
+
+}
+
+func InitConfig() (err error) {
+	viper.SetConfigFile("config.toml") // 指定配置文件路径
+	viper.SetConfigName("config")      // 配置文件名称(无扩展名)
+	viper.SetConfigType("toml")        // 如果配置文件的名称中没有扩展名，则需要配置此项
+
+	viper.AddConfigPath("./")
+	viper.AddConfigPath("./conf/")  // 还可以在工作目录中查找配置
+	viper.AddConfigPath("../conf/") // 还可以在工作目录中查找配置
+	err = viper.ReadInConfig()      // 查找并读取配置文件
+	if err != nil {                 // 处理读取配置文件的错误
+		return
+	}
+
+	err = viper.Unmarshal(&GF)
+
+	return err
+
+}
+
+func InitLog() {
+	now := time.Now()
+	err := log.InitLog(
+		log.Path(GF.Log.LogPath),
+		log.Level(GF.Log.LogLevel),
+		log.Compress(GF.Log.Compress),
+		log.MaxSize(GF.Log.MaxSize),
+		log.MaxBackups(GF.Log.MaxBackups),
+		log.MaxAge(GF.Log.MaxAge),
+		log.Format(GF.Log.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	log.Info("InitLog", zap.Any("duration", time.Since(now).Seconds()))
+}
+
+// InitClickhouse 初始化Clickhouse
+func InitClickhouse() {
+	username := GF.Clickhouse.Username
+	password := GF.Clickhouse.Password
+	host := GF.Clickhouse.Host
+	dbname := GF.Clickhouse.DB
+
+	// 测试环境
+	//host := "192.168.3.207:19000"
+	//username := "jytop"
+	//password := "pwdTopJy123"
+	//dbname := "information"
+	var err error
+	ClickHouseConn, err = connectClickhouse(host, username, password, dbname)
+	if err != nil {
+		log.Info("InitClickhouse", zap.Error(err))
+	}
+}
+
+// InitMgo 初始化标讯,mixdata.qyxy_std
+func InitMgo() {
+	// qyxy_std 数据
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: GF.Mongo.Host,
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   GF.Mongo.DB,
+		UserName: GF.Mongo.Username,
+		Password: GF.Mongo.Password,
+		Direct:   GF.Mongo.Direct,
+	}
+	Mgo.InitPool()
+
+}
+
+func InitMgoQY() {
+	//181 凭安库
+	MgoQY = &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: GF.MongoQy.Host,
+		DbName:      GF.MongoQy.DB,
+		Size:        10,
+		UserName:    GF.MongoQy.Username,
+		Password:    GF.MongoQy.Password,
+		Direct:      GF.MongoQy.Direct,
+	}
+	MgoQY.InitPool()
+
+	log.Info("InitMgoQY", zap.String("初始化成功", GF.MongoQy.Host))
+}
+
+func InitMysql() {
+	username := GF.Mysql.Username
+	password := GF.Mysql.Password
+	host := GF.Mysql.Host // 本地
+	//host := "172.17.162.27:14000" //线上
+	database := GF.Mysql.DB
+	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", username, password, host, database)
+	// 连接到数据库
+	var err error
+	MysqlDB, err = gorm.Open(mysql.Open(dsn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Error), //不打印日志
+	})
+
+	if err != nil {
+		log.Info("InitMysql", zap.String("初始化MySQL失败", database), zap.Error(err))
+		os.Exit(1)
+		return
+	} else {
+		log.Info("InitMysql", zap.String("初始化MySQL成功", database))
+	}
+}
+
+// CodeArea 表示 code_area 数据表的结构体
+type CodeArea struct {
+	ID       uint16 `gorm:"primaryKey;autoIncrement;not null"` // id 为主键，自增
+	Code     string `gorm:"size:10;not null;unique"`           // code 列，长度 10，唯一
+	Area     string `gorm:"size:45;not null"`                  // area 列，长度 45，不允许为空
+	City     string `gorm:"size:45"`                           // city 列，长度 45，可以为空
+	District string `gorm:"size:45"`                           // district 列，长度 45，可以为空
+	Alias    string `gorm:"size:10"`                           // alias 列，长度 10，可以为空
+}
+
+// TableName 指定数据库表名
+func (CodeArea) TableName() string {
+	return "code_area"
+}
+
+// InitAreaCode 初始化地区code；
+func InitAreaCode() {
+	// 查询所有数据
+	var codeAreas []CodeArea
+	MysqlDB.Find(&codeAreas) // 查询所有记录
+	for _, ca := range codeAreas {
+		key := ca.Area + "~" + ca.City + "~" + ca.District + "~"
+		code := ca.Code
+		RegionCodeData[key] = code
+	}
+}
+
+// InitNorm 初始化叶子节点标准
+func InitNorm() {
+	f, err := excelize.OpenFile("./法人类型代码表.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		f.Save()
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("叶子节点")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for i := 1; i < len(rows); i++ {
+		var code_norm EntInfoNorm
+		code_norm.Level = util.IntAll(rows[i][0])
+		code_norm.Code = strings.TrimSpace(rows[i][2])
+		code_norm.Name = strings.TrimSpace(rows[i][3])
+		code_norm.AllName = strings.TrimSpace(rows[i][7])
+		code_norm.Tag = strings.TrimSpace(rows[i][4])
+		err = MysqlDB.Create(&code_norm).Error
+		if err != nil {
+			log.Error("InitNorm", zap.Error(err))
+		}
+	}
+
+	log.Info("InitNorm", zap.Int("叶子节点读取完毕", len(rows)))
+}
+
+func InitNameNorm() {
+	var list []EntInfoNorm
+	if err := MysqlDB.Find(&list).Error; err != nil {
+		// 处理错误
+		log.Info("InitNameNorm", zap.Error(err))
+		os.Exit(1)
+	}
+
+	for _, info := range list {
+		nameNorm[info.Name] = info
+	}
+
+	log.Info("InitNameNorm", zap.Int("nameNorm 初始化完毕", len(list)))
+}

faren_tidb/main.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"gorm.io/gorm"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+)
+
+var (
+	GF      GlobalConf
+	MysqlDB *gorm.DB
+	MgoQY   *mongodb.MongodbSim //181凭安
+	Mgo     *mongodb.MongodbSim //标讯地址-凭安库 mixdata
+	//Esa            *es.Elastic
+	//Esb            *es.Elastic
+	ClickHouseConn driver.Conn
+	entLabelMap    = make(map[uint64]string)
+	nameBitMap     = make(map[string]uint64) //和上面k-v 反过来
+	//更新es
+	updateEsPool   = make(chan []map[string]interface{}, 5000)
+	updateEsSp     = make(chan bool, 5) //保存协程
+	RegionCodeData = map[string]string{}
+
+	//
+	nameNorm = make(map[string]EntInfoNorm) //存储叶子节点标准
+)
+
+func main() {
+	dealAllFromCompanyBase() //企业法人存量数据
+
+}

faren_tidb/readme.txt

@@ -0,0 +1,20 @@
+法人库-tidb 设计实现
+
+
+数据库设计文档地址：https://www.kdocs.cn/l/cq29qcZ7LWdm
+
+
+
+从Clickhouse 数据库法人库中，迁移数据到tidb 中
+
+1、先处理存量数据
+2、处理增量数据
+
+
+
+三、时间节点
+tidb数据库表设计：2025-08-01~2025-08-06
+历史数据同步及更新：2025-08-06~2025-08-14
+第一批叶子节点处理：2025-08-14~2025-08-23
+国营、民营标签处理：2025-08-23~2025-08-30
+测试交付：2025-08-15~2025-08-30

faren_tidb/tools.go

@@ -0,0 +1,493 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	ckgo "github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"io/ioutil"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"log"
+	"os"
+	"strings"
+	"time"
+)
+
+// connectClickhouse 连接Clickhouse,其它方式连接;bitmap字段无法处理，需要使用下面方法
+func connectClickhouse(host, username, password, dbname string) (driver.Conn, error) {
+	//host := "192.168.3.207:19000"
+	//username := "jytop"
+	//password := "pwdTopJy123"
+	//dbname := "pub_tags"
+	var (
+		ctx       = context.Background()
+		conn, err = ckgo.Open(&ckgo.Options{
+			Addr:         []string{host},
+			DialTimeout:  10 * time.Second,
+			MaxIdleConns: 3,
+			MaxOpenConns: 30,
+			Auth: ckgo.Auth{
+				Database: dbname,
+				Username: username,
+				Password: password,
+			},
+			Debugf: func(format string, v ...interface{}) {
+				log.Println(format, v)
+			},
+		})
+	)
+	if err != nil {
+		return nil, err
+	}
+	if err := conn.Ping(ctx); err != nil {
+		if exception, ok := err.(*ckgo.Exception); ok {
+			log.Println("Exception [%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
+		}
+		return nil, err
+	}
+	return conn, nil
+}
+
+// 动态构建 toUInt64 数组字符串
+func buildToUInt64Array(labels []uint64) string {
+	if len(labels) == 0 {
+		return "[]"
+	}
+	toUInt64Labels := make([]string, len(labels))
+	for i, label := range labels {
+		toUInt64Labels[i] = fmt.Sprintf("toUInt64(%d)", label)
+	}
+	return fmt.Sprintf("[%s]", strings.Join(toUInt64Labels, ", "))
+}
+
+// IndustryClassification 国标行业分类的结构体
+type IndustryClassification struct {
+	Code     string                   `json:"code"`
+	EntCode  uint64                   `json:"ent_code"` // 法人库bitmap
+	Name     string                   `json:"name"`
+	Children []IndustryClassification `json:"children"`
+	Level    int                      `json:"level"` // 新增字段 Level
+}
+
+// 全局变量 seen 用于记录已经分配的 Name 和 EntCode
+var seen = make(map[string]uint64)
+
+// assignLevel 为每个分类分配层级
+func assignLevel(classifications []IndustryClassification, level int) {
+	for i := range classifications {
+		classifications[i].Level = level
+		// 递归处理子分类
+		if len(classifications[i].Children) > 0 {
+			assignLevel(classifications[i].Children, level+1)
+		}
+	}
+}
+
+// assignEntCode 为每个分类分配 EntCode
+func assignEntCode(classifications []IndustryClassification, entCode *uint64) {
+	for i := range classifications {
+		// 如果是第三级或更高层级，且只有一个子分类且名称相同，父子共享EntCode
+		if len(classifications[i].Children) == 1 && classifications[i].Level == 3 && classifications[i].Name == classifications[i].Children[0].Name {
+			classifications[i].EntCode = *entCode
+			classifications[i].Children[0].EntCode = *entCode
+			//seen[classifications[i].Name] = *entCode
+			(*entCode)++
+			continue
+		} else {
+			classifications[i].EntCode = *entCode
+			//seen[classifications[i].Name] = *entCode
+			(*entCode)++
+		}
+
+		// 如果有子分类，则递归处理
+		if len(classifications[i].Children) > 0 {
+			assignEntCode(classifications[i].Children, entCode)
+		}
+	}
+}
+
+// readIndustryClassifications 读取行业分类数据
+func readIndustryClassifications(filePath string) ([]IndustryClassification, error) {
+	// 打开文件
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	// 读取文件内容
+	bytes, err := ioutil.ReadAll(file)
+	if err != nil {
+		return nil, err
+	}
+
+	// 解析 JSON
+	var classifications []IndustryClassification
+	err = json.Unmarshal(bytes, &classifications)
+	if err != nil {
+		return nil, err
+	}
+
+	// 先分配层级
+	assignLevel(classifications, 1)
+
+	// 然后分配EntCode
+	startEntCode := uint64(192)
+	assignEntCode(classifications, &startEntCode)
+
+	return classifications, nil
+}
+
+// readFile d读取文件
+func readFile() []IndustryClassification {
+	// 读取 JSON 文件
+	filePath := "./国民经济行业分类_2017.json"
+	//var err error
+	classifications, err := readIndustryClassifications(filePath)
+	if err != nil {
+		fmt.Println("Error reading industry classifications:", err)
+		return []IndustryClassification{}
+	}
+
+	return classifications
+	//// 打印结果（调试用）
+	//output, _ := json.MarshalIndent(classifications, "", "  ")
+	//fmt.Println(string(output))
+}
+
+// removeDuplicates 去除重复字符串
+func removeDuplicates(arr []string) []string {
+	uniqueMap := make(map[string]bool)
+	var result []string
+	for _, str := range arr {
+		if !uniqueMap[str] {
+			uniqueMap[str] = true
+			result = append(result, str)
+		}
+	}
+	return result
+}
+
+// getMondayOfCurrentWeek 获取本周周一
+func getMondayOfCurrentWeek() time.Time {
+	// 获取当前时间
+	now := time.Now()
+	// 获取今天是本周的第几天（0 表示周日，1 表示周一，...，6 表示周六）
+	weekday := int(now.Weekday())
+	// 如果是周日，转换为 7（以便计算为上一周的最后一天）
+	if weekday == 0 {
+		weekday = 7
+	}
+	// 计算周一日期
+	monday := now.AddDate(0, 0, -weekday+1)
+	// 清除时间部分，保留日期
+	monday = time.Date(monday.Year(), monday.Month(), monday.Day(), 0, 0, 0, 0, monday.Location())
+	return monday
+}
+
+// CalculateRegionCode 处理地域代码
+func CalculateRegionCode(area string, city string, district string) (area_code string, city_code string, district_code string) {
+	area_code, city_code, district_code = "000000", "", ""
+	if district != "" {
+		key := area + "~" + city + "~" + district + "~"
+		code := RegionCodeData[key]
+		if code != "" {
+			district_code = code
+			city_code = code[:4] + "00"
+			area_code = code[:2] + "0000"
+			return
+		}
+	}
+	if city != "" {
+		key := area + "~" + city + "~" + "" + "~"
+		code := RegionCodeData[key]
+		if code != "" {
+			city_code = code
+			area_code = city_code[:2] + "0000"
+			return
+		}
+	}
+	if area != "" {
+		key := area + "~" + "" + "~" + "" + "~"
+		code := RegionCodeData[key]
+		if code != "" {
+			area_code = code
+			return
+		}
+	}
+	return
+}
+
+// getCompanyLabelBitmap 根据统一信用代码和ID获取bitmap 和对应标签字符串
+func getCompanyLabelBitmap(creditNo, companyID string) (bitLabels []uint64, nameLabels []string) {
+	//var bitLabels = make([]uint64, 0)  //bitmap数组
+	//var nameLabels = make([]string, 0) //标签数组
+	//1 dealCompanyNo
+	tags1 := dealCompanyNo(creditNo)
+	if len(tags1) > 0 {
+		bitLabels = append(bitLabels, tags1...)
+	}
+	//获取对应Bitmap的标签字符串
+	for _, v := range tags1 {
+		if la, ok := entLabelMap[v]; ok {
+			nameLabels = append(nameLabels, la)
+		}
+	}
+	//2.获取凭安库里的国标经济行业分类
+	label := getLabelsByID(companyID)
+	if label != "" {
+		labels := strings.Split(label, "-")
+		for _, v := range labels {
+			if bit, ok := nameBitMap[v]; ok {
+				bitLabels = append(bitLabels, bit)
+				nameLabels = append(nameLabels, v)
+			}
+		}
+	}
+
+	return
+}
+
+// getLabelsByID 根据名称，获取凭安库的国标经济行业分类
+func getLabelsByID(companyID string) string {
+	whereIndustry := map[string]interface{}{
+		"company_id": companyID,
+	}
+	industry, _ := MgoQY.FindOne("company_industry", whereIndustry)
+	if industry != nil && len(*industry) > 0 {
+		label := ""
+		if util.ObjToString((*industry)["industry_l1_name"]) != "" {
+			label = util.ObjToString((*industry)["industry_l1_name"])
+		} else {
+			return ""
+		}
+		//二级名称
+		if util.ObjToString((*industry)["industry_l2_name"]) != "" {
+			label = label + "-" + util.ObjToString((*industry)["industry_l2_name"])
+		}
+		if util.ObjToString((*industry)["industry_l3_name"]) != "" {
+			label = label + "-" + util.ObjToString((*industry)["industry_l3_name"])
+		}
+		if util.ObjToString((*industry)["industry_l4_name"]) != "" {
+			label = label + "-" + util.ObjToString((*industry)["industry_l4_name"])
+		}
+
+		return label
+	}
+
+	return ""
+}
+
+// dealCompanyNo dealCompanyNo 统一信用代码
+func dealCompanyNo(companyNo string) (newTags []uint64) {
+	// 前缀与标签映射表
+	prefixTagMap := map[string]uint64{
+		"11": 151, "12": 152, "13": 153, "19": 154,
+		"21": 155, "29": 156, "31": 157, "32": 158,
+		"33": 159, "34": 160, "35": 161, "39": 162,
+		"41": 163, "49": 164, "51": 165, "52": 166,
+		"53": 167, "59": 168, "61": 169, "62": 170,
+		"69": 171, "71": 172, "72": 173, "79": 174,
+		"81": 175, "89": 176, "91": 177, "92": 178,
+		"93": 179, "A1": 180, "A2": 181, "N1": 182,
+		"N2": 183, "N3": 184, "N9": 185, "Y1": 186,
+	}
+
+	// 遍历映射表进行前缀匹配
+	for prefix, tag := range prefixTagMap {
+		if strings.HasPrefix(companyNo, prefix) {
+			newTags = append(newTags, tag)
+		}
+	}
+
+	return
+}
+
+// dealOrgTags 处理国民经济行业分类
+func dealOrgTags(org_tags string) (newTags []uint64) {
+	var categoryMap = map[string]uint64{
+		"外交":         2193,
+		"发展和改革":      2194,
+		"科学技术/科技":    2195,
+		"民族事务":       2196,
+		"保密局":        2197,
+		"国安局":        2198,
+		"司法":         2199,
+		"法院":         2200,
+		"检察院":        2201,
+		"人力资源和社会保障":  2202,
+		"生态环境":       2203,
+		"交通运输":       2204,
+		"农业农村":       2205,
+		"退役军人事务":     2206,
+		"人民银行":       2207,
+		"国防":         2208,
+		"教育":         2209,
+		"党校":         2210,
+		"工业和信息化":     2211,
+		"公安":         2212,
+		"民政":         2213,
+		"财政":         2214,
+		"自然资源(包含规划)": 2215,
+		"住建":         2216,
+		"水利":         2217,
+		"商务":         2218,
+		"卫生健康":       2219,
+		"应急管理":       2220,
+		"审计":         2221,
+		"国有资产监督管理":   2222,
+		"海关":         2223,
+		"市场监督":       2224,
+		"证券监督管理":     2225,
+		"体育":         2226,
+		"统计":         2227,
+		"国际发展合作":     2228,
+		"税务":         2229,
+		"金融":         2230,
+		"广播电视":       2231,
+		"信访":         2232,
+		"知识产权":       2233,
+		"医疗保障":       2234,
+		"新华通讯社":      2235,
+		"气象":         2236,
+		"科学院":        2237,
+		"工程院":        2238,
+		"粮食和物资储备":    2239,
+		"数据":         2240,
+		"烟草专卖":       2241,
+		"林业和草原":      2242,
+		"民用航空":       2243,
+		"文物":         2244,
+		"疾病预防控制":     2245,
+		"消防救援":       2246,
+		"药品监督":       2247,
+		"能源":         2248,
+		"移民":         2249,
+		"铁路":         2250,
+		"邮政":         2251,
+		"中医药":        2252,
+		"外汇":         2253,
+		"供销合作社":      2254,
+		"公共资源交易中心":   2255,
+		"监狱":         2256,
+		"城乡建设":       2257,
+		"文旅":         2258,
+		"人民防空":       2259,
+		"园林":         2260,
+		"物流口岸":       2261,
+		"大数据":        2262,
+		"政务服务":       2263,
+		"地方史志":       2264,
+		"住房公积金管理中心":  2265,
+		"仲裁":         2266,
+		"招商":         2267,
+		"社保中心":       2268,
+		"管委会":        2269,
+		"人民政府":       2270,
+		"工商联":        2271,
+		"残联":         2272,
+		"妇联":         2273,
+		"艺术联":        2274,
+		"侨联":         2275,
+		"台联":         2276,
+		"城管":         2277,
+		"编办":         2278,
+		"政协":         2279,
+		"民主党派":       2280,
+		"党委":         2281,
+		"团委":         2282,
+		"人大":         2283,
+		"档案局":        2284,
+		"武装":         2285,
+		"医院":         2286,
+		"渔业":         2287,
+		"学校":         2288,
+	}
+
+	orgTags := strings.TrimSpace(org_tags)
+	if strings.Contains(orgTags, "-") {
+		classifications := readFile() //读取国标行业分类
+		if len(classifications) > 0 {
+			ss := findEntCodesByLabel(orgTags, classifications)
+			newTags = append(newTags, ss...)
+		}
+	} else {
+		//其它单独标签
+		for prefix, tag := range categoryMap {
+			if orgTags == prefix {
+				newTags = append(newTags, tag)
+			}
+		}
+	}
+
+	return
+}
+
+// 根据label返回对应的EntCode数组
+func findEntCodesByLabel(label string, classifications []IndustryClassification) []uint64 {
+	// 将label分割成多个层级
+	labels := strings.Split(label, "-")
+	var result []uint64
+	if len(labels) > 0 {
+		for k, v := range labels {
+			rs := findIndustryClassification(k+1, v, classifications)
+			if rs != nil {
+				result = append(result, rs.EntCode)
+			}
+		}
+	}
+	return result
+}
+
+// 根据level和name查找对应的IndustryClassification
+func findIndustryClassification(level int, name string, classifications []IndustryClassification) *IndustryClassification {
+	// 遍历每个分类
+	for _, classification := range classifications {
+		// 如果当前分类的level和name匹配，返回当前分类
+		if classification.Level == level && classification.Name == name {
+			return &classification
+		}
+
+		// 如果当前分类有子分类，则递归查找子分类
+		if len(classification.Children) > 0 {
+			if result := findIndustryClassification(level, name, classification.Children); result != nil {
+				return result
+			}
+		}
+	}
+
+	// 如果没有找到匹配的分类，返回nil
+	return nil
+}
+
+// getQyxyStd 获取qyxy_std 数据
+func getQyxyStd(companyName string) map[string]interface{} {
+	qyxy_info := map[string]interface{}{}
+	where := map[string]interface{}{
+		"company_name": companyName,
+	}
+
+	dataArr, _ := Mgo.FindOne("qyxy_std", where)
+	if len(*dataArr) > 0 {
+		qyxy_info = *dataArr
+		return qyxy_info
+	} else {
+		where2 := map[string]interface{}{
+			"history_name": companyName,
+		}
+		hisData, _ := MgoQY.FindOne("company_history_name", where2)
+		if len(*hisData) > 0 {
+			where3 := map[string]interface{}{
+				"company_name": companyName,
+			}
+			dataArr, _ = Mgo.FindOne("qyxy_std", where3)
+			if len(*dataArr) > 0 {
+				qyxy_info = *dataArr
+				return qyxy_info
+			}
+		}
+	}
+
+	return qyxy_info
+}

zhaiquan/export.go

@@ -14,8 +14,8 @@ import (
 // exportData 导出专项债数据
 func exportData() {
 	//查询条件
-	area := "天津市" //省份
-	area_list := "天津市"
+	area := "福建省" //省份
+	area_list := "厦门市"
 	start := "2025" // 发售时间
 
 	//