更新

bidding_test/bidd.go

@@ -7,7 +7,9 @@ import (
 	"errors"
 	"fmt"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+
 	"log"
 	"net/http"
 	"strconv"
@@ -396,3 +398,75 @@ func getCount() {
 	log.Println("count8", count8)
 	log.Println("over")
 }
+
+func deleteEs() {
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "qfw",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	//es
+	Es := &elastic.Elastic{
+		//S_esurl: "http://127.0.0.1:19908",
+		S_esurl:  "http://172.17.4.184:19908",
+		I_size:   5,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	Es.InitElasticSize()
+
+	//es 新集群
+	EsNew := &elastic.Elastic{
+		//S_esurl: "http://127.0.0.1:19905",
+		S_esurl:  "http://172.17.4.184:19905",
+		I_size:   5,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	EsNew.InitElasticSize()
+
+	where := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  mongodb.StringTOBsonId("673458dbb25c3e1deb2fef58"),
+			"$lte": mongodb.StringTOBsonId("6734638db25c3e1deb303821"),
+		},
+		"extracttype": -1,
+	}
+
+	query := sess.DB("qfw").C("bidding").Find(&where).Select(nil).Sort("-_id").Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%100 == 0 {
+			log.Println("current:", count)
+		}
+
+		id := mongodb.BsonIdToSId(tmp["_id"])
+
+		err := Es.DeleteByID("bidding", id)
+		if err != nil {
+			log.Println("es bidding", id)
+		}
+		err = EsNew.DeleteByID("bidding", id)
+		if err != nil {
+			log.Println("es new bidding", id)
+		}
+
+		err = EsNew.DeleteByID("bidding_year", id)
+		if err != nil {
+			log.Println("es new bidding_year", id)
+		}
+		err = EsNew.DeleteByID("bidding_free", id)
+		if err != nil {
+			log.Println("es new bidding_free", id)
+		}
+
+	}
+}

bidding_test/bidding_test.go

@@ -88,3 +88,76 @@ func TestUpdateBiddingEs(t *testing.T) {
 	}
 
 }
+
+func TestDeleteEsById(t *testing.T) {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	//es
+	Es := &elastic.Elastic{
+		S_esurl: "http://127.0.0.1:19908",
+		//S_esurl:  "http://172.17.4.184:19908",
+		I_size:   5,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	Es.InitElasticSize()
+
+	//es 新集群
+	EsNew := &elastic.Elastic{
+		S_esurl: "http://127.0.0.1:19905",
+		//S_esurl:  "http://172.17.4.184:19905",
+		I_size:   5,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	EsNew.InitElasticSize()
+
+	where := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  mongodb.StringTOBsonId("673458dbb25c3e1deb2fef58"),
+			"$lte": mongodb.StringTOBsonId("6734638db25c3e1deb303821"),
+		},
+		"extracttype": -1,
+	}
+
+	query := sess.DB("qfw").C("bidding").Find(&where).Select(nil).Sort("-_id").Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%100 == 0 {
+			log.Println("current:", count)
+		}
+
+		id := mongodb.BsonIdToSId(tmp["_id"])
+
+		err := Es.DeleteByID("bidding", id)
+		if err != nil {
+			log.Println("es bidding", id)
+		}
+		err = EsNew.DeleteByID("bidding", id)
+		if err != nil {
+			log.Println("es new bidding", id)
+		}
+
+		err = EsNew.DeleteByID("bidding_year", id)
+		if err != nil {
+			log.Println("es new bidding_year", id)
+		}
+		err = EsNew.DeleteByID("bidding_free", id)
+		if err != nil {
+			log.Println("es new bidding_free", id)
+		}
+
+	}
+
+}

bidding_test/main.go

@@ -52,10 +52,15 @@ func InitMgo() {
 }
 
 func main() {
-	InitMgo()
+
+	//deleteEs()
+	log.Println("数据删除完成")
+
+	//------------------//
+	//InitMgo()
 	//getBidding2()
 	//callAi()
-	getCount()
+	//getCount()
 
 	//updateTop()
 	//findData()

company_invest/allData.go

@@ -57,6 +57,7 @@ func investData() {
 	//host := "localhost:9090"
 	encodedPassword := url.QueryEscape(password)
 	dn := fmt.Sprintf("clickhouse://%s:%s@%s/information?dial_timeout=10s&read_timeout=20s", username, encodedPassword, host)
+
 	////dsn := "clickhouse://jybi_admin:Da#jy20230825@127.0.0.1:8123/default?dial_timeout=10s&read_timeout=20s"
 	////dsn := "clickhouse://jytop:pwdTopJy123@192.168.3.207:18123/information?dial_timeout=10s&read_timeout=20s"
 	////dsn := "clickhouse://wcc:123@localhost:9090/test?dial_timeout=10s&read_timeout=20s"

data_tags/config.toml

@@ -11,6 +11,7 @@
     host = "127.0.0.1:27017"
     #        host = "172.17.189.140:27080"
     db = "wcc"
+#    coll = "wcc_special_enterprise"
     coll = "wcc_special_enterprise"
     username = ""
     password = ""
@@ -164,7 +165,7 @@
     name = "交通运输" ## 标签名称
     field = ["company_name","organizer"] ## 识别字段
     sfield = "company_types" ## 标签保存字段
-    rule =["(公路事业发展中心|船业发展中心|道路运输业|公路质量检测中心)"]
+    rule =["(公路事业发展中心|船业发展中心|道路运输业|公路质量检测中心|公路管理所|公路联网管理中心|道路运输服务中心)"]
     weight =[0.8,0.5]  ## 权重
 
 
@@ -207,7 +208,7 @@
     name = "教育" ## 标签名称
     field = ["company_name","organizer"] ## 识别字段
     sfield = "company_types" ## 标签保存字段
-    rule =["(教育|教体局|小学|中学|幼儿园|技校|大学|学校)"]
+    rule =["(教育|教体局|小学|中学|幼儿园|技校|大学|学校|学院|图书馆)"]
     weight =[1,1]  ## 权重
 
     [labels.18a]
@@ -270,7 +271,7 @@
     name = "住建" ## 标签名称
     field = ["company_name","organizer"] ## 识别字段
     sfield = "company_types" ## 标签保存字段
-    rule =["(住房和城乡建设|建设局|房屋征收事务局|建筑业管理局|住房和城乡规划建设管理局|建设指挥部|住建|房保障和房地产管理|住房保障和房屋管理|住房保障局|房产服务中心|住房保障中心|不动产登记和交易中心|房产管理经营|房屋管理事务中心|房产管理局)"]
+    rule =["(住房和城乡建设|建设局|房屋征收事务局|建筑业管理局|住房和城乡规划建设管理局|建设指挥部|住建|房保障和房地产管理|住房保障和房屋管理|住房保障局|房产服务中心|住房保障中心|不动产登记和交易中心|房产管理经营|房屋管理事务中心|房产管理局|不动产登记中心)"]
     weight =[1,0.7]  ## 权重
 
 
@@ -285,7 +286,7 @@
     name = "水利" ## 标签名称
     field = ["company_name","organizer"] ## 识别字段
     sfield = "company_types" ## 标签保存字段
-    rule =["(水利|农林水牧局|水文水资源测报中心|水库管理局|黄河河务局|水库事务中心|水库保障所|水管所|水文水资源)"]
+    rule =["(水利|农林水牧局|水文水资源测报中心|水库管理局|黄河河务局|水库事务中心|水库保障所|水管所|水文水资源|南水北调工程)"]
     weight =[1,0.8]  ## 权重
 
     [labels.27]
@@ -299,7 +300,7 @@
     name = "卫生健康" ## 标签名称
     field = ["company_name","organizer"] ## 识别字段
     sfield = "company_types" ## 标签保存字段
-    rule =["(卫生健康|卫生和计划生育|中医管理局|卫生局|人口和计划生育|卫生运动|健康委员会|卫生服务中心|市容环卫|市容卫生|计划生育|卫生所|市容环境卫生)"]
+    rule =["(卫生健康|卫生和计划生育|中医管理局|卫生局|人口和计划生育|卫生运动|健康委员会|卫生服务中心|市容环卫|市容卫生|计划生育|卫生所|市容环境卫生|血站)"]
     weight =[1,0.8]  ## 权重
 
     [labels.29]
@@ -383,7 +384,7 @@
     name = "金融" ## 标签名称
     field = ["company_name","organizer"] ## 识别字段
     sfield = "company_types" ## 标签保存字段
-    rule =["(金融监督|金融)"]
+    rule =["(金融监督|金融|商品交易所)"]
     weight =[1,1]  ## 权重
 
     [labels.40]
@@ -579,7 +580,7 @@
     name = "文旅" ## 标签名称
     field = ["company_name","organizer"] ## 识别字段
     sfield = "company_types" ## 标签保存字段
-    rule =["(广电科技局|旅游和文化广电局|文化和广播影视局|旅游局|文化厅|文化局|旅游委员会|旅游发展委|旅游管理区|文化广电局|文化和旅游|文旅|文化服务中心)"]
+    rule =["(广电科技局|旅游和文化广电局|文化和广播影视局|旅游局|文化厅|文化局|旅游委员会|旅游发展委|旅游管理区|文化广电局|文化和旅游|文旅|文化服务中心|日报社|博物馆|博物院)"]
     weight =[1,0.8]  ## 权重
 
     [labels.68]
@@ -614,7 +615,7 @@
     name = "政务服务" ## 标签名称
     field = ["company_name","organizer"] ## 识别字段
     sfield = "company_types" ## 标签保存字段
-    rule =["(政务服务|综合行政执法|行政服务中心|登记管理局|行政审批和政务信息|行政管理局|政务信息)"]
+    rule =["(政务服务|综合行政执法|行政服务中心|登记管理局|行政审批和政务信息|行政管理局|政务信息|事业服务中心|福利院|地震局|退休干部)"]
     weight =[1,0.7]  ## 权重
 
     [labels.72a]
@@ -670,7 +671,7 @@
     name = "管委会" ## 标签名称
     field = ["company_name","organizer"] ## 识别字段
     sfield = "company_types" ## 标签保存字段
-    rule =["(管委会|风景区开发建设委员会|香山湖管理区|管理区管理委员会|景区管理委员会)^(中共|政协|人大)","(示范区|开发区|产业集聚区)(委员会)"]
+    rule =["(管委会|风景区开发建设委员会|香山湖管理区|管理区管理委员会|景区管理委员会|公园管理委员会)^(中共|政协|人大)","(示范区|开发区|产业集聚区)(委员会)"]
     weight =[1,0.8]  ## 权重
 
     [labels.79]
@@ -824,5 +825,5 @@
     name = "学校" ## 标签名称
     field = ["company_name","organizer"] ## 识别字段
     sfield = "company_types" ## 标签保存字段
-    rule =["(小学|中学|幼儿园|技校|大学|学校)"]
+    rule =["(小学|中学|幼儿园|技校|大学|学校|初中|高中)"]
     weight =[1,1]  ## 权重

data_tags/go.mod

@@ -2,7 +2,6 @@ module data_tags
 
 go 1.22.6
 
-
 require (
 	github.com/itcwc/go-zhipu v0.0.0-20240626065325-ffc8bf1cfaaa
 	github.com/olivere/elastic/v7 v7.0.32
@@ -23,6 +22,8 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
 	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/go-resty/resty/v2 v2.15.1 // indirect
+	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -59,12 +60,13 @@ require (
 	go.mongodb.org/mongo-driver v1.11.4 // indirect
 	go.opentelemetry.io/otel v1.26.0 // indirect
 	go.opentelemetry.io/otel/trace v1.26.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gorm.io/driver/mysql v1.5.7 // indirect
 )

data_tags/go.sum

@@ -36,7 +36,11 @@ github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AY
 github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-resty/resty/v2 v2.15.1 h1:vuna8FM2EaQ6IYbtjh+Gjh00uu7xEWuuGyTKeIaYkvE=
+github.com/go-resty/resty/v2 v2.15.1/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
@@ -199,6 +203,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -223,6 +229,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
 golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -232,6 +240,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -244,6 +254,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
 golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -252,6 +264,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -302,6 +316,9 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/clickhouse v0.6.1 h1:t7JMB6sLBXxN8hEO6RdzCbJCwq/jAEVZdwXlmQs1Sd4=
 gorm.io/driver/clickhouse v0.6.1/go.mod h1:riMYpJcGZ3sJ/OAZZ1rEP1j/Y0H6cByOAnwz7fo2AyM=
+gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
+gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
 gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

data_tags/main.go

@@ -31,9 +31,20 @@ type DataRes struct {
 
 func main() {
 
-	//esLabels()
+	esLabels()
 	//zpAi()
-	zpAi2()
+	//zpAi2()
+
+	//companyNameLabel() //根据企业名称关键词，打行业标签；规则在tidb
+	//updateBusinessScope() //提取企业主营营业范围
+	//businessScopeLabel() //经营范围 ，行业标签
+	//getJiao() // 名称标签和业务范围 标签交集
+	getIndustry() //凭安标签+大模型
+	//testIndustry() // 测试采购单位数据。凭安标签
+	//TestLabel()
+	//testBidamount()
+	log.Println("所有程序结束")
+
 }
 
 // esLabels 组织机构标签生索引
@@ -320,3 +331,125 @@ func zpAi2() {
 
 	log.Println("over-------")
 }
+
+// TestLabel 测试无标签数据，获取凭安标签
+func TestLabel() {
+	f, err := excelize.OpenFile("./招标项目-汇总(截至0807).xlsx")
+	//f, err := excelize.OpenFile("./无标签数据-大模型优化.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	//181 凭安库
+	MgoQY := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.181:27001",
+		//MongodbAddr: "127.0.0.1:27001",
+		DbName:   "mixdata",
+		Size:     10,
+		UserName: "",
+		Password: "",
+		//Direct:      true,
+	}
+	MgoQY.InitPool()
+
+	rows, err := f.GetRows("招标项目清单")
+	//rows, err := f.GetRows("Sheet7")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for i := 1; i < len(rows); i++ {
+		if i%100 == 0 {
+			log.Println("current:", i, rows[i][2])
+		}
+		name := rows[i][2]
+		where := map[string]interface{}{
+			"company_name": name,
+		}
+
+		data, _ := MgoQY.FindOne("company_base", where)
+		companyID := util.ObjToString((*data)["company_id"])
+		if companyID != "" {
+			whereIndustry := map[string]interface{}{
+				"company_id": companyID,
+			}
+			industry, _ := MgoQY.FindOne("company_industry", whereIndustry)
+			if industry != nil && len(*industry) > 0 {
+				if util.ObjToString((*industry)["industry_l1_name"]) != "" {
+					f.SetCellValue("Sheet7", fmt.Sprintf("%s%d", "F", i+1), (*industry)["industry_l1_name"])
+				}
+				//二级名称
+				if util.ObjToString((*industry)["industry_l2_name"]) != "" {
+					f.SetCellValue("Sheet7", fmt.Sprintf("%s%d", "G", i+1), (*industry)["industry_l2_name"])
+				}
+				if util.ObjToString((*industry)["industry_l3_name"]) != "" {
+					f.SetCellValue("Sheet7", fmt.Sprintf("%s%d", "H", i+1), (*industry)["industry_l3_name"])
+				}
+				if util.ObjToString((*industry)["industry_l4_name"]) != "" {
+					f.SetCellValue("Sheet7", fmt.Sprintf("%s%d", "I", i+1), (*industry)["industry_l4_name"])
+				}
+			}
+		}
+
+		f.Save()
+	}
+
+	log.Println("ooooooooooooo")
+}
+
+func testBidamount() {
+	MgoB := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "qfw",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	MgoB.InitPool()
+
+	type BidAmount struct {
+		TotalAmount float64
+		Total       int
+	}
+	mapTotalAmount := make(map[string]BidAmount, 0)
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+	it := sess.DB("qfw").C("wcc_project_0925").Find(nil).Select(nil).Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count)
+		}
+		key := util.ObjToString(tmp["totalIndustry"])
+		if key == "" {
+			continue
+		}
+		ex, _ := mapTotalAmount[key]
+		ex.Total++
+		ex.TotalAmount += util.Float64All(tmp["bidamount"])
+		mapTotalAmount[key] = ex
+	}
+
+	for k, _ := range mapTotalAmount {
+		insert := map[string]interface{}{
+			"totalAmount": mapTotalAmount[k].TotalAmount,
+			"total":       mapTotalAmount[k].Total,
+			"name":        k,
+		}
+
+		MgoB.Save("wcc_project_0925_amount", insert)
+	}
+	log.Println("ooooooooooooo")
+
+}

data_tags/name.go

@@ -0,0 +1,891 @@
+package main
+
+import (
+	"fmt"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"sort"
+	"strings"
+	"sync"
+	"unicode/utf8"
+)
+
+type CodeIndCode struct {
+	CODE         string `gorm:"column:CODE;type:varchar(6);not null"`
+	NAME         string `gorm:"column:NAME;type:varchar(200);not null"`
+	KINDSIGN     string `gorm:"column:KINDSIGN;type:char(1);not null"`
+	MAXKINDSIGN  string `gorm:"column:MAXKINDSIGN;type:char(1);not null"`
+	MIDKINDSIGN  string `gorm:"column:MIDKINDSIGN;type:char(1);not null"`
+	MINKINDSIGN  string `gorm:"column:MINKINDSIGN;type:char(1);not null"`
+	PARIND       string `gorm:"column:PARIND;type:varchar(6);default:null"`
+	CHOSIGN      string `gorm:"column:CHOSIGN;type:char(1);not null"`
+	RISKTYPECODE string `gorm:"column:RISKTYPECODE;type:varchar(20);default:null"`
+	ENTTYPES     string `gorm:"column:ENTTYPES;type:varchar(100);default:null"`
+}
+
+type CodeINDUSTRYFEAT struct {
+	ID             int64  `gorm:"column:ID;type:bigint(20);not null"`
+	HYCODE         string `gorm:"column:HYCODE;type:varchar(6);not null"`
+	HYPECUL        string `gorm:"column:HYPECUL;type:varchar(100);not null"`
+	BUSITYPE       string `gorm:"column:BUSITYPE;type:varchar(100);default:null"`
+	ENTTYPE        string `gorm:"column:ENTTYPE;type:varchar(200);default:null"`
+	ENTTYPEGROUP   string `gorm:"column:ENTTYPEGROUP;type:varchar(200);default:null"`
+	FROMTYPE       string `gorm:"column:FROMTYPE;type:char(1);default:null"`
+	ENTMATTERCODES string `gorm:"column:ENTMATTERCODES;type:varchar(3000);default:null"`
+	ISRECOMM       string `gorm:"column:ISRECOMM;type:char(1);default:null"`
+}
+
+// CodeBUSITERMCODE 经营范围行业
+type CodeBUSITERMCODE struct {
+	CODE              string `gorm:"column:CODE;not null"`
+	NAME              string `gorm:"column:NAME"`
+	PARENTCODE        string `gorm:"column:PARENTCODE"`
+	COMMONSIGN        string `gorm:"column:COMMONSIGN"`
+	POSITIONSIGN      string `gorm:"column:POSITIONSIGN"`
+	CHOSIGN           string `gorm:"column:CHOSIGN"`
+	BIZTYPE           string `gorm:"column:BIZTYPE"`
+	POSITIONNAME      string `gorm:"column:POSITIONNAME"`
+	TIPS              string `gorm:"column:TIPS"`
+	INDUSTYPECODE     string `gorm:"column:INDUSTYPECODE"`
+	OTHERTYPE         string `gorm:"column:OTHERTYPE"`
+	RESON             string `gorm:"column:RESON"`
+	INDUSTYPENAME     string `gorm:"column:INDUSTYPENAME"`
+	FLAG              string `gorm:"column:FLAG"`
+	ENTTYPE           string `gorm:"column:ENTTYPE"`
+	ENTPROPERTY       string `gorm:"column:ENTPROPERTY"`
+	ISBLONGINDUSTRIAL string `gorm:"column:ISBLONGINDUSTRIAL"`
+	MATTERSCODE       string `gorm:"column:MATTERSCODE"`
+	MATTERSNAME       string `gorm:"column:MATTERSNAME"`
+	LAWREGULATION     string `gorm:"column:LAWREGULATION"`
+	CATEGORY          string `gorm:"column:CATEGORY"`
+	HEADING           string `gorm:"column:HEADING"`
+	MATTERSTYPE       string `gorm:"column:MATTERSTYPE"`
+}
+
+var (
+	industryFeats []CodeINDUSTRYFEAT
+	businessScope []CodeBUSITERMCODE
+	indCodes      []CodeIndCode
+	// 将 CODE_IndCode 表数据加载到内存中的 map
+	indCodeMap = make(map[string]CodeIndCode)
+)
+
+// companyNameLabel 根据公司名称，给企业打标记，利用拓扑标签
+func companyNameLabel() {
+	//username := "datascbi"
+	//password := "Da#Bi20221111SC"
+	//host := "127.0.0.1:4001"
+	////host := "172.17.162.25:4000"
+
+	//测试环境tidb
+	username := "root"
+	password := "=PDT49#80Z!RVv52_z"
+	host := "192.168.3.14:4000"
+	database := "wcc"
+	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", username, password, host, database)
+	// 连接到数据库
+	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
+	if err != nil {
+		log.Println("Failed to connect to database:", err)
+		return
+	}
+
+	db.Table("CODE_INDUSTRYFEAT").Find(&industryFeats)
+
+	db.Table("CODE_IndCode").Find(&indCodes)
+
+	for _, indCode := range indCodes {
+		indCodeMap[indCode.CODE] = indCode
+	}
+
+	//开始调用
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw").C("ent_info_buyer_912").Find(nil).Select(nil).Sort("_id").Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+
+	ch := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		name := util.ObjToString(tmp["name"])
+		if count%1000 == 0 {
+			log.Println(count, "----", name)
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			//
+			res := GetIndustryNames(name)
+			res = removeDuplicates(res)
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			if len(res) > 0 {
+				update := map[string]interface{}{
+					//"top_labels": res,
+					"top_labels_name": res,
+				}
+				if len(res) == 1 {
+					update["credibility"] = 5
+				}
+				MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("企业名称匹配结束")
+}
+
+// GetIndustryNames2 根据企业名称关键词 获取对应行业标签
+func GetIndustryNames2(companyName string) []string {
+	var hycodes []CodeINDUSTRYFEAT
+	for _, feat := range industryFeats {
+		if strings.Contains(companyName, feat.HYPECUL) {
+			hycodes = append(hycodes, feat)
+		}
+	}
+
+	var industryNames []string
+	if len(hycodes) > 0 {
+		for _, hycode := range hycodes {
+			indCode, ok := indCodeMap[hycode.HYCODE]
+			if ok {
+				var path []string
+				path = append(path, hycode.HYPECUL)
+				path = append(path, indCode.NAME)
+				for indCode.PARIND != "" {
+					indCode = indCodeMap[indCode.PARIND]
+					path = append(path, indCode.NAME)
+				}
+				// 反转路径
+				for i, j := 0, len(path)-1; i < j; i, j = i+1, j-1 {
+					path[i], path[j] = path[j], path[i]
+				}
+				industryNames = append(industryNames, strings.Join(path, "-"))
+			}
+		}
+	}
+
+	return industryNames
+}
+
+// GetIndustryNames 根据企业名称关键词 获取对应行业标签
+func GetIndustryNames(companyName string) []string {
+	var hycodes []CodeINDUSTRYFEAT
+	for _, feat := range industryFeats {
+		if strings.Contains(companyName, feat.HYPECUL) {
+			hycodes = append(hycodes, feat)
+		}
+	}
+
+	// 按 HYPECUL 长度降序排序
+	sort.Slice(hycodes, func(i, j int) bool {
+		return len(hycodes[i].HYPECUL) > len(hycodes[j].HYPECUL)
+	})
+
+	longestLength := 0
+	if len(hycodes) > 0 {
+		longestLength = len(hycodes[0].HYPECUL)
+	}
+
+	var longestHycodes []CodeINDUSTRYFEAT
+	for _, hycode := range hycodes {
+		if len(hycode.HYPECUL) == longestLength {
+			longestHycodes = append(longestHycodes, hycode)
+		}
+	}
+
+	var industryNames []string
+	if len(longestHycodes) > 0 {
+		//// 选择第一个最长的
+		//selectedHycode := longestHycodes[0]
+		for _, selectedHycode := range longestHycodes {
+			indCode, ok := indCodeMap[selectedHycode.HYCODE]
+			if ok {
+				var path []string
+				path = append(path, selectedHycode.HYPECUL) //匹配上的关键词
+				path = append(path, indCode.NAME)           // 标记分类名称；
+				for indCode.PARIND != "" {
+					indCode = indCodeMap[indCode.PARIND]
+					path = append(path, indCode.NAME)
+				}
+				// 反转路径
+				for i, j := 0, len(path)-1; i < j; i, j = i+1, j-1 {
+					path[i], path[j] = path[j], path[i]
+				}
+				industryNames = append(industryNames, strings.Join(path, "-"))
+			}
+		}
+
+	}
+
+	return industryNames
+}
+
+/**
+------------------------------------------------------------------
+*/
+// businessScopeLabel 经营范围标签
+func businessScopeLabel() {
+	//测试环境tidb
+	username := "root"
+	password := "=PDT49#80Z!RVv52_z"
+	host := "192.168.3.14:4000"
+	database := "wcc"
+	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", username, password, host, database)
+	// 连接到数据库
+	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
+	if err != nil {
+		log.Println("Failed to connect to database:", err)
+		return
+	}
+
+	db.Table("CODE_BUSITERMCODE").Find(&businessScope)
+	db.Table("CODE_IndCode").Find(&indCodes)
+
+	for _, indCode := range indCodes {
+		indCodeMap[indCode.CODE] = indCode
+	}
+	//
+
+	//开始调用
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw").C("ent_info_buyer_912").Find(nil).Select(nil).Sort("_id").Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+
+	ch := make(chan bool, 15)
+	wg := &sync.WaitGroup{}
+
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count)
+		}
+		bus := util.ObjToString(tmp["businessScope"])
+		if bus == "" {
+			continue
+		}
+
+		//if _, ok := tmp["top_labels2"]; ok {
+		//	continue
+		//}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			//
+			business := util.ObjToString(tmp["businessScope"])
+
+			//if utf8.RuneCountInString(business) > 50 {
+			//	if strings.Contains(business, "。") {
+			//		if utf8.RuneCountInString(business[:strings.Index(business, "。")]) < 50 {
+			//			business = business[:strings.Index(business, "。")]
+			//		} else if strings.Contains(business, "；") {
+			//			business = business[:strings.Index(business, "；")]
+			//		}
+			//	} else if strings.Contains(business, "；") {
+			//		business = business[:strings.Index(business, "；")]
+			//	}
+			//}
+
+			res := GetIndustryBusiness(business)
+			res = removeDuplicates(res)
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			if len(res) > 0 {
+				update := map[string]interface{}{
+					"top_labels_businessScope": res,
+				}
+				//代表只有一个分类；
+				if len(res) == 1 {
+					update["credibility"] = 10
+				}
+				MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+
+	}
+
+	wg.Wait()
+
+	log.Println("结束~~~~~~~~~~~")
+}
+
+// GetIndustryBusiness 根据经营范围获取行业标签
+func GetIndustryBusiness(business string) []string {
+	var hycodes []CodeBUSITERMCODE
+	for _, feat := range businessScope {
+		if strings.Contains(business, feat.NAME) {
+			hycodes = append(hycodes, feat)
+		}
+	}
+
+	var industryNames []string
+	if len(hycodes) > 0 {
+		for _, hycode := range hycodes {
+			business_codes := strings.Split(hycode.INDUSTYPECODE, ";")
+			for _, v := range business_codes {
+				indCode, ok := indCodeMap[v]
+				if ok {
+					var path []string
+					//path = append(path, hycode.NAME)
+					path = append(path, indCode.NAME)
+					for indCode.PARIND != "" {
+						indCode = indCodeMap[indCode.PARIND]
+						path = append(path, indCode.NAME)
+					}
+					// 反转路径
+					for i, j := 0, len(path)-1; i < j; i, j = i+1, j-1 {
+						path[i], path[j] = path[j], path[i]
+					}
+					industryNames = append(industryNames, strings.Join(path, "-"))
+				}
+			}
+
+		}
+	}
+
+	return industryNames
+}
+
+// updateBusinessScope 提取企业主营营业范围
+func updateBusinessScope() {
+	//开始调用
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw").C("ent_info_buyer_912").Find(nil).Select(nil).Sort("_id").Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+	ch := make(chan bool, 15)
+	wg := &sync.WaitGroup{}
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count)
+		}
+		bus := util.ObjToString(tmp["business_scope"])
+		if bus == "" {
+			continue
+		}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			//
+			//business := util.ObjToString(tmp["business_scope"])
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			//update := map[string]interface{}{
+			//	"businessScope": extractMainBusiness(business),
+			//}
+			//MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+
+			//=========//
+			if topBus, ok := tmp["top_labels_businessScope"]; ok {
+				if topBus2, ok2 := topBus.([]interface{}); ok2 {
+					if len(topBus2) == 1 {
+						update := map[string]interface{}{
+							"credibility": 10,
+						}
+						MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+					}
+				}
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+
+	}
+
+	wg.Wait()
+	log.Println("1234444444")
+}
+
+// getJiao 获取根据企业名称，营业范围标记结果的交集
+func getJiao() {
+	//开始调用
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw").C("ent_info_buyer_912").Find(nil).Select(nil).Sort("_id").Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+	ch := make(chan bool, 15)
+	wg := &sync.WaitGroup{}
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count)
+		}
+		bus := util.ObjToString(tmp["business_scope"])
+		if bus == "" {
+			continue
+		}
+
+		if _, ok := tmp["top_labels_businessScope"]; !ok {
+			continue
+		}
+
+		if _, ok := tmp["top_labels_name"]; !ok {
+			continue
+		}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			//
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			intersection := findIntersection(tmp)
+			// 只有一个交集
+			if len(intersection) == 1 {
+				update := map[string]interface{}{
+					"credibility":      11,
+					"top_intersection": intersection,
+				}
+				MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("1234444444")
+}
+
+// getIndustry 获取凭安企业标签
+func getIndustry() {
+	//开始调用
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+	//181 凭安库
+	MgoQY := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: "127.0.0.1:27001",
+		DbName:      "mixdata",
+		Size:        10,
+		UserName:    "",
+		Password:    "",
+		Direct:      true,
+	}
+	MgoQY.InitPool()
+
+	key := "4d5206b1b297c1e7b77f9578edcb2cf7.TNU2i8G1oUNdR02i"
+	model := "glm-4-flash"
+
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw").C("ent_info_buyer_912").Find(nil).Select(nil).Sort("_id").Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+	ch := make(chan bool, 15)
+	wg := &sync.WaitGroup{}
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count)
+		}
+		// 名称小于5 认为不
+		if utf8.RuneCountInString(util.ObjToString(tmp["name"])) < 5 {
+			continue
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			//
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			name := mongodb.BsonIdToSId(tmp["name"])
+			//if name == "某部" || name == "某单位" {
+			//
+			//}
+			where := map[string]interface{}{
+				"company_name": name,
+			}
+			data, _ := MgoQY.FindOne("company_base", where)
+			companyID := util.ObjToString((*data)["company_id"])
+			//1.公司ID不为空，直接使用凭安标签
+			if companyID != "" {
+				whereIndustry := map[string]interface{}{
+					"company_id": companyID,
+				}
+				industry, _ := MgoQY.FindOne("company_industry", whereIndustry)
+				if industry != nil && len(*industry) > 0 {
+					totalIndustry := make([]interface{}, 0)
+					label := ""
+					if util.ObjToString((*industry)["industry_l1_name"]) != "" {
+						label = util.ObjToString((*industry)["industry_l1_name"])
+					}
+					//二级名称
+					if util.ObjToString((*industry)["industry_l2_name"]) != "" {
+						label = label + "-" + util.ObjToString((*industry)["industry_l2_name"])
+					}
+					if util.ObjToString((*industry)["industry_l3_name"]) != "" {
+						label = label + "-" + util.ObjToString((*industry)["industry_l3_name"])
+					}
+					if util.ObjToString((*industry)["industry_l4_name"]) != "" {
+						label = label + "-" + util.ObjToString((*industry)["industry_l4_name"])
+					}
+
+					tmpD := map[string]interface{}{
+						"name":  label,
+						"score": 1.0,
+					}
+					totalIndustry = append(totalIndustry, tmpD)
+					update := map[string]interface{}{
+						"totalIndustry": totalIndustry,
+						"industry_type": 1, //通过凭安
+					}
+					MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+				} else {
+					//2.company_base 表存在，但是没有对应标签，需要调用大模型
+					business := util.ObjToString(tmp["businessScope"])
+					if utf8.RuneCountInString(business) > 50 {
+						r := []rune(business)
+						business = string(r[:50])
+					}
+					res := ZpAI3(key, model, name, business)
+					if len(res) > 0 {
+						if resD, ok := res["result"].([]interface{}); ok {
+							labels := make([]interface{}, 0)
+							for _, resDa := range resD {
+								if v, ok := resDa.(map[string]interface{}); ok {
+									//log.Println(v)
+									label := ""
+									if util.ObjToString(v["label1"]) != "" {
+										label = util.ObjToString(v["label1"])
+									}
+									if util.ObjToString(v["label2"]) != "" {
+										label = label + "-" + util.ObjToString(v["label2"])
+									}
+									if util.ObjToString(v["label3"]) != "" {
+										label = label + "-" + util.ObjToString(v["label3"])
+									}
+									if len(label) > 0 && util.Float64All(v["score"]) > 0 {
+										tm := map[string]interface{}{
+											"name":  label,
+											"score": v["score"],
+										}
+										labels = append(labels, tm)
+									}
+								}
+							}
+							if len(labels) > 0 {
+								update := map[string]interface{}{
+									"totalIndustry": labels,
+									"industry_type": 2, //通过大模型
+								}
+								MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+							}
+						}
+					}
+				}
+			} else {
+				//3.凭安不存在，直接使用大模型
+				//2.company_base 表存在，但是没有对应标签，需要调用大模型
+				business := util.ObjToString(tmp["businessScope"])
+				if utf8.RuneCountInString(business) > 50 {
+					r := []rune(business)
+					business = string(r[:50])
+				}
+				res := ZpAI3(key, model, name, business)
+				if len(res) > 0 {
+					if resD, ok := res["result"].([]interface{}); ok {
+						labels := make([]interface{}, 0)
+						for _, resDa := range resD {
+							if v, ok := resDa.(map[string]interface{}); ok {
+								//log.Println(v)
+								label := ""
+								if util.ObjToString(v["label1"]) != "" {
+									label = util.ObjToString(v["label1"])
+								}
+								if util.ObjToString(v["label2"]) != "" {
+									label = label + "-" + util.ObjToString(v["label2"])
+								}
+								if util.ObjToString(v["label3"]) != "" {
+									label = label + "-" + util.ObjToString(v["label3"])
+								}
+								if len(label) > 0 && util.Float64All(v["score"]) > 0 {
+									tm := map[string]interface{}{
+										"name":  label,
+										"score": v["score"],
+									}
+									labels = append(labels, tm)
+								}
+							}
+						}
+						if len(labels) > 0 {
+							update := map[string]interface{}{
+								"totalIndustry": labels,
+								"industry_type": 2, //通过大模型
+							}
+							MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+						}
+					}
+				}
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("1234444444")
+}
+
+// testIndustry 测试采购单位，凭安库标签
+func testIndustry() {
+	MgoB := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "qfw",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	MgoB.InitPool()
+	//181 凭安库
+	MgoQY := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.181:27001",
+		//MongodbAddr: "127.0.0.1:27001",
+		DbName:   "mixdata",
+		Size:     10,
+		UserName: "",
+		Password: "",
+		//Direct:      true,
+	}
+	MgoQY.InitPool()
+
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw").C("wcc_project_0925").Find(nil).Select(nil).Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+	ch := make(chan bool, 15)
+	wg := &sync.WaitGroup{}
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count)
+		}
+
+		if _, ok := tmp["totalIndustry"]; ok {
+			continue
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			name := mongodb.BsonIdToSId(tmp["buyer"])
+			where := map[string]interface{}{
+				"company_name": name,
+			}
+			if name == "" {
+				return
+			}
+			data, _ := MgoQY.FindOne("company_base", where)
+			companyID := util.ObjToString((*data)["company_id"])
+
+			//1.公司ID不为空，直接使用凭安标签
+			if companyID != "" {
+				whereIndustry := map[string]interface{}{
+					"company_id": companyID,
+				}
+				industry, _ := MgoQY.FindOne("company_industry", whereIndustry)
+				if industry != nil && len(*industry) > 0 {
+					//totalIndustry := make([]interface{}, 0)
+					label := ""
+					if util.ObjToString((*industry)["industry_l1_name"]) != "" {
+						label = util.ObjToString((*industry)["industry_l1_name"])
+					}
+					//二级名称
+					if util.ObjToString((*industry)["industry_l2_name"]) != "" {
+						label = label + "-" + util.ObjToString((*industry)["industry_l2_name"])
+					}
+					//if util.ObjToString((*industry)["industry_l3_name"]) != "" {
+					//	label = label + "-" + util.ObjToString((*industry)["industry_l3_name"])
+					//}
+					//if util.ObjToString((*industry)["industry_l4_name"]) != "" {
+					//	label = label + "-" + util.ObjToString((*industry)["industry_l4_name"])
+					//}
+
+					//tmpD := map[string]interface{}{
+					//	"name":  label,
+					//	"score": 1.0,
+					//}
+					//totalIndustry = append(totalIndustry, tmpD)
+					update := map[string]interface{}{
+						"totalIndustry": label,
+						"industry_type": 1, //通过凭安
+					}
+					MgoB.UpdateById("wcc_project_0925", id, map[string]interface{}{"$set": update})
+				}
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("1234444444")
+
+}
+
+//---------------------------//
+
+type Category struct {
+	//ParentCode string
+	Level     int
+	CODE      string `gorm:"column:CODE;type:varchar(6);not null"`
+	NAME      string `gorm:"column:NAME;type:varchar(200);not null"`
+	TotalName string //完整分类路径全称
+	//KINDSIGN     string `gorm:"column:KINDSIGN;type:char(1);not null"`
+	//MAXKINDSIGN  string `gorm:"column:MAXKINDSIGN;type:char(1);not null"`
+	//MIDKINDSIGN  string `gorm:"column:MIDKINDSIGN;type:char(1);not null"`
+	//MINKINDSIGN  string `gorm:"column:MINKINDSIGN;type:char(1);not null"`
+	PARIND string `gorm:"column:PARIND;type:varchar(6);default:null"`
+	//CHOSIGN      string `gorm:"column:CHOSIGN;type:char(1);not null"`
+	//RISKTYPECODE string `gorm:"column:RISKTYPECODE;type:varchar(20);default:null"`
+	//ENTTYPES     string `gorm:"column:ENTTYPES;type:varchar(100);default:null"`
+}
+
+func connectToDB() (*gorm.DB, error) {
+	username := "root"
+	password := "=PDT49#80Z!RVv52_z"
+	host := "192.168.3.14:4000"
+	database := "wcc"
+	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", username, password, host, database)
+	//dsn := "username:password@tcp(host:port)/database_name?charset=utf8mb4&parseTime=True&loc=Local"
+	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
+	if err != nil {
+		return nil, err
+	}
+	return db, nil
+}
+
+func buildCategoryTree(db *gorm.DB) ([]Category, error) {
+	topLevelCategories := []Category{}
+	result := db.Debug().Table("CODE_IndCode").Where("PARIND IS NULL").Find(&topLevelCategories)
+	if result.Error != nil {
+		return nil, result.Error
+	}
+
+	var categories []Category
+	for _, topCategory := range topLevelCategories {
+		topCategory.Level = 1
+		topCategory.TotalName = topCategory.NAME
+		categories = append(categories, topCategory)
+		subCategories, err := findSubCategories(db, topCategory.CODE, 2, topCategory.TotalName)
+		if err != nil {
+			return nil, err
+		}
+		categories = append(categories, subCategories...)
+	}
+
+	return categories, nil
+}
+
+func findSubCategories(db *gorm.DB, parentCode string, level int, parentName string) ([]Category, error) {
+	subCategories := []Category{}
+	result := db.Table("CODE_IndCode").Where("PARIND =?", parentCode).Find(&subCategories)
+	if result.Error != nil {
+		return nil, result.Error
+	}
+
+	for i := range subCategories {
+		subCategories[i].Level = level
+		subCategories[i].TotalName = parentName + "-" + subCategories[i].NAME
+	}
+
+	var allSubCategories []Category
+	for _, subCategory := range subCategories {
+		subSubCategories, err := findSubCategories(db, subCategory.CODE, level+1, subCategory.TotalName)
+		if err != nil {
+			return nil, err
+		}
+		allSubCategories = append(allSubCategories, subCategory)
+		allSubCategories = append(allSubCategories, subSubCategories...)
+	}
+
+	return allSubCategories, nil
+}

data_tags/readme.md

@@ -1,6 +1,6 @@
 
 ### 功能
-给数据打标记，使用分类程序规则结构；y依据配置文件配置 `labels` 标记数据，标记结果如下：
+给数据打标记，使用分类程序规则结构；依据配置文件配置 `labels` 标记数据，标记结果如下：
 ```json
     "base_types" : [
         {
@@ -20,4 +20,6 @@
 本次数据只涉及181 凭安的 `special_gov_unit` 和 `special_enterprise`
 
 ### 增量
-    针对增量数据，可以根据 `_id` 处理增量数据单独导出数据，再针对增量数据打标记和更新索引
+    针对增量数据，可以根据 `_id` 处理增量数据单独导出数据，再针对增量数据打标记和更新索引
+
+

data_tags/tag_test.go

@@ -1,15 +1,22 @@
 package main
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"github.com/olivere/elastic/v7"
 	"github.com/xuri/excelize/v2"
+	"io/ioutil"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
+	"net/http"
 	"regexp"
 	"strings"
+	"sync"
 	"testing"
 	"time"
+	"unicode/utf8"
 )
 
 func TestMergeLabelData(t *testing.T) {
@@ -59,11 +66,13 @@ func TestUpdateIndexByCompanyName(t *testing.T) {
 	}
 }
 
+// TestXlsx 测试组织机构标签
 func TestXlsx(t *testing.T) {
 	REG, _ = regexp.Compile(`\(.*?\)\d*`)
 	InitConfig()
 
-	f, err := excelize.OpenFile("./招标项目-汇总(截至0807).xlsx")
+	//f, err := excelize.OpenFile("./招标项目-汇总(截至0807).xlsx")
+	f, err := excelize.OpenFile("./业主打标结果分析.xlsx")
 	if err != nil {
 		fmt.Println(err)
 		return
@@ -74,7 +83,7 @@ func TestXlsx(t *testing.T) {
 		}
 	}()
 
-	rows, err := f.GetRows("招标项目清单")
+	rows, err := f.GetRows("售后--样例数据")
 	if err != nil {
 		fmt.Println(err)
 		return
@@ -82,7 +91,7 @@ func TestXlsx(t *testing.T) {
 	//log.Println(rows)
 
 	for i := 1; i < len(rows); i++ {
-		name := rows[i][4]
+		name := rows[i][2]
 		ResMap := make([]LabelData, 0)
 		for _, v := range GF.Labels {
 			v.RegRule = DealRules(v.Rule)
@@ -94,20 +103,75 @@ func TestXlsx(t *testing.T) {
 			}
 		}
 		mergedMap := MergeLabelData(ResMap)
-		tagValues := make([]string, 0)
-		for _, values := range mergedMap {
-			for _, value := range values {
-				tagValues = append(tagValues, value.Name)
+		if len(mergedMap) > 0 {
+			tagValues := make([]string, 0)
+			for _, values := range mergedMap {
+				for _, value := range values {
+					tagValues = append(tagValues, value.Name)
+				}
 			}
+			f.SetCellValue("售后--样例数据", fmt.Sprintf("%s%d", "F", i+1), strings.Join(tagValues, ","))
+			log.Println(name, "----", strings.Join(tagValues, ","))
 		}
-		f.SetCellValue("招标项目清单", fmt.Sprintf("%s%d", "I", i+1), strings.Join(tagValues, ","))
-
-		log.Println(name, "----", strings.Join(tagValues, ","))
 	}
 	f.Save()
 	log.Println("over-------")
 }
 
+func TestXlsx2(t *testing.T) {
+	f, err := excelize.OpenFile("./招标项目-汇总(截至0807).xlsx")
+	//f, err := excelize.OpenFile("./无标签数据-大模型优化.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	//181 凭安库
+	MgoQY := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: "127.0.0.1:27001",
+		DbName:      "mixdata",
+		Size:        10,
+		UserName:    "",
+		Password:    "",
+		Direct:      true,
+	}
+	MgoQY.InitPool()
+
+	rows, err := f.GetRows("招标项目清单")
+	//rows, err := f.GetRows("Sheet7")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	for i := 1; i < len(rows); i++ {
+		if i%100 == 0 {
+			log.Println("current:", i, rows[i][4])
+		}
+		name := rows[i][4]
+		where := map[string]interface{}{
+			"company_name": name,
+		}
+		data, _ := MgoQY.FindOne("company_base", where)
+		if data != nil {
+			f.SetCellValue("招标项目清单", fmt.Sprintf("%s%d", "F", i+1), (*data)["company_type"])
+		}
+		data, _ = MgoQY.FindOne("special_gov_unit", where)
+		if data != nil {
+			f.SetCellValue("招标项目清单", fmt.Sprintf("%s%d", "F", i+1), (*data)["company_type"])
+		}
+	}
+	f.Save()
+	log.Println("结束了。。。。。。。")
+}
+
+////
+
 func TestZpAI(t *testing.T) {
 
 	// 本地
@@ -198,4 +262,357 @@ func TestAI1(t *testing.T) {
 	//model := "glm-4-air"
 	//name := "河南双汇投资发展股份有限公司"
 
+	text := "房地产开发、棚户区改造、廉租房、公租房建设投资及运营；基础设施及供热、供水、供气、垃圾治理、污水处理等项目投资；园林绿化等公用项目建设投资及运营。"
+
+	log.Println(extractMainBusiness(text))
+
+}
+
+func TestTidb(t *testing.T) {
+	db, err := connectToDB()
+	if err != nil {
+		panic(err)
+	}
+
+	categories, err := buildCategoryTree(db)
+	if err != nil {
+		panic(err)
+	}
+
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	for _, v := range categories {
+		MgoB.Save("wcc_top_categories2", structToMap(v))
+	}
+
+}
+
+// TestInsert 测试请求向量相似度，刘康佳代码
+func TestInsert(t *testing.T) {
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw").C("ent_info_buyer_912").Find(nil).Select(nil).Sort("_id").Limit(1000).Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		name := util.ObjToString(tmp["name"])
+		if count%100 == 0 {
+			log.Println("current:", count, name)
+		}
+		filter := "拓扑-国标行业"
+		url := "http://192.168.3.240:20612"
+		request := map[string]interface{}{
+			"text":   name,
+			"topk":   2,
+			"filter": filter,
+		}
+
+		result, err := postRequest(url, request)
+
+		if err != nil {
+			log.Println(err, name)
+			continue
+		}
+
+		id := mongodb.BsonIdToSId(tmp["_id"])
+
+		if len(result) > 0 {
+			if result2, ok := result["result"].([]interface{}); ok {
+				re := make([]interface{}, 0)
+				for _, v := range result2 {
+					if da, ok2 := v.(map[string]interface{}); ok2 {
+						if out, ok3 := da["output"].([]interface{}); ok3 {
+							for _, dda := range out {
+								if ddd, ok4 := dda.(map[string]interface{}); ok4 {
+									re = append(re, map[string]interface{}{
+										"score": da["score"],
+										"name":  ddd["name"],
+									})
+								}
+							}
+						}
+					}
+				}
+				//log.Println(insertDa)
+				update := map[string]interface{}{
+					"top_ddd": re,
+				}
+				MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+			}
+		}
+	}
+
+}
+
+// TestZhuYing 企业名称+主营业务，调用大模型
+func TestZhuYing(t *testing.T) {
+	key := "4d5206b1b297c1e7b77f9578edcb2cf7.TNU2i8G1oUNdR02i"
+	model := "glm-4-air"
+	//model := "glm-4-flash"
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw").C("ent_info_buyer_912").Find(nil).Select(nil).Sort("_id").Limit(200).Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+	ch := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		name := util.ObjToString(tmp["name"])
+		if count%10 == 0 {
+			log.Println("current:", count, name)
+		}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			//
+			business := util.ObjToString(tmp["businessScope"])
+			if utf8.RuneCountInString(business) > 50 {
+				r := []rune(business)
+				business = string(r[:50])
+			}
+			res := ZpAI4(key, model, name, business)
+			if len(res) > 0 {
+				id := mongodb.BsonIdToSId(tmp["_id"])
+				if resD, ok := res["result"].([]interface{}); ok {
+					labels := make([]interface{}, 0)
+					for _, resDa := range resD {
+						if v, ok := resDa.(map[string]interface{}); ok {
+							//log.Println(v)
+							label := ""
+							if util.ObjToString(v["label1"]) != "" {
+								label = util.ObjToString(v["label1"])
+							}
+							if util.ObjToString(v["label2"]) != "" {
+								label = label + "-" + util.ObjToString(v["label2"])
+							}
+							if util.ObjToString(v["label3"]) != "" {
+								label = label + "-" + util.ObjToString(v["label3"])
+							}
+							if util.ObjToString(v["label4"]) != "" {
+								label = label + "-" + util.ObjToString(v["label4"])
+							}
+							if len(label) > 0 && util.Float64All(v["score"]) > 0 {
+								tm := map[string]interface{}{
+									"name":  label,
+									"score": v["score"],
+								}
+								labels = append(labels, tm)
+							}
+						}
+					}
+					if len(labels) > 0 {
+						update := map[string]interface{}{
+							"ai_industry_09262": labels, //大模型通过名称和主营
+						}
+						MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+					}
+				}
+
+				//if resD, ok := res["result"].([]interface{}); ok {
+				//	labels := make([]interface{}, 0)
+				//	for _, resDa := range resD {
+				//		if v, ok := resDa.(map[string]interface{}); ok {
+				//			//log.Println(v)
+				//			if len(fmt.Sprintf("%v-%v-%v", v["label1"], v["label2"], v["label3"])) > 0 && util.Float64All(v["score"]) > 0 {
+				//				tm := map[string]interface{}{
+				//					"name":  fmt.Sprintf("%v-%v-%v", v["label1"], v["label2"], v["label3"]),
+				//					"score": v["score"],
+				//				}
+				//				labels = append(labels, tm)
+				//			}
+				//		}
+				//	}
+				//	if len(labels) > 0 {
+				//		id := mongodb.BsonIdToSId(tmp["_id"])
+				//		update := map[string]interface{}{
+				//			"top_ai_label09201": labels,
+				//		}
+				//		MgoB.UpdateById("ent_info_buyer_912", id, map[string]interface{}{"$set": update})
+				//	}
+				//}
+			}
+
+		}(tmp)
+		tmp = map[string]interface{}{}
+	}
+	wg.Wait()
+}
+
+// TestIndustry 获取凭安企业标签,半年项目数据
+func TestIndustry(t *testing.T) {
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+	//181 凭安库
+	MgoQY := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: "127.0.0.1:27001",
+		DbName:      "mixdata",
+		Size:        10,
+		UserName:    "",
+		Password:    "",
+		Direct:      true,
+	}
+	MgoQY.InitPool()
+
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw").C("wcc_project_0925").Find(nil).Select(nil).Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+	ch := make(chan bool, 30)
+	wg := &sync.WaitGroup{}
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count)
+		}
+
+		if _, ok := tmp["totalIndustry"]; ok {
+			continue
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			name := mongodb.BsonIdToSId(tmp["buyer"])
+			where := map[string]interface{}{
+				"company_name": name,
+			}
+			if name == "" {
+				return
+			}
+			data, _ := MgoQY.FindOne("company_base", where)
+			companyID := util.ObjToString((*data)["company_id"])
+
+			//1.公司ID不为空，直接使用凭安标签
+			if companyID != "" {
+				whereIndustry := map[string]interface{}{
+					"company_id": companyID,
+				}
+				industry, _ := MgoQY.FindOne("company_industry", whereIndustry)
+				if industry != nil && len(*industry) > 0 {
+					//totalIndustry := make([]interface{}, 0)
+					label := ""
+					if util.ObjToString((*industry)["industry_l1_name"]) != "" {
+						label = util.ObjToString((*industry)["industry_l1_name"])
+					}
+					//二级名称
+					if util.ObjToString((*industry)["industry_l2_name"]) != "" {
+						label = label + "-" + util.ObjToString((*industry)["industry_l2_name"])
+					}
+					//if util.ObjToString((*industry)["industry_l3_name"]) != "" {
+					//	label = label + "-" + util.ObjToString((*industry)["industry_l3_name"])
+					//}
+					//if util.ObjToString((*industry)["industry_l4_name"]) != "" {
+					//	label = label + "-" + util.ObjToString((*industry)["industry_l4_name"])
+					//}
+
+					//tmpD := map[string]interface{}{
+					//	"name":  label,
+					//	"score": 1.0,
+					//}
+					//totalIndustry = append(totalIndustry, tmpD)
+					update := map[string]interface{}{
+						"totalIndustry": label,
+						"industry_type": 1, //通过凭安
+					}
+					MgoB.UpdateById("wcc_project_0925", id, map[string]interface{}{"$set": update})
+				}
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("1234444444")
+
+}
+
+// postRequest
+func postRequest(url string, params map[string]interface{}) (map[string]interface{}, error) {
+	jsonData, err := json.Marshal(params)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var result map[string]interface{}
+	err = json.Unmarshal(body, &result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
 }

data_tags/tools.go

@@ -1,17 +1,23 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	zhipu "github.com/itcwc/go-zhipu/model_api"
 	"github.com/olivere/elastic/v7"
+	"io/ioutil"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"log"
 	"math"
+	"net/http"
+	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
+	"unicode/utf8"
 )
 
 var REG *regexp.Regexp
@@ -465,6 +471,105 @@ func ZpAI2(apiKey, model, text string) (rest map[string]interface{}) {
 	return
 }
 
+// ZpAI3 传参主营业务
+func ZpAI3(apiKey, model, name string, businessScope string) (rest map[string]interface{}) {
+	expireAtTime := int64(1719803252) // token 过期时间
+	sys := `请根据我给出的公司名称，依据其单位性质、单位职能和业务范围，准确给出最符合的二个国标行业分类标签，分别给出大类、中类和小类，并给出每一个的可靠性，可靠性用浮点数表示，比如:0.85，输出结果以JSON格式返回，格式如下:{
+  "result":[
+      {"label1":"制造业","label2":"金属制品业","label3":"金属结构制造","score":0.83},
+      {"label1":"制造业","label2":"金属制品业","label3":"工具制造","score":0.80}
+    ]
+}；返回结果按照score 降序排序。我只要最匹配的二个国标行业标签，只需要返回对应的中文。如果无法识别出类别，直接给我空字符串。按照以上要求输出，不要联想，不要无中生有，不要生成解释。`
+
+	text := fmt.Sprintf(sys, name)
+
+	if businessScope != "" {
+		text = text + "；主营业务是：" + businessScope
+	}
+	mssage := zhipu.PostParams{
+		Model: model,
+		Messages: []zhipu.Message{
+			{
+				Role:    "user", // 消息的角色信息 详见文档
+				Content: text,
+			},
+		},
+	}
+
+	postResponse, err := zhipu.BeCommonModel(expireAtTime, mssage, apiKey)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	if choices, ok := postResponse["choices"].([]interface{}); ok {
+		if choice, ok2 := choices[0].(map[string]interface{}); ok2 {
+			if message, ok3 := choice["message"].(map[string]interface{}); ok3 {
+				if content, ok4 := message["content"].(string); ok4 {
+					content = strings.ReplaceAll(content, "\n", "")
+					content = strings.ReplaceAll(content, "json", "")
+					content = strings.ReplaceAll(content, "`", "")
+					err = json.Unmarshal([]byte(content), &rest)
+					if err != nil {
+						log.Println("Unmarshal err", err, "content:", content)
+					}
+				}
+			}
+		}
+	}
+	return
+}
+
+// ZpAI4 传参主营业务
+func ZpAI4(apiKey, model, name string, businessScope string) (rest map[string]interface{}) {
+	expireAtTime := int64(1719803252) // token 过期时间
+	sys := `请根据我给出的公司名称，依据其单位性质、单位职能和业务范围，准确给出最符合的二个国标行业分类标签，分别给出一级标签、二级标签、三级标签和四级标签，并给出每一个的可靠性，可靠性用浮点数表示，比如:0.85，输出结果以JSON格式返回，格式如下:
+{
+  "result":[
+      {"label1":"制造业","label2":"通用设备制造业","label3":"通用零部件制造","label4":"机械零部件加工","score":0.83},
+      {"label1":"制造业","label2":"食品制造业","label3":"罐头食品制造","label4":"蔬菜、水果罐头制造","score":0.80}
+    ]
+}；返回结果按照score 降序排序。我只要最匹配的二个国标行业标签，只需要返回对应的中文。如果无法识别出类别，直接给我空字符串。按照以上要求输出，不要联想，不要无中生有，不要生成解释。`
+
+	text := fmt.Sprintf(sys, name)
+
+	if businessScope != "" {
+		text = text + "；主营业务是：" + businessScope
+	}
+	mssage := zhipu.PostParams{
+		Model: model,
+		Messages: []zhipu.Message{
+			{
+				Role:    "user", // 消息的角色信息 详见文档
+				Content: text,
+			},
+		},
+	}
+
+	postResponse, err := zhipu.BeCommonModel(expireAtTime, mssage, apiKey)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	if choices, ok := postResponse["choices"].([]interface{}); ok {
+		if choice, ok2 := choices[0].(map[string]interface{}); ok2 {
+			if message, ok3 := choice["message"].(map[string]interface{}); ok3 {
+				if content, ok4 := message["content"].(string); ok4 {
+					content = strings.ReplaceAll(content, "\n", "")
+					content = strings.ReplaceAll(content, "json", "")
+					content = strings.ReplaceAll(content, "`", "")
+					err = json.Unmarshal([]byte(content), &rest)
+					if err != nil {
+						log.Println("Unmarshal err", err, "content:", content)
+					}
+				}
+			}
+		}
+	}
+	return
+}
+
 func checkString(s string) bool {
 	for _, char := range s {
 		if ('a' <= char && char <= 'z') || ('A' <= char && char <= 'Z') || strings.ContainsAny(string(char), "12345678910") {
@@ -473,3 +578,226 @@ func checkString(s string) bool {
 	}
 	return false
 }
+
+// extractMainBusiness 根据营业范围提取主营业务
+func extractMainBusiness(business string) (res string) {
+	// 清除开头的 "许可项目：" 或 "一般项目："
+	if strings.HasPrefix(business, "许可项目：") {
+		business = strings.TrimPrefix(business, "许可项目：")
+	} else if strings.HasPrefix(business, "一般项目：") {
+		business = strings.TrimPrefix(business, "一般项目：")
+	}
+
+	if business == "" {
+		return ""
+	}
+
+	business = extractByDelimiter(business, "。", 1) + "。"
+	firstPunctuation := findFirstPunctuation(business)
+	switch firstPunctuation {
+	case "。":
+		res = extractByDelimiter(business, "。", 1)
+	case "、":
+		secondPunctuation := findSecondPunctuation(business)
+		if secondPunctuation == "、" {
+			if strings.Count(business, "，") > strings.Count(business, "；") {
+				res = extractByDelimiter(business, "，", 2)
+			} else {
+				res = extractByDelimiter(business, "；", 2)
+			}
+		} else if secondPunctuation == "；" {
+			res = extractByDelimiter(business, "；", 2)
+		} else if secondPunctuation == "。" {
+			res = extractByDelimiter(business, "。", 1)
+		} else {
+			res = extractByDelimiter(business, "，", 2)
+		}
+	case "，":
+		res = extractByDelimiter(business, "，", 2)
+	case "；":
+		res = extractByDelimiter(business, "；", 2)
+	default:
+		res = business
+	}
+	if utf8.RuneCountInString(res) > 50 {
+		r := []rune(res)
+		res = string(r[:50])
+	}
+	return
+}
+
+// extractByDelimiter 根据指定的分隔符和数量提取字符串
+func extractByDelimiter(s, delimiter string, count int) string {
+	parts := strings.Split(s, delimiter)
+	if len(parts) > count {
+		return strings.Join(parts[:count], delimiter)
+	}
+	return s
+}
+
+// findFirstPunctuation 查找第一个标点符号
+func findFirstPunctuation(s string) string {
+	for _, r := range s {
+		if strings.ContainsRune("。、，；", r) {
+			return string(r)
+		}
+	}
+	return ""
+}
+
+// findSecondPunctuation 查找第二个标点符号
+func findSecondPunctuation(s string) string {
+	firstPunct := findFirstPunctuation(s)
+	if firstPunct == "" {
+		return ""
+	}
+	return findFirstPunctuation(s[strings.Index(s, firstPunct)+1:])
+}
+
+// structToMap 结构体转map
+func structToMap(obj interface{}) map[string]interface{} {
+	result := make(map[string]interface{})
+	v := reflect.ValueOf(obj)
+	t := reflect.TypeOf(obj)
+
+	// Ensure the input is a struct
+	if t.Kind() == reflect.Ptr {
+		t = t.Elem()
+		v = v.Elem()
+	}
+
+	if t.Kind() != reflect.Struct {
+		return nil
+	}
+
+	for i := 0; i < t.NumField(); i++ {
+		field := t.Field(i)
+		value := v.Field(i)
+
+		// Use the JSON tag if available
+		tag := field.Tag.Get("json")
+		if tag == "" {
+			tag = field.Name
+		}
+
+		result[tag] = value.Interface()
+	}
+
+	return result
+}
+
+// HTTPRequest 封装 HTTP 请求并处理请求和响应为 map 类型
+func HTTPRequest(method, url string, headers map[string]string, body map[string]interface{}) (map[string]interface{}, error) {
+	// 创建 HTTP 客户端
+	client := &http.Client{
+		Timeout: 10 * time.Second, // 设置超时时间
+	}
+
+	// 将请求体转换为 JSON 格式
+	jsonBody, err := json.Marshal(body)
+	if err != nil {
+		return nil, fmt.Errorf("请求体序列化失败: %v", err)
+	}
+
+	// 创建 HTTP 请求
+	req, err := http.NewRequest(method, url, bytes.NewBuffer(jsonBody))
+	if err != nil {
+		return nil, fmt.Errorf("创建请求失败: %v", err)
+	}
+
+	// 设置请求头
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
+
+	// 发送请求
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("请求发送失败: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// 读取响应体
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("读取响应体失败: %v", err)
+	}
+
+	// 解析响应体为 map
+	var result map[string]interface{}
+	err = json.Unmarshal(respBody, &result)
+	if err != nil {
+		return nil, fmt.Errorf("解析响应体失败: %v", err)
+	}
+
+	// 返回响应的 map
+	return result, nil
+}
+
+// removeLastElement 去除最后一个元素
+func removeLastElement(s string) string {
+	parts := strings.Split(s, "-")
+	if len(parts) > 1 {
+		return strings.Join(parts[:len(parts)-1], "-")
+	}
+	return s
+}
+
+// intersectSlices 求字符串数组交集
+func intersectSlices(slice1, slice2 []string) []string {
+	m := make(map[string]bool)
+	result := []string{}
+
+	for _, item := range slice1 {
+		m[item] = true
+	}
+
+	for _, item := range slice2 {
+		if m[item] {
+			result = append(result, item)
+		}
+	}
+
+	return result
+}
+
+// findIntersection 针对拓扑标签，名称和营业范围求交集
+func findIntersection(data map[string]interface{}) []string {
+	topLabelsName, ok1 := data["top_labels_name"].([]interface{})
+	topLabelsBusinessScope, ok2 := data["top_labels_businessScope"].([]interface{})
+	if !ok1 || !ok2 {
+		return []string{}
+	}
+
+	if len(topLabelsName) == 1 || len(topLabelsBusinessScope) == 1 {
+		return []string{}
+	}
+
+	var processedName, processedScope []string
+	for _, name := range topLabelsName {
+		if nameStr, ok := name.(string); ok {
+			processedName = append(processedName, removeLastElement(nameStr))
+		}
+	}
+
+	for _, scope := range topLabelsBusinessScope {
+		if scopeStr, ok := scope.(string); ok {
+			processedScope = append(processedScope, scopeStr)
+		}
+	}
+
+	return intersectSlices(processedName, processedScope)
+}
+
+// removeDuplicates 去除重复字符串
+func removeDuplicates(arr []string) []string {
+	uniqueMap := make(map[string]bool)
+	var result []string
+	for _, str := range arr {
+		if !uniqueMap[str] {
+			uniqueMap[str] = true
+			result = append(result, str)
+		}
+	}
+	return result
+}

export_bidding/main.go

@@ -7,7 +7,9 @@ func main() {
 	//bankWinnerStatistic()
 	//getPengWinner()
 
-	dealXmData() //处理厦门数据
+	//dealXmData() //处理厦门数据
 	//CountBidamount()
+
+	exportXiaMenJsonFile2() //
 	log.Println("over ---- over")
 }

export_bidding/utils.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"fmt"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+)
+
+// GetJyURLByID 获取剑鱼地址
+func GetJyURLByID(id string) string {
+	var Url = "https://www.jianyu360.com/article/content/%s.html"
+	url := fmt.Sprintf(Url, util.CommonEncodeArticle("content", id))
+
+	return url
+}

export_bidding/xiamen.go

@@ -1,10 +1,16 @@
 package main
 
 import (
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
 	"go.mongodb.org/mongo-driver/bson"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
+	"math"
+	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -197,3 +203,795 @@ func CountBidamount() {
 
 	log.Println("total", total)
 }
+
+// exportXiaMenJson 导出厦门数据，JSON格式导出
+func exportXiaMenJson() {
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.31.31.202:27081,172.20.45.128:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "qfw",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+
+	MgoC := &mongodb.MongodbSim{
+		MongodbAddr: "172.31.31.202:27081,172.20.45.128:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "mixdata",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	MgoC.InitPool()
+
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 10) //处理协程
+	wg := &sync.WaitGroup{}
+
+	// 设置查询条件
+	filter := bson.D{
+		//{"publishtime", bson.M{"$gte": 1640966400, "$lt": 1672502400}},
+		{"publishtime", bson.M{"$gte": 1609430400}}, //2021-1-1
+		{"subtype", bson.M{"$in": []string{"中标", "成交", "合同", "单一"}}},
+	}
+
+	selected := map[string]interface{}{
+		"detail":        1,
+		"href":          1,
+		"title":         1,
+		"publishtime":   1,
+		"area":          1,
+		"bidopentime":   1,
+		"budget":        1,
+		"buyer":         1,
+		"bidamount":     1,
+		"buyertel":      1,
+		"buyerperson":   1,
+		"city":          1,
+		"subtype":       1,
+		"projectname":   1,
+		"projectcode":   1,
+		"projectscope":  1,
+		"agency":        1,
+		"s_winner":      1,
+		"winnerperson ": 1,
+		"winnertel ":    1,
+	}
+
+	it := sess.DB("qfw").C("bidding").Find(&filter).Select(&selected).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current :", count)
+		}
+
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+
+			// 创建一个新的map用于goroutine，避免重用
+			docCopy := make(map[string]interface{})
+			for k, v := range tmp {
+				docCopy[k] = v
+			}
+			docCopy["id"] = mongodb.BsonIdToSId(tmp["_id"])
+			docCopy["jybxhref"] = GetJyURLByID(mongodb.BsonIdToSId(tmp["_id"]))
+			//
+			if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
+				tmp = make(map[string]interface{})
+				return
+			}
+			// 针对存量数据，重复数据不进索引
+			if util.IntAll(tmp["extracttype"]) == -1 {
+				return
+			}
+
+			swinner := util.ObjToString(tmp["s_winner"])
+			if swinner == "" {
+				return
+			}
+
+			if utf8.RuneCountInString(swinner) < 4 {
+				return
+			}
+
+			if strings.Contains(swinner, ",") {
+				winner_credit_no_arr := make([]string, 0)
+				var personMap = make([]string, 0)
+				var phoneMap = make([]string, 0)
+				var emaolMap = make([]string, 0)
+				winners := strings.Split(swinner, ",")
+				for _, v := range winners {
+					if utf8.RuneCountInString(v) < 4 {
+						continue
+					} else {
+						da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": v})
+						winner_credit_no_arr = append(winner_credit_no_arr, util.ObjToString((*da)["credit_no"]))
+						//2.联系人，连线方式
+						if reports, ok := (*da)["annual_reports"]; ok {
+							if rs, ok := reports.([]interface{}); ok {
+								// 2. 初始化变量来保存最新的年份及对应的报告信息
+								var latestReport map[string]interface{}
+								latestYear := math.MinInt // 初始设置为最小值
+								// 3. 遍历年度报告
+								for _, report := range rs {
+									if r, ok := report.(map[string]interface{}); ok {
+										// 获取报告年份
+										if year, ok := r["report_year"].(float64); ok {
+											reportYear := int(year) // 转换为int
+											if reportYear > latestYear {
+												latestYear = reportYear
+												latestReport = r
+											}
+										}
+									}
+								}
+
+								// 4. 输出最新的报告信息
+								if latestReport != nil {
+									personMap = append(personMap, util.ObjToString(latestReport["operator_name"]))
+									phoneMap = append(phoneMap, util.ObjToString(latestReport["company_phone"]))
+									emaolMap = append(emaolMap, util.ObjToString(latestReport["company_email"]))
+									//fmt.Println("最新的报告信息：", latestReport)
+								}
+							}
+						}
+					}
+				}
+
+				docCopy["winner_credit_no"] = strings.Join(winner_credit_no_arr, ",")
+				docCopy["legal_person"] = strings.Join(personMap, ",")
+				docCopy["company_phone"] = strings.Join(phoneMap, ",")
+				docCopy["company_email"] = strings.Join(emaolMap, ",")
+				Mgo.Save("wcc_xiamen_winner_1221", docCopy)
+			} else {
+				da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": swinner})
+				docCopy["winner_credit_no"] = util.ObjToString((*da)["credit_no"])
+
+				// 1. 检查是否存在 "annual_reports" 字段，并且它是一个切片类型
+				if reports, ok := (*da)["annual_reports"]; ok {
+					if rs, ok := reports.([]interface{}); ok {
+						// 2. 初始化变量来保存最新的年份及对应的报告信息
+						var latestReport map[string]interface{}
+						latestYear := math.MinInt // 初始设置为最小值
+						// 3. 遍历年度报告
+						for _, report := range rs {
+							if r, ok := report.(map[string]interface{}); ok {
+								// 获取报告年份
+								if year, ok := r["report_year"].(float64); ok {
+									reportYear := int(year) // 转换为int
+									if reportYear > latestYear {
+										latestYear = reportYear
+										latestReport = r
+									}
+								}
+							}
+						}
+						// 4. 输出最新的报告信息
+						if latestReport != nil {
+							docCopy["legal_person"] = latestReport["operator_name"]
+							docCopy["company_phone"] = latestReport["company_phone"]
+							docCopy["company_email"] = latestReport["company_email"]
+							//fmt.Println("最新的报告信息：", latestReport)
+						}
+					}
+				}
+				//
+				Mgo.Save("wcc_xiamen_winner_1221", docCopy)
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+
+	wg.Wait()
+	log.Println("结束")
+}
+
+const MaxRecordsPerFile = 1000000 // 每个文件的最大记录数
+
+// exportXiaMenJsonFile 导出JSON文件
+func exportXiaMenJsonFile() {
+	// 配置文件夹路径
+	outputDir := "./output" // 你可以根据需要设置这个路径
+	// 创建文件夹，如果不存在的话
+	if _, err := os.Stat(outputDir); os.IsNotExist(err) {
+		err := os.MkdirAll(outputDir, os.ModePerm)
+		if err != nil {
+			log.Fatalf("无法创建文件夹: %v", err)
+		}
+	}
+
+	// 模拟数据和其他初始化
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.31.31.202:27081,172.20.45.128:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "qfw",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+
+	MgoC := &mongodb.MongodbSim{
+		MongodbAddr: "172.31.31.202:27081,172.20.45.128:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "mixdata",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	MgoC.InitPool()
+
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 10) // 处理协程
+	wg := &sync.WaitGroup{}
+
+	// 用于存储数据
+	var allRecords []map[string]interface{}
+	var mu sync.Mutex // 用于同步访问 allRecords
+
+	// 设置查询条件
+	filter := bson.D{
+		{"publishtime", bson.M{"$gte": 1609430400}},
+		{"subtype", bson.M{"$in": []string{"中标", "成交", "合同", "单一"}}},
+	}
+
+	selected := map[string]interface{}{
+		"detail":       1,
+		"href":         1,
+		"title":        1,
+		"publishtime":  1,
+		"area":         1,
+		"bidopentime":  1,
+		"budget":       1,
+		"buyer":        1,
+		"bidamount":    1,
+		"buyertel":     1,
+		"buyerperson":  1,
+		"city":         1,
+		"subtype":      1,
+		"projectname":  1,
+		"projectcode":  1,
+		"projectscope": 1,
+		"agency":       1,
+		"s_winner":     1,
+		"winnerperson": 1,
+		"winnertel":    1,
+		"extracttype":  1,
+		"sensitive":    1,
+	}
+
+	SE := util.SimpleEncrypt{Key: "topJYBX2019"}
+
+	curren := 0
+	it := sess.DB("qfw").C("bidding").Find(&filter).Select(&selected).Limit(101000).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current :", count, tmp["title"], tmp["_id"], curren)
+		}
+		// 过滤敏感数据
+		if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" {
+			continue
+		}
+
+		// 针对存量数据，重复数据不进索引
+		if util.IntAll(tmp["extracttype"]) == -1 {
+			continue
+		}
+
+		mu.Lock()
+		// 每当累积到 MaxRecordsPerFile 条数据时，保存文件并清空 allRecords
+		if len(allRecords) == MaxRecordsPerFile {
+			log.Println("len allRecords", len(allRecords), curren)
+			saveToFile2(outputDir, allRecords)
+			allRecords = nil // 清空数据
+		}
+		mu.Unlock()
+
+		// 创建一个新的map用于goroutine，避免重用
+		docCopy2 := make(map[string]interface{})
+		for k, v := range tmp {
+			docCopy2[k] = v
+		}
+		delete(docCopy2, "sensitive")
+		delete(docCopy2, "extracttype")
+		curren++
+
+		pool <- true
+		wg.Add(1)
+
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+
+			docCopy := make(map[string]interface{})
+			for k, v := range tmp {
+				docCopy[k] = v
+			}
+
+			docCopy["id"] = SE.EncodeString(mongodb.BsonIdToSId(tmp["_id"]))
+			docCopy["jybxhref"] = GetJyURLByID(mongodb.BsonIdToSId(tmp["_id"]))
+
+			// 合并数据
+			swinner := util.ObjToString(tmp["s_winner"])
+			if strings.Contains(swinner, ",") {
+				winner_credit_no_arr := make([]string, 0)
+				var personMap = make([]string, 0)
+				var phoneMap = make([]string, 0)
+				var emaolMap = make([]string, 0)
+				winners := strings.Split(swinner, ",")
+				for _, v := range winners {
+					if utf8.RuneCountInString(v) < 4 {
+						continue
+					} else {
+						da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": v})
+						winner_credit_no_arr = append(winner_credit_no_arr, util.ObjToString((*da)["credit_no"]))
+						personMap = append(personMap, util.ObjToString((*da)["legal_person"]))
+
+						//2.联系人，连线方式
+						if reports, ok := (*da)["annual_reports"]; ok {
+							if rs, ok := reports.([]interface{}); ok {
+								// 2. 初始化变量来保存最新的年份及对应的报告信息
+								var latestReport map[string]interface{}
+								latestYear := math.MinInt // 初始设置为最小值
+								// 3. 遍历年度报告
+								for _, report := range rs {
+									if r, ok := report.(map[string]interface{}); ok {
+										// 获取报告年份
+										if year, ok := r["report_year"].(float64); ok {
+											reportYear := int(year) // 转换为int
+											if reportYear > latestYear {
+												latestYear = reportYear
+												latestReport = r
+											}
+										}
+									}
+								}
+
+								// 4. 输出最新的报告信息
+								if latestReport != nil {
+									phoneMap = append(phoneMap, util.ObjToString(latestReport["company_phone"]))
+									emaolMap = append(emaolMap, util.ObjToString(latestReport["company_email"]))
+									//fmt.Println("最新的报告信息：", latestReport)
+								}
+							}
+						}
+					}
+				}
+
+				docCopy["winner_credit_no"] = strings.Join(winner_credit_no_arr, ",")
+				docCopy["legal_person"] = strings.Join(personMap, ",")
+				docCopy["company_phone"] = strings.Join(phoneMap, ",")
+				docCopy["company_email"] = strings.Join(emaolMap, ",")
+			} else {
+				da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": swinner})
+				docCopy["winner_credit_no"] = util.ObjToString((*da)["credit_no"])
+				docCopy["legal_person"] = util.ObjToString((*da)["legal_person"])
+				// 1. 检查是否存在 "annual_reports" 字段，并且它是一个切片类型
+				if reports, ok := (*da)["annual_reports"]; ok {
+					if rs, ok := reports.([]interface{}); ok {
+						// 2. 初始化变量来保存最新的年份及对应的报告信息
+						var latestReport map[string]interface{}
+						latestYear := math.MinInt // 初始设置为最小值
+						// 3. 遍历年度报告
+						for _, report := range rs {
+							if r, ok := report.(map[string]interface{}); ok {
+								// 获取报告年份
+								if year, ok := r["report_year"].(float64); ok {
+									reportYear := int(year) // 转换为int
+									if reportYear > latestYear {
+										latestYear = reportYear
+										latestReport = r
+									}
+								}
+							}
+						}
+						// 4. 输出最新的报告信息
+						if latestReport != nil {
+							docCopy["company_phone"] = latestReport["company_phone"]
+							docCopy["company_email"] = latestReport["company_email"]
+						}
+					}
+				}
+			}
+
+			// 加锁，确保并发安全地修改 allRecords
+			//Mgo.SaveByOriID("wcc_xiamen_winner_1227", docCopy)
+			mu.Lock()
+			delete(docCopy, "_id")
+			allRecords = append(allRecords, docCopy)
+
+			mu.Unlock()
+
+		}(docCopy2)
+		tmp = make(map[string]interface{})
+	}
+
+	// 等待所有协程处理完
+	wg.Wait()
+
+	// 如果还有剩余的数据，保存最后一个文件
+	if len(allRecords) > 0 {
+		log.Println("len allRecords", len(allRecords), curren)
+		saveToFile2(outputDir, allRecords)
+	}
+
+	log.Println("所有数据导出完成", curren)
+}
+
+// 将数据保存到文件
+func saveToFile2(outputDir string, records []map[string]interface{}) {
+	// 创建文件名，包含时间戳
+	fileName := fmt.Sprintf("%s_%s.json.gz", "xiaMenData", time.Now().Format("20060102_150405"))
+	filePath := filepath.Join(outputDir, fileName)
+
+	log.Println("开始写入文件：", fileName, "数据总量", len(records))
+	// 创建文件
+	file, err := os.Create(filePath)
+	if err != nil {
+		log.Fatalf("无法创建文件: %v", err)
+	}
+	defer file.Close()
+
+	// 使用gzip压缩
+	gzipWriter := gzip.NewWriter(file)
+	defer gzipWriter.Close()
+
+	// 创建 JSON 编码器
+	jsonEncoder := json.NewEncoder(gzipWriter)
+	jsonEncoder.SetIndent("", "  ")
+
+	// 批量写入数据，逐条处理并写入文件
+	batchSize := 10000 // 每次写入的数据条数
+	for i := 0; i < len(records); i += batchSize {
+		// 计算本次要写入的批次数据
+		end := i + batchSize
+		if end > len(records) {
+			end = len(records)
+		}
+
+		batch := records[i:end] // 获取当前批次数据
+
+		// 编码数据并写入压缩文件
+		if err := jsonEncoder.Encode(batch); err != nil {
+			log.Fatalf("无法写入 JSON 数据: %v", err)
+		}
+
+		// 打印进度，确保文件写入及时反馈
+		log.Printf("成功写入 %d/%d 条数据", end, len(records))
+	}
+
+	log.Printf("数据成功写入压缩文件: %s", filePath)
+}
+
+// exportXiaMenJsonFile 导出JSON文件
+func exportXiaMenJsonFile2() {
+	// 配置文件夹路径
+	var outputDir = "./output" // 你可以根据需要设置这个路径
+	// 创建文件夹，如果不存在的话
+	if _, err := os.Stat(outputDir); os.IsNotExist(err) {
+		err := os.MkdirAll(outputDir, os.ModePerm)
+		if err != nil {
+			log.Fatalf("无法创建文件夹: %v", err)
+		}
+	}
+
+	// 模拟数据和其他初始化
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.31.31.202:27081,172.20.45.128:27080",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+	}
+	Mgo.InitPool()
+
+	MgoC := &mongodb.MongodbSim{
+		MongodbAddr: "172.31.31.202:27081,172.20.45.128:27080",
+		DbName:      "mixdata",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+	}
+	MgoC.InitPool()
+
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 10) // 处理协程
+	wg := &sync.WaitGroup{}
+
+	// 用于存储数据
+	var allRecords []map[string]interface{}
+	var mu sync.Mutex // 用于同步访问 allRecords
+
+	// 设置查询条件
+	filter := bson.D{
+		{"publishtime", bson.M{"$gte": 1609430400}},
+		{"subtype", bson.M{"$in": []string{"中标", "成交", "合同", "单一"}}},
+	}
+
+	selected := map[string]interface{}{
+		"detail":       1,
+		"href":         1,
+		"title":        1,
+		"publishtime":  1,
+		"area":         1,
+		"bidopentime":  1,
+		"budget":       1,
+		"buyer":        1,
+		"bidamount":    1,
+		"buyertel":     1,
+		"buyerperson":  1,
+		"city":         1,
+		"subtype":      1,
+		"projectname":  1,
+		"projectcode":  1,
+		"projectscope": 1,
+		"agency":       1,
+		"s_winner":     1,
+		"winnerperson": 1,
+		"winnertel":    1,
+		"extracttype":  1,
+		"sensitive":    1,
+	}
+
+	SE := util.SimpleEncrypt{Key: "topJYBX2019"}
+
+	curren := 0
+	it := sess.DB("qfw").C("bidding").Find(&filter).Select(&selected).Iter()
+	count := 0
+
+	// 使用通道来传递记录
+	dataChannel := make(chan map[string]interface{}, 10000)
+
+	// 保存数据的协程
+	go func() {
+		for records := range dataChannel {
+			mu.Lock()
+			allRecords = append(allRecords, records)
+			// 每当累积到 MaxRecordsPerFile 条数据时，保存文件并清空 allRecords
+			if len(allRecords) >= MaxRecordsPerFile {
+				log.Println("保存文件，记录数：", len(allRecords))
+				saveToFile3(outputDir, allRecords)
+				allRecords = nil // 清空数据
+			}
+			mu.Unlock()
+		}
+
+		// 在通道关闭后，检查是否有剩余数据需要保存
+		mu.Lock()
+		if len(allRecords) > 0 {
+			log.Println("保存最后一批数据，记录数：", len(allRecords))
+			saveToFile3(outputDir, allRecords)
+			allRecords = nil // 清空数据
+		}
+		mu.Unlock()
+	}()
+
+	//读取数据库
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current :", count, tmp["title"], tmp["_id"], curren)
+		}
+		// 过滤敏感数据
+		if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" {
+			continue
+		}
+
+		// 针对存量数据，重复数据不进索引
+		if util.IntAll(tmp["extracttype"]) == -1 {
+			continue
+		}
+
+		// 创建一个新的map用于goroutine，避免重用
+		docCopy2 := make(map[string]interface{})
+		for k, v := range tmp {
+			docCopy2[k] = v
+		}
+		delete(docCopy2, "sensitive")
+		delete(docCopy2, "extracttype")
+		curren++
+
+		// 启动一个新的协程来处理每条数据
+		pool <- true
+		wg.Add(1)
+
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+
+			docCopy := make(map[string]interface{})
+			for k, v := range tmp {
+				docCopy[k] = v
+			}
+
+			docCopy["id"] = SE.EncodeString(mongodb.BsonIdToSId(tmp["_id"]))
+			docCopy["jybxhref"] = GetJyURLByID(mongodb.BsonIdToSId(tmp["_id"]))
+
+			// 合并数据
+			swinner := util.ObjToString(tmp["s_winner"])
+			if strings.Contains(swinner, ",") {
+				// 处理多个赢家的情况
+				winner_credit_no_arr := make([]string, 0)
+				var personMap = make([]string, 0)
+				var phoneMap = make([]string, 0)
+				var emaolMap = make([]string, 0)
+				winners := strings.Split(swinner, ",")
+				for _, v := range winners {
+					if utf8.RuneCountInString(v) < 4 {
+						continue
+					} else {
+						da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": v})
+						winner_credit_no_arr = append(winner_credit_no_arr, util.ObjToString((*da)["credit_no"]))
+						personMap = append(personMap, util.ObjToString((*da)["legal_person"]))
+
+						//2.联系人，连线方式
+						if reports, ok := (*da)["annual_reports"]; ok {
+							if rs, ok := reports.([]interface{}); ok {
+								// 2. 初始化变量来保存最新的年份及对应的报告信息
+								var latestReport map[string]interface{}
+								latestYear := math.MinInt // 初始设置为最小值
+								// 3. 遍历年度报告
+								for _, report := range rs {
+									if r, ok := report.(map[string]interface{}); ok {
+										// 获取报告年份
+										if year, ok := r["report_year"].(float64); ok {
+											reportYear := int(year) // 转换为int
+											if reportYear > latestYear {
+												latestYear = reportYear
+												latestReport = r
+											}
+										}
+									}
+								}
+
+								// 4. 输出最新的报告信息
+								if latestReport != nil {
+									phoneMap = append(phoneMap, util.ObjToString(latestReport["company_phone"]))
+									emaolMap = append(emaolMap, util.ObjToString(latestReport["company_email"]))
+								}
+							}
+						}
+					}
+				}
+
+				docCopy["winner_credit_no"] = strings.Join(winner_credit_no_arr, ",")
+				docCopy["legal_person"] = strings.Join(personMap, ",")
+				docCopy["company_phone"] = strings.Join(phoneMap, ",")
+				docCopy["company_email"] = strings.Join(emaolMap, ",")
+			} else {
+				da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": swinner})
+				docCopy["winner_credit_no"] = util.ObjToString((*da)["credit_no"])
+				docCopy["legal_person"] = util.ObjToString((*da)["legal_person"])
+				// 1. 检查是否存在 "annual_reports" 字段，并且它是一个切片类型
+				if reports, ok := (*da)["annual_reports"]; ok {
+					if rs, ok := reports.([]interface{}); ok {
+						// 2. 初始化变量来保存最新的年份及对应的报告信息
+						var latestReport map[string]interface{}
+						latestYear := math.MinInt // 初始设置为最小值
+						// 3. 遍历年度报告
+						for _, report := range rs {
+							if r, ok := report.(map[string]interface{}); ok {
+								// 获取报告年份
+								if year, ok := r["report_year"].(float64); ok {
+									reportYear := int(year) // 转换为int
+									if reportYear > latestYear {
+										latestYear = reportYear
+										latestReport = r
+									}
+								}
+							}
+						}
+						// 4. 输出最新的报告信息
+						if latestReport != nil {
+							docCopy["company_phone"] = latestReport["company_phone"]
+							docCopy["company_email"] = latestReport["company_email"]
+						}
+					}
+				}
+			}
+
+			// 通过channel发送数据
+			delete(docCopy, "_id")
+			dataChannel <- docCopy
+		}(docCopy2)
+
+		tmp = make(map[string]interface{})
+	}
+
+	// 等待所有协程处理完
+	wg.Wait()
+	// 关闭channel，确保保存线程能够结束
+	close(dataChannel)
+	log.Println("所有数据导出完成", curren)
+	select {}
+}
+
+func saveToFile3(outputDir string, records []map[string]interface{}) {
+	// 创建文件名，包含时间戳
+	fileName := fmt.Sprintf("%s_%s.json.gz", "xiaMenData", time.Now().Format("20060102_150405"))
+	filePath := filepath.Join(outputDir, fileName)
+
+	log.Println("开始写入文件：", fileName, "数据总量", len(records))
+
+	// 创建文件
+	file, err := os.Create(filePath)
+	if err != nil {
+		log.Fatalf("无法创建文件: %v", err)
+	}
+	defer file.Close()
+
+	// 使用gzip压缩
+	gzipWriter := gzip.NewWriter(file)
+	defer gzipWriter.Close()
+
+	// 创建 JSON 编码器
+	jsonEncoder := json.NewEncoder(gzipWriter)
+	// 不需要设置缩进，因为每条记录将是单独的 JSON 对象，通常不需要美化
+	// jsonEncoder.SetIndent("", "  ")
+
+	// 批量写入数据，逐条处理并写入文件
+	for k, record := range records {
+		// 编码每条记录并写入压缩文件，每条记录一行
+		if err := jsonEncoder.Encode(record); err != nil {
+			log.Fatalf("无法写入 JSON 数据: %v", err)
+		}
+		if k%1000 == 0 {
+			log.Println("成功写入文件行数：", k, fileName)
+		}
+	}
+
+	log.Printf("数据成功写入压缩文件: %s", filePath)
+}
+
+// 将数据保存到文件
+func saveToFile(outputDir string, records []map[string]interface{}) {
+	// 创建文件名，包含时间戳
+	fileName := fmt.Sprintf("%s_%s.json.gz", "xiaMenData", time.Now().Format("20060102_150405"))
+	filePath := filepath.Join(outputDir, fileName)
+
+	// 创建文件
+	file, err := os.Create(filePath)
+	if err != nil {
+		log.Fatalf("无法创建文件: %v", err)
+	}
+	defer file.Close()
+
+	// 使用gzip压缩
+	gzipWriter := gzip.NewWriter(file)
+	defer gzipWriter.Close()
+
+	// 创建 JSON 编码器
+	jsonEncoder := json.NewEncoder(gzipWriter)
+	jsonEncoder.SetIndent("", "  ")
+
+	// 编码数据
+	if err := jsonEncoder.Encode(records); err != nil {
+		log.Fatalf("无法写入 JSON 数据: %v", err)
+	}
+
+	log.Printf("数据成功写入压缩文件: %s", filePath)
+}

fix_qyxy_std/go.mod

@@ -0,0 +1,28 @@
+module fix_qyxy_std
+
+go 1.22.6
+
+require jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.10.1 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+)

fix_qyxy_std/go.sum

@@ -0,0 +1,198 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3 h1:mTokQIoOu/oZ2oCSAPayIFfnglIHP0qbOw1Ez6biKDo=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

fix_qyxy_std/main.go

@@ -0,0 +1,159 @@
+package main
+
+import (
+	"fmt"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	es "jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"time"
+)
+
+var (
+	//更新es
+	updateEsPool = make(chan []map[string]interface{}, 5000)
+	updateEsSp   = make(chan bool, 3) //保存协程
+	Es           *es.Elastic
+	esIndex      = "qyxy"
+	Mgo          *mongodb.MongodbSim
+	MgoQY        *mongodb.MongodbSim
+)
+
+func initEs() {
+	Es = &es.Elastic{
+		//S_esurl: "http://127.0.0.1:19908",
+		S_esurl:  "http://172.17.4.184:19908",
+		I_size:   10,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	Es.InitElasticSize()
+}
+
+// initMgo initMgo
+func initMgo() {
+	//181 凭安库
+	MgoQY = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.181:27001",
+		//MongodbAddr: "127.0.0.1:27001",
+		DbName:   "mixdata",
+		Size:     10,
+		UserName: "",
+		Password: "",
+		//Direct:      true,
+	}
+	MgoQY.InitPool()
+	//qyxy_std
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "mixdata",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+}
+
+func main() {
+	go updateEsMethod()
+	initEs()
+	initMgo()
+
+	dealCapitalData()
+
+	select {}
+}
+
+// dealCapitalData 处理数据 注册资金
+func dealCapitalData() {
+	defer util.Catch()
+	sess := MgoQY.GetMgoConn()
+	defer MgoQY.DestoryMongoConn(sess)
+	it := sess.DB("mixdata").C("company_base").Find(nil).Select(nil).Sort("_id").Iter()
+	fmt.Println("taskRun 开始")
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count, tmp["company_name"])
+		}
+		where := map[string]interface{}{
+			"company_name": tmp["company_name"],
+		}
+		update := make(map[string]interface{})
+		std, _ := Mgo.FindOne("qyxy_std", where)
+		companyType := util.ObjToString(tmp["company_type"])
+		if companyType == "事业单位" {
+			special, _ := MgoQY.FindOne("special_enterprise", where)
+			//1.事业单位数据，注册资金错误
+			if util.ObjToString((*special)["capital"]) != "" {
+				text := util.ObjToString((*special)["capital"])
+				capital := ObjToMoney(text)
+				capital = capital / 10000
+				update["capital"] = capital
+			}
+		} else {
+			//2.企业的capital =nil,需要更新为0
+			if util.ObjToString(tmp["capital"]) == "" {
+				update["capital"] = float64(0)
+			}
+		}
+
+		if _, ok := (*std)["capital"]; ok {
+			if len(update) > 0 {
+				//Mgo.Update(GF.Mongo.Coll, where, map[string]interface{}{"$set": update}, true, false)
+				Mgo.Update("qyxy_std", where, map[string]interface{}{"$set": update}, true, false)
+				////更新es
+				//err := Es.UpdateDocument(esIndex, util.ObjToString(tmp["company_id"]), update)
+				//if err != nil {
+				//	log.Println("err", err, update, where)
+				//}
+				updateEsPool <- []map[string]interface{}{
+					{"_id": util.ObjToString(tmp["company_id"])},
+					update,
+				}
+			}
+		}
+	}
+
+	log.Println("数据迭代结束")
+}
+
+// updateEsMethod 更新es
+func updateEsMethod() {
+	arru := make([][]map[string]interface{}, 200) //200条一组更新es
+	indexu := 0
+	for {
+		select {
+		case v := <-updateEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk(esIndex, arru...)
+
+				}(arru)
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk(esIndex, arru...)
+
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}

fix_qyxy_std/util.go

@@ -0,0 +1,288 @@
+package main
+
+import (
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	regNumFloat, _  = regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
+	regStrUnit, _   = regexp.Compile(`[元|万|亿]`)
+	contentUnit, _  = regexp.Compile(`(万元|单位/万)`)
+	numCapitals, _  = regexp.Compile(`([〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]{4,40})`)
+	regStrChar      = `[〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
+	moneyRegChar, _ = regexp.Compile(regStrChar)
+	regQianw, _     = regexp.Compile(`\d{1,2}千万`)
+
+	cutAllSpace, _ = regexp.Compile(`\s*`)
+	spaces         = []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n"}
+
+	moneyChar = map[string]interface{}{ //"〇": "0", "零": "0",
+		"一": float64(1), "壹": float64(1), "二": float64(2), "贰": float64(2), "三": float64(3), "叁": float64(3), "四": float64(4), "肆": float64(4), "五": float64(5), "伍": float64(5),
+		"六": float64(6), "陆": float64(6), "七": float64(7), "柒": float64(7), "八": float64(8), "捌": float64(8), "九": float64(9), "玖": float64(9), "十": float64(10), "拾": float64(10),
+		"百": float64(100), "佰": float64(100), "千": float64(1000), "仟": float64(1000), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000),
+		"零": float64(0), "点": ".", "角": float64(0.1), "分": float64(0.01),
+	}
+	moneyUnit = map[string]float64{
+		"元": float64(1), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000), //单位
+	}
+)
+
+var currencyItem = map[string]string{
+	"人民币": "人民币",
+	"rmb": "人民币",
+	"RMB": "人民币",
+	"$":   "美元",
+	"＄":   "美元",
+	"美元":  "美元",
+	"港元":  "港币",
+	"港币":  "港币",
+	"澳币":  "澳币",
+	"澳元":  "澳币",
+}
+
+// 获取币种
+func GetCurrency(text string) (currency string) {
+	if text == "" {
+		return
+	}
+	currency = "人民币"
+	for k, v := range currencyItem {
+		if strings.Contains(text, k) {
+			currency = v
+			return
+		}
+	}
+	return
+}
+
+// 金额转换
+func ObjToMoney(text string) float64 {
+	isfindUnit := true
+	ret := capitalMoney(text)
+	if ret < float64(10000) || ret > float64(50000000000) {
+		ret2, b := numMoney(text)
+		isfindUnit = b
+		if ret2 > ret {
+			ret = ret2
+		}
+	}
+	f, _ := strconv.ParseFloat(strconv.FormatFloat(ret, 'f', 4, 64), 64)
+	// if f < 1 {
+	// 	f = 0
+	// }
+	//如果金额小于50，全文检索单位：万
+	if f < 50 && f > 0 && isfindUnit {
+		rep := contentUnit.FindAllStringIndex(text, -1)
+		if len(rep) > 0 {
+			f = f * 10000
+		}
+	}
+	return f
+}
+
+func capitalMoney(text string) float64 {
+	nodes := []float64{}
+	node := float64(0)
+	tmp := float64(0)
+	decimals := 0.0
+	ishaspoint := false //是否含小数点
+	fnum := float64(0)
+	end := false
+	//str := fmt.Sprint(data[0])
+	//提取第一个大写信息
+	strmatch := numCapitals.FindAllStringSubmatch(text, -1)
+	if len(strmatch) > 0 {
+		text = strmatch[0][0]
+	}
+	suffixUnit := float64(1)
+	if strings.HasSuffix(text, "万") || strings.HasSuffix(text, "万元") || strings.HasSuffix(text, "万元整") {
+		index := strings.LastIndex(text, "万")
+		text = text[0:index]
+		suffixUnit = float64(10000)
+	}
+	moneyRegChar.ReplaceAllStringFunc(text, func(key string) string {
+		if key == "元" || key == "圆" || key == "点" {
+			ishaspoint = true
+		}
+		if v, ok := moneyChar[key].(float64); ok && !end {
+			if ishaspoint && v > 10 { //排除后面有其他的单位
+				return ""
+			}
+			//fmt.Println(key, v, fnum)
+			if v < 10 && v >= 0 {
+				if ishaspoint { //小数部分
+					if v >= 1 {
+						fnum = v
+					} else if v < 1 && v > 0 {
+						decimals += fnum * v
+					}
+				} else {
+					if tmp != float64(0) {
+						node += tmp
+					}
+					tmp = float64(v)
+				}
+			} else if v == 10000 || v == 100000000 { //单位万、亿
+				if tmp != float64(0) {
+					node += tmp
+					tmp = float64(0)
+				}
+				nodes = append(nodes, node*float64(v))
+				node = float64(0)
+			} else {
+				if v == 10 && tmp == 0 {
+					tmp = 1
+				}
+				tmp = tmp * float64(v)
+				node += tmp
+				tmp = float64(0)
+			}
+		}
+		if key == "整" || key == "正" || key == "分" {
+			end = true
+		}
+		return ""
+	})
+	nodes = append(nodes, node, tmp)
+	ret := float64(0)
+	for _, v := range nodes {
+		ret += v
+	}
+	return (ret + decimals) * suffixUnit
+}
+
+// 数字金额转换
+func numMoney(text string) (moneyFloat float64, flag bool) {
+	//tmp := fmt.Sprintf("%f", data[0])
+	repUnit := float64(1)
+	if regQianw.MatchString(text) {
+		text = strings.Replace(text, "千万", "万", -1)
+		repUnit = float64(1000)
+	}
+	text = replaceSymbol(text, []string{",", "，", "（", "）", "(", ")", "：", "\n"})
+	text = replaceString(text, []string{"万元", "亿元", "．"}, []string{"万", "亿", "."})
+	text = CutAllSpace(text)
+	rets := regNumFloat.FindAllString(text, -1)
+	fnums := []float64{}
+	unitstrs := []string{}
+	if len(rets) > 0 {
+		pindex := 0 //单位前置
+		for k, v := range rets {
+			f, err := strconv.ParseFloat(v, 64)
+			if err == nil {
+				fnums = append(fnums, f)
+				index := strings.Index(text, v)
+				//单位后置
+				start := index + len(v)
+				end := start + 3
+				//log.Println("vvv", tmp, v, pindex, index, start)
+				if k > 0 {
+					if start >= pindex+3 {
+						pstart := pindex + 3
+						if pstart >= index {
+							pstart = index
+						}
+						if len(text) > end {
+							unitstrs = append(unitstrs, text[pstart:index]+text[start:end])
+						} else {
+							unitstrs = append(unitstrs, text[pstart:index]+text[start:])
+						}
+					} else {
+						if len(text) > end {
+							unitstrs = append(unitstrs, text[start:end])
+						} else {
+							unitstrs = append(unitstrs, text[start:])
+						}
+					}
+				} else {
+					if len(text) > end {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, text[index-3:index]+text[start:end])
+						} else {
+							unitstrs = append(unitstrs, text[start:end])
+						}
+					} else {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, text[index-3:index]+text[start:])
+						} else {
+							unitstrs = append(unitstrs, text[start:])
+						}
+					}
+				}
+				pindex = start
+			}
+		}
+	}
+	//log.Println("unitstrs", fnums, unitstrs)
+	unit := float64(0)
+	fnum := float64(0)
+	for k, v := range fnums {
+		fnum = v
+		units := regStrUnit.FindAllString(unitstrs[k], -1)
+		for _, v := range units {
+			if moneyUnit[v] != 0 {
+				unit = moneyUnit[v]
+				break
+			}
+		}
+		if unit != float64(0) { //取第一个
+			break
+		}
+	}
+	fnum = fnum * repUnit
+	if unit == float64(0) {
+		moneyFloat = fnum
+	} else {
+		moneyFloat = fnum * unit
+	}
+	if unit == 10000 {
+		flag = false
+	} else {
+		flag = true
+	}
+	return
+}
+
+// 清理所有空白符
+func CutAllSpace(text string) string {
+	tmp := cutAllSpace.ReplaceAllString(text, "")
+	tmp = replaceSymbol(tmp, spaces)
+	return tmp
+}
+
+// 符号替换
+func replaceString(con string, ret, rep []string) string {
+	for k, v := range ret {
+		if len(rep) > k {
+			con = strings.Replace(con, v, rep[k], -1)
+		}
+	}
+	return con
+}
+
+// 过滤符号
+func replaceSymbol(con string, rep []string) string {
+	for _, v := range rep {
+		con = strings.Replace(con, v, "", -1)
+	}
+	return con
+}
+
+// IsInSlice 判断目标字符串是否是在切片中
+func IsInSlice(slice []string, s string) bool {
+	if len(slice) == 0 {
+		return false
+	}
+
+	isIn := false
+	for _, f := range slice {
+		if f == s {
+			isIn = true
+			break
+		}
+	}
+
+	return isIn
+}

getEs/bidding.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+)
+
+// updateHrefByEs 更新mgo by es
+func updateHrefByEs() {
+	//url := "http://172.17.4.184:19908"
+	//url := "http://127.0.0.1:19908"
+	url := "http://127.0.0.1:19905"
+	username := "jybid"
+	password := "Top2023_JEB01i@31"
+	//index := "bidding" //索引名称
+	index := "biddingall" //索引名称
+	// 创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(url),
+		elastic.SetBasicAuth(username, password),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+	where := map[string]interface{}{
+		"itype": 0,
+	}
+
+	it := sess.DB("qfw").C("bidding_es_update_id").Find(where).Select(nil).Iter()
+	log.Println("taskRun 开始")
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%100 == 0 {
+			log.Println("current:", count)
+		}
+
+		id := mongodb.BsonIdToSId(tmp["_id"])
+		res, _ := GetByID(client, index, id)
+		if res == nil {
+			//没有找到
+			update := map[string]interface{}{
+				"itype": 0,
+			}
+			MgoB.UpdateById("bidding_es_update_id", id, map[string]interface{}{"$set": update})
+
+		} else {
+			// 找到对应数据了
+			update := map[string]interface{}{
+				"href":  res["href"],
+				"itype": 1,
+			}
+			update2 := map[string]interface{}{
+				"href": res["href"],
+			}
+			MgoB.UpdateById("bidding_es_update_id", id, map[string]interface{}{"$set": update})
+			MgoB.UpdateById("bidding", id, map[string]interface{}{"$set": update2})
+		}
+
+	}
+
+	log.Println("over", count)
+}
+
+// GetByID 根据索引和 ID 获取数据
+func GetByID(ec *elastic.Client, index string, id string) (map[string]interface{}, error) {
+	ctx := context.Background()
+	// 执行 Get 请求
+	res, err := ec.Get().
+		Index(index).
+		Id(id).
+		Do(ctx)
+	if err != nil {
+		if elastic.IsNotFound(err) {
+			return nil, fmt.Errorf("document not found for index '%s' and id '%s'", index, id)
+		}
+		return nil, fmt.Errorf("failed to get document: %w", err)
+	}
+
+	// 解析文档源
+	if !res.Found {
+		return nil, fmt.Errorf("document not found for index '%s' and id '%s'", index, id)
+	}
+
+	// 解析 JSON 数据到 map[string]interface{}
+	var source map[string]interface{}
+	if err := json.Unmarshal(res.Source, &source); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal document source: %w", err)
+	}
+
+	return source, nil
+}

getEs/clickhouse.go

@@ -1,17 +1,24 @@
 package main
 
 import (
+	"context"
 	"fmt"
+	ckgo "github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	_ "github.com/gogf/gf/contrib/drivers/clickhouse/v2"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/os/gctx"
+	"github.com/xuri/excelize/v2"
 	"gorm.io/driver/clickhouse"
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
+	"net/url"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 )
@@ -287,3 +294,213 @@ func click2() {
 	db.Model(&result).Select("name", "age", "update_time").Updates(ExampleModel{Name: "wwwwww", Age: 333, UpdateTime: time.Now()})
 
 }
+
+// ClickhouseData  处理数据到Clickhouse
+func ClickhouseData() {
+	//正式环境
+	host := "cc-2ze9tv451wov14w9e.clickhouse.ads.aliyuncs.com:9000"
+	username := "biservice"
+	password := "Bi_top95215#"
+
+	//-----------------------------//
+	// 测试环境
+	//host := "192.168.3.207:19000"
+	//username := "jytop"
+	//password := "pwdTopJy123"
+	////
+
+	encodedPassword := url.QueryEscape(password)
+	dn := fmt.Sprintf("clickhouse://%s:%s@%s/pub_tags?dial_timeout=10s&read_timeout=20s", username, encodedPassword, host)
+
+	db, err := gorm.Open(clickhouse.Open(dn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		log.Fatal("打开数据库失败：", err)
+	} else {
+		log.Println("连接数据库成功", db.Name())
+	}
+
+	f, err := excelize.OpenFile("./主体代码表2.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		f.Save()
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+}
+
+// connectClickhouse 连接Clickhouse,其它方式连接;bitmap字段无法处理，需要使用下面方法
+func connectClickhouse(host, username, password, dbname string) (driver.Conn, error) {
+	//host := "192.168.3.207:19000"
+	//username := "jytop"
+	//password := "pwdTopJy123"
+	//dbname := "pub_tags"
+
+	var (
+		ctx       = context.Background()
+		conn, err = ckgo.Open(&ckgo.Options{
+			Addr:         []string{host},
+			DialTimeout:  10 * time.Second,
+			MaxIdleConns: 3,
+			MaxOpenConns: 30,
+			Auth: ckgo.Auth{
+				Database: dbname,
+				Username: username,
+				Password: password,
+			},
+			Debugf: func(format string, v ...interface{}) {
+				log.Println(format, v)
+			},
+		})
+	)
+	if err != nil {
+		return nil, err
+	}
+	if err := conn.Ping(ctx); err != nil {
+		if exception, ok := err.(*ckgo.Exception); ok {
+			log.Println("Exception [%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
+		}
+		return nil, err
+	}
+	return conn, nil
+}
+
+// dealClickhouse 处理法人库标签数组bitmap到Clickhouse
+func dealClickhouse() {
+	//正式环境
+	host := "cc-2ze9tv451wov14w9e.clickhouse.ads.aliyuncs.com:9000"
+	username := "biservice"
+	password := "Bi_top95215#"
+
+	// 测试环境
+	//host := "192.168.3.207:19000"
+	//username := "jytop"
+	//password := "pwdTopJy123"
+
+	//-----//
+
+	dbname := "pub_tags"
+	ClickHouseConn, _ := connectClickhouse(host, username, password, dbname)
+	//
+	sql := `
+	INSERT INTO pub_tags.dwd_d_tag (id, code, name, bitobj, groupid, created_at, created_by, bit_num)
+	VALUES (?, ?, ?, bitmapBuild(?), ?, ?, ?, ?)
+`
+	bitMapCodeArr := []uint64{}
+	if len(bitMapCodeArr) == 0 {
+		bitMapCodeArr = []uint64{uint64(0)}
+	}
+
+	f, err := excelize.OpenFile("./主体代码表2.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		f.Save()
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	//2.专项债详情
+	rows, err := f.GetRows("法人库标签")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for i := 1; i < len(rows); i++ {
+		if i%100 == 0 {
+			log.Println("iiiii", i, rows[i][1])
+		}
+		//插入数据
+		err = ClickHouseConn.Exec(context.Background(), sql,
+			util.Int64All(rows[i][3]),     // ID
+			"",                            // Code
+			strings.TrimSpace(rows[i][1]), // Name
+			bitMapCodeArr,                 // 初始化 bitobj 的 UInt64 值
+			util.Int64All(rows[i][0]),     // GroupID
+			time.Now(),                    // CreatedAt
+			"admin",                       // CreatedBy
+			util.Int64All(rows[i][2]),     // BitNum
+		)
+
+		if err != nil {
+			log.Println(err)
+		}
+	}
+
+	log.Println("数据处理完毕")
+}
+
+func testUpdateBitmap() {
+	// 测试环境
+	host := "192.168.3.207:19000"
+	username := "jytop"
+	password := "pwdTopJy123"
+	//dbname := "pub_tags"
+	dbname := "information"
+	ClickHouseConn, _ := connectClickhouse(host, username, password, dbname)
+
+	rows, err := ClickHouseConn.Query(context.Background(), "SELECT company_name,bitmapToArray(company_label) labels  FROM ent_info where company_name = '长春市建工建设监理有限公司'")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for rows.Next() {
+		var companyName string
+		var companyLabels = make([]uint64, 0)
+		var addLabels = make([]uint64, 0)
+		if err := rows.Scan(&companyName, &companyLabels); err != nil {
+			log.Println("eerrr", err)
+		}
+
+		addLabels = append(addLabels, uint64(13), uint64(14), uint64(15))
+		log.Println(companyName, companyLabels)
+
+		// 构建 toUInt64 数组字符串
+		toUInt64Array := buildToUInt64Array(addLabels)
+		// SQL 动态生成
+		sql := fmt.Sprintf(`
+		ALTER TABLE information.ent_info
+		UPDATE company_label = bitmapOr(company_label, bitmapBuild(%s))
+		WHERE company_name = ?
+	`, toUInt64Array)
+
+		log.Println(sql)
+		err = ClickHouseConn.Exec(context.Background(), sql, companyName)
+		if err != nil {
+			fmt.Printf("SQL execution failed: %v\n", err)
+		} else {
+			fmt.Println("Update succeeded!")
+		}
+
+	}
+	/**
+	alter table message_user_summary UPDATE readMsg = bitmapOr(readMsg,bitmapBuild([toUInt64(%d)])) where userId = '%s'
+	*/
+
+	//	sql := `
+	//alter table pub_tags.dwd_d_tag_wcc UPDATE bitobj = bitmapOr(readMsg,bitmapBuild(%v) where name = '通用仓储'
+	//`
+
+}
+
+// 动态构建 toUInt64 数组字符串
+func buildToUInt64Array(labels []uint64) string {
+	if len(labels) == 0 {
+		return "[]"
+	}
+	toUInt64Labels := make([]string, len(labels))
+	for i, label := range labels {
+		toUInt64Labels[i] = fmt.Sprintf("toUInt64(%d)", label)
+	}
+	return fmt.Sprintf("[%s]", strings.Join(toUInt64Labels, ", "))
+}

getEs/go.mod

@@ -1,12 +1,15 @@
 module getEs
 
-go 1.21.1
+go 1.22
 
 toolchain go1.22.6
 
 require (
+	github.com/ClickHouse/clickhouse-go/v2 v2.30.0
+	github.com/elastic/go-elasticsearch/v7 v7.17.10
 	github.com/gogf/gf/contrib/drivers/clickhouse/v2 v2.6.0
 	github.com/gogf/gf/v2 v2.6.0
+	github.com/itcwc/go-zhipu v0.0.0-20240626065325-ffc8bf1cfaaa
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/xuri/excelize/v2 v2.8.0
 	gorm.io/driver/clickhouse v0.6.0
@@ -17,32 +20,29 @@ require (
 
 require (
 	github.com/BurntSushi/toml v1.2.1 // indirect
-	github.com/ClickHouse/ch-go v0.58.2 // indirect
-	github.com/ClickHouse/clickhouse-go/v2 v2.15.0 // indirect
+	github.com/ClickHouse/ch-go v0.61.5 // indirect
 	github.com/PuerkitoBio/goquery v1.8.0 // indirect
-	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/clbanning/mxj/v2 v2.7.0 // indirect
 	github.com/dchest/captcha v1.0.0 // indirect
-	github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
-	github.com/go-faster/errors v0.6.1 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/uuid v1.3.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grokify/html-strip-tags-go v0.0.1 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
-	github.com/itcwc/go-zhipu v0.0.0-20240626065325-ffc8bf1cfaaa // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -51,14 +51,14 @@ require (
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/paulmach/orb v0.10.0 // indirect
-	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/paulmach/orb v0.11.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/richardlehane/mscfb v1.0.4 // indirect
 	github.com/richardlehane/msoleps v1.0.3 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
-	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
@@ -66,15 +66,15 @@ require (
 	github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	go.mongodb.org/mongo-driver v1.11.4 // indirect
-	go.opentelemetry.io/otel v1.19.0 // indirect
-	go.opentelemetry.io/otel/metric v1.19.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.16.0 // indirect
-	go.opentelemetry.io/otel/trace v1.19.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	go.opentelemetry.io/otel v1.32.0 // indirect
+	go.opentelemetry.io/otel/metric v1.32.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.32.0 // indirect
+	go.opentelemetry.io/otel/trace v1.32.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

getEs/go.sum

@@ -629,11 +629,13 @@ github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi
 github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0=
 github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw=
+github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4=
+github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg=
 github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
-github.com/ClickHouse/clickhouse-go/v2 v2.15.0 h1:G0hTKyO8fXXR1bGnZ0DY3vTG01xYfOGW76zgjg5tmC4=
 github.com/ClickHouse/clickhouse-go/v2 v2.15.0/go.mod h1:kXt1SRq0PIRa6aKZD7TnFnY9PQKmc2b13sHtOYcK6cQ=
+github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo=
+github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo=
 github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
@@ -691,8 +693,9 @@ github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY
 github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
 github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -1051,8 +1054,9 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
 github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
-github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI=
 github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
 github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g=
 github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks=
 github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
@@ -1080,8 +1084,11 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
 github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
@@ -1202,8 +1209,9 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0=
 github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -1236,8 +1244,9 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
@@ -1370,8 +1379,9 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
 github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -1586,8 +1596,9 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
 github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
-github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s=
 github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
+github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
 github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
@@ -1601,8 +1612,9 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk
 github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -1704,8 +1716,9 @@ github.com/shirou/gopsutil/v3 v3.23.8/go.mod h1:7hmCaBn+2ZwaZOr6jmPBZDfawwMGuo1i
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
-github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
@@ -1771,8 +1784,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -1831,6 +1845,8 @@ github.com/xuri/excelize/v2 v2.8.0 h1:Vd4Qy809fupgp1v7X+nCS/MioeQmYVVzi495UCTqB7
 github.com/xuri/excelize/v2 v2.8.0/go.mod h1:6iA2edBTKxKbZAa7X5bDhcCg51xdOn1Ar5sfoXRGrQg=
 github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a h1:Mw2VNrNNNjDtw68VsEj2+st+oCSn4Uz7vZw6TbhcV1o=
 github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yashtewari/glob-intersection v0.1.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
@@ -1898,8 +1914,11 @@ go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRM
 go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ=
 go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
 go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
-go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
 go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
+go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs=
+go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4=
+go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
+go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
 go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
 go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4=
 go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0/go.mod h1:M1hVZHNxcbkAlcvrOMlpQ4YOO3Awf+4N2dxkZL3xm04=
@@ -1922,8 +1941,11 @@ go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP
 go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A=
 go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s=
 go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4=
-go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
 go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
+go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
+go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
+go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M=
+go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8=
 go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
 go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
 go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI=
@@ -1931,8 +1953,11 @@ go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1t
 go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU=
 go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE=
 go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM=
-go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE=
 go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4=
+go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
+go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
+go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4=
+go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU=
 go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
 go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
 go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
@@ -1944,8 +1969,11 @@ go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaT
 go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM=
 go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
 go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
-go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
 go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
+go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA=
+go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0=
+go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
+go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg=
 go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ=
@@ -2002,8 +2030,9 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2152,8 +2181,9 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2199,8 +2229,9 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -2350,8 +2381,11 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -2385,8 +2419,9 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

getEs/main.go

@@ -27,7 +27,7 @@ func main() {
 
 	//getProject()
 	//getQyLimitData()
-	//getBiddingData()
+	getBiddingData()
 	//getQyxytData()
 	//getTidb()
 
@@ -38,7 +38,16 @@ func main() {
 	//mgoBidding()
 	//updateMgoEntInfoBuyer()
 
-	getZhiMa()
+	//getZhiMa()
+	//log.Println("over ------------------ over")
+
+	//fixProjectPortrait()
+	//
+	//ClickhouseData() //gorm 操作Clickhouse;gorm 对Clickhouse的bitmap兼容性不行，放弃
+	//dealClickhouse() //clickhouse-go 操作
+	//testUpdateBitmap() //测试环境测试更新Clickhouse bitmap字段
+	///-------//
+	//updateHrefByEs()
 	log.Println("over ------------------ over")
 }
 
@@ -85,11 +94,11 @@ func getBiddingData() {
 	//2023-10-1 2024-1-1;1696089600-1704038400
 	//areaTermsQuery := elastic.NewTermsQuery("area", "江苏", "安徽", "上海", "天津", "河北", "浙江", "天津市", "上海市", "河北省", "安徽省", "江苏省", "浙江省", "北京", "北京市")
 	//rangeQuery := elastic.NewRangeQuery("firsttime").Gte(1696089600).Lt(1704038400)
+	rangeQuery := elastic.NewRangeQuery("publishtime").Gte("1733760000").Lt("1734451200")
 	query := elastic.NewBoolQuery().
-		Must(elastic.NewTermQuery("toptype", "结果")).
-		Must(elastic.NewTermQuery("subtype", "招标"))
+		//Must(rangeQuery).
+		Must(elastic.NewTermQuery("site", "中国招标与采购网")).Must(rangeQuery)
 
-	//rangeQuery := elastic.NewRangeQuery("comeintime").Gte("1640966400").Lt("1703952000")
 	//query := elastic.NewBoolQuery().
 	//	//北京，天津，河北，上海，江苏，浙江，安徽
 	//	//Must(elastic.NewTermQuery("area", "北京市")).
@@ -138,7 +147,7 @@ func getBiddingData() {
 			delete(doc, "detail")
 
 			//存入新表
-			err = MgoB.InsertOrUpdate("qfw", "wcc_subtype_err_0429", doc)
+			err = MgoB.InsertOrUpdate("qfw", "wcc_bidding_test_1218", doc)
 			if err != nil {
 				log.Println("error", doc["id"])
 			}
@@ -984,6 +993,138 @@ func mgoBidding() {
 
 }
 
+// fixProjectPortrait 修复画像数据重复
+func fixProjectPortrait() {
+	url := "http://172.17.4.184:19908"
+	//url := "http://127.0.0.1:19908"
+	username := "jybid"
+	password := "Top2023_JEB01i@31"
+	index := "project_portrait" //索引名称
+	buyerMap := make(map[string]int)
+	buyerDatas := make(map[string][]map[string]interface{})
+	// 创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(url),
+		elastic.SetBasicAuth(username, password),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+
+	query := elastic.NewBoolQuery()
+	//query.Must(elastic.NewMatchQuery("business_scope", "招投标代理"))
+	query.Must(elastic.NewTermQuery("class", "情报_安防"))
+	ctx := context.Background()
+	//开始滚动搜索
+	scrollID := ""
+	scroll := "10m"
+	searchSource := elastic.NewSearchSource().
+		Query(query).
+		Size(10000).
+		//Sort("_doc", true) //升序排序
+		Sort("_doc", false) //降序排序
+
+	searchService := client.Scroll(index).
+		Size(10000).
+		Scroll(scroll).
+		SearchSource(searchSource)
+
+	res, err := searchService.Do(ctx)
+
+	if err != nil {
+		if err == io.EOF {
+			fmt.Println("没有数据")
+		} else {
+			panic(err)
+		}
+
+	}
+	//defer client.ClearScroll().ScrollId(scrollID).Do(ctx) // 在退出时清理资源
+	fmt.Println("project_portrait 总数是：", res.TotalHits())
+
+	MgoB := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "qfw",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	MgoB.InitPool()
+
+	//wher := map[string]interface{}{
+	//	"_id": mongodb.StringTOBsonId("66faf189bf905908d4a252d6"),
+	//}
+	//MgoB.Delete("project_portrait", wher)
+	//
+	//return
+
+	total := 0
+	for len(res.Hits.Hits) > 0 {
+		for _, hit := range res.Hits.Hits {
+			var doc map[string]interface{}
+			err := json.Unmarshal(hit.Source, &doc)
+			if err != nil {
+				log.Printf("解析文档失败：%s", err)
+				continue
+			}
+
+			buyerName := util.ObjToString(doc["buyer"])
+			buyerMap[buyerName]++
+			buyerArr := buyerDatas[buyerName]
+			buyerArr = append(buyerArr, doc)
+			buyerDatas[buyerName] = buyerArr
+		}
+
+		total = total + len(res.Hits.Hits)
+		scrollID = res.ScrollId
+		res, err = client.Scroll().ScrollId(scrollID).Scroll(scroll).Do(ctx)
+		log.Println("current count:", total)
+		if err != nil {
+			if err == io.EOF {
+				// 滚动到最后一批数据，退出循环
+				break
+			}
+			log.Println("滚动搜索失败：", err, res)
+			break // 处理错误时退出循环
+		}
+	}
+	// 在循环外调用 ClearScroll
+	_, err = client.ClearScroll().ScrollId(scrollID).Do(ctx)
+	if err != nil {
+		log.Printf("清理滚动搜索失败：%s", err)
+	}
+
+	fmt.Println("迭代结束~~~~~~~~~~~~~~~")
+
+	su := 0
+	for k, v := range buyerMap {
+		su++
+		if su%1000 == 0 {
+			log.Println("su", su)
+		}
+		if v > 1 {
+			buyerName := k
+			buyerArr := buyerDatas[buyerName]
+			doc := buyerArr[0]
+			doc["_id"] = mongodb.StringTOBsonId(util.ObjToString(doc["id"]))
+			MgoB.SaveByOriID("project_portrait_1030_test", doc)
+			for kk, vv := range buyerArr {
+				id := util.ObjToString(vv["id"])
+				where := map[string]interface{}{
+					"_id": mongodb.StringTOBsonId(util.ObjToString(doc["id"])),
+				}
+				MgoB.Delete("project_portrait", where)
+				if kk > 0 {
+					client.Delete().Index(index).Id(id).Do(context.Background())
+				}
+			}
+		}
+	}
+}
+
 // updateMgoEntInfoBuyer updateMgoEntInfoBuyer
 func updateMgoEntInfoBuyer() {
 	MgoB := &mongodb.MongodbSim{

mobile_tag/main.go

@@ -6,6 +6,7 @@ import (
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
+	"mobile_tag/oss"
 	"sync"
 	"time"
 )
@@ -26,6 +27,9 @@ var (
 )
 
 func main() {
+	testOss()
+
+	return
 	go updateEsMethod() // 更新es
 	go updateMethod()
 	Init()
@@ -37,6 +41,12 @@ func main() {
 	<-c
 }
 
+func testOss() {
+	oss.InitOss()
+	detail := oss.OssGetObject("5a862f0640d2d9bbe88e3cec", "jy-datadetail")
+	log.Println("detail:", detail)
+}
+
 // taskRun 执行
 func taskRun() {
 	defer util.Catch()

mobile_tag/oss/ossclient.go

@@ -3,7 +3,7 @@ package oss
 import (
 	"fmt"
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
-	"io/ioutil"
+	"io"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"os"
 	"strconv"
@@ -14,8 +14,9 @@ var (
 	//ossEndpoint        = "oss-cn-beijing.aliyuncs.com" //测试环境：oss-cn-beijing.aliyuncs.com
 	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
 	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
-	ossBucketName      = "topjy"
-	ossclient          *oss.Client
+	ossBucketName      = "jy-datadetail"
+	//ossBucketName      = "topjy"
+	ossclient *oss.Client
 )
 
 func InitOss() {
@@ -27,10 +28,10 @@ func InitOss() {
 	ossclient = client
 }
 
-func OssGetObject(objectName, id string) string {
+func OssGetObject(objectName, bucketName string) string {
 	util.Catch()
 	// 获取存储空间。
-	bucket, err := ossclient.Bucket(ossBucketName)
+	bucket, err := ossclient.Bucket(bucketName)
 	if err != nil {
 		return ""
 	}
@@ -38,14 +39,13 @@ func OssGetObject(objectName, id string) string {
 	// 下载文件到流。
 	body, err := bucket.GetObject(objectName)
 	if err != nil {
-		//log.Info("OssGetObject", zap.String("key", objectName), zap.String("id", id))
-		//log.Info("OssGetObject", zap.Error(err))
+		fmt.Println("Error:", err)
 		return ""
 	}
 	defer body.Close()
-	data, err := ioutil.ReadAll(body)
+	data, err := io.ReadAll(body)
 	if err != nil {
-		fmt.Println("ReadAll", err)
+		fmt.Println("Error:", err)
 		return ""
 	}
 	return string(data)

modify_bidamount/main.go

@@ -21,7 +21,8 @@ var (
 func Init() {
 	//mongodb
 	MgoB = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "172.17.189.140:27080",//老的集群
+		MongodbAddr: "172.20.45.128:27080", // 迁移后的新地址
 		//MongodbAddr: "127.0.0.1:27083",
 		DbName:   "qfw",
 		Size:     10,

project_portrait/main.go

@@ -26,7 +26,8 @@ var (
 	portraitMgo   = "project_portrait"    // MongoDB 的表名
 	GF            GlobalConf
 	// 情报分类一级标签
-	topInfos = []string{"情报_物业", "情报_环境采购", "情报_印务商机", "情报_家具招投标", "情报_车辆租赁"}
+	//topInfos = []string{"情报_物业", "情报_环境采购", "情报_印务商机", "情报_家具招投标", "情报_车辆租赁", "情报_安防"}
+	topInfos = []string{"情报_安防"}
 )
 
 type PortraitData struct {
@@ -112,21 +113,14 @@ func Init() {
 
 }
 func main() {
-	//Init()
-	Es = &es.Elastic{
-		S_esurl:  "http://192.168.3.149:9201",
-		I_size:   5,
-		Username: "",
-		Password: "",
-	}
-	Es.InitElasticSize()
+	Init()
 
 	go SaveEsMethod() // 生索引
 
 	//dealAllDataB()
 	allDataEs()
 
-	////定时任务
+	//定时任务
 	//local, _ := time.LoadLocation("Asia/Shanghai")
 	//c := cron.New(cron.WithLocation(local), cron.WithSeconds())
 	//_, err := c.AddFunc(GF.Env.Spec, dealIncData)
@@ -354,16 +348,16 @@ func dealAllData() {
 
 // dealAllDataB 处理存量数据，协程处理全部类型
 func dealAllDataB() {
-	//url := "http://172.17.4.184:19908"
-	//username := "jybid"
-	//password := "Top2023_JEB01i@31"
-	//index := "buyer"
-
-	url := "http://192.168.3.149:9201"
-	username := ""
-	password := ""
+	url := "http://172.17.4.184:19908"
+	username := "jybid"
+	password := "Top2023_JEB01i@31"
 	index := "buyer"
 
+	//url := "http://192.168.3.149:9201"
+	//username := ""
+	//password := ""
+	//index := "buyer"
+
 	// 创建 Elasticsearch 客户端
 	client, err := elastic.NewClient(
 		elastic.SetURL(url),
@@ -374,24 +368,24 @@ func dealAllDataB() {
 		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
 	}
 
-	//MgoB := &mongodb.MongodbSim{
-	//	MongodbAddr: "172.17.189.140:27080",
-	//	Size:        10,
-	//	DbName:      "qfw",
-	//	UserName:    "SJZY_RWbid_ES",
-	//	Password:    "SJZY@B4i4D5e6S",
-	//}
-	//MgoB.InitPool()
-
 	MgoB := &mongodb.MongodbSim{
-		MongodbAddr: "192.168.3.206:27002",
+		MongodbAddr: "172.17.189.140:27080",
 		Size:        10,
-		DbName:      "qfw_data",
-		UserName:    "root",
-		Password:    "root",
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
 	}
 	MgoB.InitPool()
 
+	//MgoB := &mongodb.MongodbSim{
+	//	MongodbAddr: "192.168.3.206:27002",
+	//	Size:        10,
+	//	DbName:      "qfw_data",
+	//	UserName:    "root",
+	//	Password:    "root",
+	//}
+	//MgoB.InitPool()
+
 	ctx := context.Background()
 	scroll := "10m"
 	searchSource := elastic.NewSearchSource().
@@ -510,7 +504,10 @@ func processHit(hit *elastic.SearchHit, client *elastic.Client, MgoB *mongodb.Mo
 			} else {
 				log.Println("Aggregation not found")
 			}
-			MgoB.Save("project_portrait", structToMap(portrait))
+
+			//insert := structToMap(portrait)
+			//insert[""]
+			MgoB.Save("project_portrait_0930", structToMap(portrait))
 		}
 	}
 }
@@ -617,32 +614,32 @@ func incDataEs() {
 
 // allDataEs 处理存量数据到es
 func allDataEs() {
-	//MgoB := &mongodb.MongodbSim{
-	//	MongodbAddr: "172.17.189.140:27080",
-	//	//MongodbAddr: "127.0.0.1:27083",
-	//	Size:     10,
-	//	DbName:   "qfw",
-	//	UserName: "SJZY_RWbid_ES",
-	//	Password: "SJZY@B4i4D5e6S",
-	//	//Direct:      true,
-	//}
-	//MgoB.InitPool()
-	// 测试环境
 	MgoB := &mongodb.MongodbSim{
-		MongodbAddr: "192.168.3.206:27002",
-		Size:        10,
-		DbName:      "qfw_data",
-		UserName:    "root",
-		Password:    "root",
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "qfw",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
 		//Direct:      true,
 	}
 	MgoB.InitPool()
+	// 测试环境
+	//MgoB := &mongodb.MongodbSim{
+	//	MongodbAddr: "192.168.3.206:27002",
+	//	Size:        10,
+	//	DbName:      "qfw_data",
+	//	UserName:    "root",
+	//	Password:    "root",
+	//	//Direct:      true,
+	//}
+	//MgoB.InitPool()
 
 	defer util.Catch()
 	sess := MgoB.GetMgoConn()
 	defer MgoB.DestoryMongoConn(sess)
 	count := 0
-	it := sess.DB(MgoB.DbName).C(portraitMgo).Find(nil).Select(nil).Iter()
+	it := sess.DB(MgoB.DbName).C("project_portrait_0930").Find(nil).Select(nil).Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
 		if count%5000 == 0 {
 			log.Println("current:", count, tmp["_id"])

updateBidding/bidding.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"fmt"
+	"github.com/wcc4869/common_utils/log"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+)
+
+// dealBiddingNiJian 更新bidding ，owner 不为空的赋值给buyer
+//if  toptype == "拟建"
+//        if tmp["owner"] != nil
+//                 tmp["buyer"]  =  tmp["owner"]
+
+func dealBiddingNiJian() {
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	where := map[string]interface{}{
+		"comeintime": map[string]interface{}{
+			"$lt": 1735228800,
+		},
+		"toptype": "拟建",
+	}
+
+	it := sess.DB("qfw").C("bidding").Find(where).Select(nil).Iter()
+
+	fmt.Println("taskRun 开始")
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%10000 == 0 {
+			log.Info("current", log.Int("count", count), log.Any("_id", tmp["_id"]))
+		}
+
+		if util.ObjToString(tmp["toptype"]) == "拟建" {
+			update := map[string]interface{}{}
+			esUpdate := map[string]interface{}{}
+
+			biddingID := mongodb.BsonIdToSId(tmp["_id"])
+			if tmp["owner"] != nil {
+				update["buyer"] = tmp["owner"]
+				esUpdate["buyer"] = tmp["owner"]
+			}
+
+			if len(update) > 0 {
+				//更新mongo
+				//MgoT.UpdateById("bidding", biddingID, map[string]interface{}{"$set": update})
+				//更新MongoDB
+				updatePool <- []map[string]interface{}{
+					{"_id": tmp["_id"]},
+					{"$set": update},
+				}
+
+				//2.es 项目 更新字段
+				//err := Es.UpdateDocument("bidding", biddingID, update)
+				//if err != nil && err.Error() != "Document not updated: noop" {
+				//	log.Info("bidding es update err", err, biddingID)
+				//}
+				//// 更新es
+				//updateEsPool <- []map[string]interface{}{
+				//	{"_id": biddingID},
+				//	update,
+				//}
+			}
+
+			// 更新Es 数据
+			if len(esUpdate) > 0 {
+				// 更新es
+				updateEsPool <- []map[string]interface{}{
+					{"_id": biddingID},
+					esUpdate,
+				}
+			}
+
+		}
+	}
+
+	log.Info("Run Over...Count:", log.Int("count", count))
+}

updateBidding/main.go

@@ -23,42 +23,44 @@ var (
 	MgoT     *mongodb.MongodbSim //测试环境链接
 	MgoR     *mongodb.MongodbSim
 	saveSize = 50
-	Es       *elastic.Elastic
-	EsNew    *elastic.Elastic
+	Es       *elastic.Elastic // 19908
+	EsNew    *elastic.Elastic //19905
 	//EsT      *elastic.Elastic
 
 	// 更新mongo
 	updatePool = make(chan []map[string]interface{}, 5000)
 	updateSp   = make(chan bool, 5)
 	//更新es
-	updateEsPool      = make(chan []map[string]interface{}, 5000)
-	updateEsSp        = make(chan bool, 5)                 //保存协程
-	BiddingField      = make(map[string]string, 200)       //bidding_processing_field, level=1 最外层字段，
-	BiddingLevelField = make(map[string]map[string]string) //level=2 的第二层字段
+	updateEsPool        = make(chan []map[string]interface{}, 5000)
+	updateEsSp          = make(chan bool, 5) //保存协程
+	updateProjectEsPool = make(chan []map[string]interface{}, 5000)
+	updateProjectEsSp   = make(chan bool, 5)                 //保存协程
+	BiddingField        = make(map[string]string, 200)       //bidding_processing_field, level=1 最外层字段，
+	BiddingLevelField   = make(map[string]map[string]string) //level=2 的第二层字段
 )
 
 func Init() {
-	//MgoB = &mongodb.MongodbSim{
-	//	MongodbAddr: "172.17.189.140:27080",
-	//	//MongodbAddr: "127.0.0.1:27083",
-	//	DbName:   "qfw",
-	//	Size:     10,
-	//	UserName: "SJZY_RWbid_ES",
-	//	Password: "SJZY@B4i4D5e6S",
-	//	//Direct:      true,
-	//}
-	//MgoB.InitPool()
-
-	MgoBAi = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.189.140:27080",
+	MgoB = &mongodb.MongodbSim{
+		MongodbAddr: "172.31.31.202:27081,172.20.45.128:27080",
 		//MongodbAddr: "127.0.0.1:27083",
-		DbName:   "qfw_ai",
+		DbName:   "qfw",
 		Size:     10,
 		UserName: "SJZY_RWbid_ES",
 		Password: "SJZY@B4i4D5e6S",
 		//Direct:      true,
 	}
-	MgoBAi.InitPool()
+	MgoB.InitPool()
+
+	//MgoBAi = &mongodb.MongodbSim{
+	//	//MongodbAddr: "172.17.189.140:27080",
+	//	MongodbAddr: "127.0.0.1:27083",
+	//	DbName:      "qfw_ai",
+	//	Size:        10,
+	//	UserName:    "SJZY_RWbid_ES",
+	//	Password:    "SJZY@B4i4D5e6S",
+	//	Direct:      true,
+	//}
+	//MgoBAi.InitPool()
 
 	//mongodb 163
 	//Mgo = &mongodb.MongodbSim{
@@ -115,32 +117,36 @@ func Init() {
 	Es.InitElasticSize()
 
 	//es 新集群
-	//EsNew = &elastic.Elastic{
-	//	//S_esurl: "http://127.0.0.1:19905",
-	//	S_esurl:  "http://172.17.4.184:19905",
-	//	I_size:   5,
-	//	Username: "jybid",
-	//	Password: "Top2023_JEB01i@31",
-	//}
-	//EsNew.InitElasticSize()
+	EsNew = &elastic.Elastic{
+		//S_esurl: "http://127.0.0.1:19905",
+		S_esurl:  "http://172.17.4.184:19905",
+		I_size:   5,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	EsNew.InitElasticSize()
 }
 
 func main() {
 	Init()
 	//InitEsBiddingField()
-	//go updateMethod() //更新mongodb
-	//go updateEsMethod() //更新es
+	go updateMethod()   //更新mongodb
+	go updateEsMethod() //更新es
+	//go updateEsHrefMethod() //更新es href 字段
 	//go updateProjectEsMethod()
 	//taskRunProject()
 	//taskRunBidding()
 	//dealBidding() //正式环境bidding数据处理
-	dealBiddingAi() //正式环境bidding数据处理
+	//dealBiddingAi() //正式环境bidding数据处理
 	//dealBiddingTest() // 测试环境数据处理
 
+	//dealBiddingEsHref() // 根据临时表，更新es href 字段
+	dealBiddingNiJian() //更新拟建数据中buyer = owner
+
 	//updateProject()
 	log.Info("over")
-	//c := make(chan bool, 1)
-	//<-c
+	c := make(chan bool, 1)
+	<-c
 }
 
 func InitEsBiddingField() {
@@ -646,7 +652,7 @@ func dealBiddingAi() {
 	defer util.Catch()
 	sess := MgoBAi.GetMgoConn()
 	defer MgoBAi.DestoryMongoConn(sess)
-	it := sess.DB("qfw_ai").C("zzzzz_kkk_uc_0907").Find(nil).Select(nil).Iter()
+	it := sess.DB("qfw_ai").C("zxl_20240926").Find(nil).Select(nil).Iter()
 
 	fmt.Println("taskRun 开始")
 	count := 0
@@ -657,17 +663,19 @@ func dealBiddingAi() {
 		biddingID := mongodb.BsonIdToSId(tmp["_id"])
 		update := map[string]interface{}{}
 
-		if budget, ok := tmp["budget"]; ok && budget != nil {
-			update["budget"] = budget
-		}
+		//if budget, ok := tmp["budget"]; ok && budget != nil {
+		//	update["budget"] = budget
+		//}
 
 		if bidamount, ok := tmp["bidamount"]; ok && bidamount != nil {
 			update["bidamount"] = bidamount
+		} else {
+			update["bidamount"] = 0.0
 		}
 
-		if projectcode, ok := tmp["projectcode"]; ok && projectcode != nil {
-			update["projectcode"] = projectcode
-		}
+		//if projectcode, ok := tmp["projectcode"]; ok && projectcode != nil {
+		//	update["projectcode"] = projectcode
+		//}
 
 		if len(update) > 0 {
 			MgoBAi.UpdateById("bidding", biddingID, map[string]interface{}{"$set": update})
@@ -968,8 +976,8 @@ func updateEsMethod() {
 	}
 }
 
-// updateProjectEsMethod 更新项目索引
-func updateProjectEsMethod() {
+// updateEsMethod 更新es href 字段
+func updateEsHrefMethod() {
 	arru := make([][]map[string]interface{}, 200)
 	indexu := 0
 	for {
@@ -983,7 +991,15 @@ func updateProjectEsMethod() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk("projectset", arru...)
+					Es.UpdateBulk("bidding", arru...)
+					Es.UpdateBulk("bidding_ai", arru...)
+					Es.UpdateBulk("bidding_temporary", arru...)
+					EsNew.UpdateBulk("bidding", arru...)
+					EsNew.UpdateBulk("bidding_customer", arru...)
+					EsNew.UpdateBulk("bidding_free", arru...)
+					EsNew.UpdateBulk("bidding_year", arru...)
+					EsNew.UpdateBulk("bidding_all", arru...)
+					EsNew.UpdateBulk("bidding_temporary", arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0
@@ -995,6 +1011,50 @@ func updateProjectEsMethod() {
 					defer func() {
 						<-updateEsSp
 					}()
+					Es.UpdateBulk("bidding", arru...)
+					Es.UpdateBulk("bidding_ai", arru...)
+					Es.UpdateBulk("bidding_temporary", arru...)
+					EsNew.UpdateBulk("bidding", arru...)
+					EsNew.UpdateBulk("bidding_customer", arru...)
+					EsNew.UpdateBulk("bidding_free", arru...)
+					EsNew.UpdateBulk("bidding_year", arru...)
+					EsNew.UpdateBulk("bidding_all", arru...)
+					EsNew.UpdateBulk("bidding_temporary", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}
+
+// updateProjectEsMethod 更新项目索引
+func updateProjectEsMethod() {
+	arru := make([][]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-updateProjectEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				updateProjectEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateProjectEsSp
+					}()
+					Es.UpdateBulk("projectset", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateProjectEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateProjectEsSp
+					}()
 					Es.UpdateBulk("projectset", arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 200)

updateBidding/tmp.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"fmt"
+	"github.com/wcc4869/common_utils/log"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+)
+
+// dealBiddingEs 根据临时表，更新es href 字段
+func dealBiddingEsHref() {
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw").C("bidding_es_20241129").Find(nil).Select(nil).Iter()
+
+	fmt.Println("taskRun 开始")
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%100 == 0 {
+			log.Info("current", log.Int("count", count), log.Any("_id", tmp["_id"]))
+		}
+		if util.IntAll(tmp["extracttype"]) == -1 {
+			continue
+		}
+
+		esUpdate := map[string]interface{}{}
+		biddingID := mongodb.BsonIdToSId(tmp["_id"])
+
+		esUpdate["href"] = tmp["href"]
+		// 更新Es 数据
+		if len(esUpdate) > 0 {
+			// 更新es
+			updateEsPool <- []map[string]interface{}{
+				{"_id": biddingID},
+				esUpdate,
+			}
+		}
+
+	}
+	log.Info("Run Over...Count:", log.Int("count", count))
+}

xlsx/liantong.go

@@ -0,0 +1,260 @@
+package main
+
+import (
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	"github.com/xuri/excelize/v2"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"strings"
+	"time"
+)
+
+// lianTongData1 判断联通单一数据，后续是否有中标、成交
+func lianTongData1() {
+	f, err := excelize.OpenFile("./2024.9.1-2024.12.1单一来源.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		f.Save()
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	//
+	rows, err := f.GetRows("详细数据")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	//85 项目数据
+	MgoP := &mongodb.MongodbSim{
+		MongodbAddr: "127.0.0.1:27080",
+		//MongodbAddr: "172.17.4.85:27080",
+		DbName: "qfw",
+		Size:   10,
+		Direct: true,
+	}
+	MgoP.InitPool()
+
+	//
+	se := util.SimpleEncrypt{Key: "topJYBX2019"}
+	for i := 1; i < len(rows); i++ {
+		if i%100 == 0 {
+			log.Println("current:iiiiiiiiiiii", i)
+		}
+		//id := rows[i][19]
+		id := se.DecodeString(strings.TrimSpace(rows[i][19]))
+		where := map[string]interface{}{
+			"ids": id,
+		}
+		p, _ := MgoP.FindOne("projectset_20230904", where)
+		project := *p
+		if list, ok := project["list"].([]interface{}); ok {
+			for _, v := range list {
+				if listData, ok := v.(map[string]interface{}); ok {
+					if util.ObjToString(listData["subtype"]) == "中标" {
+						f.SetCellValue("详细数据", fmt.Sprintf("%s%d", "U", i+1), "是")
+					}
+					if util.ObjToString(listData["subtype"]) == "成交" {
+						f.SetCellValue("详细数据", fmt.Sprintf("%s%d", "V", i+1), "是")
+					}
+				}
+			}
+		}
+
+		//if util.ObjToString(project["bidstatus"]) == "中标" {
+		//	f.SetCellValue("详细数据", fmt.Sprintf("%s%d", "U", i+1), "是")
+		//}
+		//if util.ObjToString(project["bidstatus"]) == "成交" {
+		//	f.SetCellValue("详细数据", fmt.Sprintf("%s%d", "V", i+1), "是")
+		//}
+		//if util.ObjToString(project["bidstatus"]) == "合同" {
+		//	f.SetCellValue("详细数据", fmt.Sprintf("%s%d", "W", i+1), "是")
+		//}
+
+	}
+	log.Println("数据处理完毕")
+}
+
+// lianTongShouLu1 判断联通数据是否收录,是否推送
+func lianTongShouLu1() {
+	//2024-12-23,赵慧磊 联通需求
+	f, err := excelize.OpenFile("./省分新增标讯数据复核_研发.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		f.Save()
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	//2.专项债详情
+	//rows, err := f.GetRows("基础数据")
+	rows, err := f.GetRows("Sheet2")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	//
+	//url := "http://172.17.4.184:19908"
+	url := "http://127.0.0.1:19908"
+	username := "jybid"
+	password := "Top2023_JEB01i@31"
+	//index := "bidding" //索引名称
+	// 创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(url),
+		elastic.SetBasicAuth(username, password),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+
+	// 86 mongo
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.4.86:27080",
+		MongodbAddr: "127.0.0.1:27082",
+		Size:        10,
+		DbName:      "jyqyfw",
+		//UserName: "SJZY_RWbid_ES",
+		//Password: "SJZY@B4i4D5e6S",
+		Direct: true,
+	}
+	Mgo.InitPool()
+
+	for i := 1; i < len(rows); i++ {
+		projectName := strings.TrimSpace(rows[i][5])
+		buyer := strings.TrimSpace(rows[i][4])
+		winner := strings.TrimSpace(rows[i][2])
+		money := strings.TrimSpace(rows[i][9])
+
+		projectCode := util.ObjToString(rows[i][6])
+		title := strings.TrimSpace(rows[i][7])
+		publishTime := strings.TrimSpace(rows[i][10])
+		layout := "2006-01-02"
+		t, err := time.Parse(layout, publishTime)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		timestamp := t.Unix()
+		rangeQuery := elastic.NewRangeQuery("publishtime").Gte(timestamp).Lt(timestamp + 24*3600)
+
+		if i%10 == 0 {
+			log.Println("iiiiii", i, projectName)
+			f.Save()
+		}
+
+		//判断1；标题+发布时间
+		query1 := elastic.NewBoolQuery().
+			Must(elastic.NewTermQuery("title.mtitle", title)).
+			Must(rangeQuery)
+
+		query3 := elastic.NewBoolQuery().
+			Must(elastic.NewTermQuery("projectcode", projectCode)).
+			Must(rangeQuery)
+
+		//1.1 判断es 是否存在
+		ex1, err, doc1 := checkBiddingData(client, query1)
+		if err != nil {
+			log.Println("checkBiddingData error", err)
+		}
+		//存在，已收录,判断是否推送
+		if ex1 {
+			f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "AA", i+1), "是")
+			f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "V", i+1), "是")
+			id := util.ObjToString(doc1["id"])
+			f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "X", i+1), id)
+			where2 := map[string]interface{}{
+				"id":    id,
+				"appid": "jyGQ1XQQsEAwNeSENOFR9D",
+			}
+
+			datas, _ := Mgo.Find("usermail", where2, nil, nil, false, -1, -1)
+			isTui := false
+			for _, dd := range *datas {
+				if util.IntAll(dd["isOptimization"]) == 1 && util.IntAll(dd["earliestDay"]) == 0 {
+					isTui = true
+				}
+			}
+			//确实推送了
+			if isTui {
+				f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "Z", i+1), "是")
+			}
+		}
+
+		//3.1 判断es 是否存在
+		ex3, err, doc3 := checkBiddingData(client, query3)
+		if err != nil {
+			log.Println("checkBiddingData error", err)
+		}
+		//存在，已收录,判断是否推送
+		if ex3 {
+			f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "AB", i+1), "是")
+			f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "V", i+1), "是")
+			id := util.ObjToString(doc3["id"])
+			f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "X", i+1), id)
+			where3 := map[string]interface{}{
+				"id":    id,
+				"appid": "jyGQ1XQQsEAwNeSENOFR9D",
+			}
+
+			datas, _ := Mgo.Find("usermail", where3, nil, nil, false, -1, -1)
+			isTui := false
+			for _, dd := range *datas {
+				if util.IntAll(dd["isOptimization"]) == 1 && util.IntAll(dd["earliestDay"]) == 0 {
+					isTui = true
+				}
+			}
+			//确实推送了
+			if isTui {
+				f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "Z", i+1), "是")
+			}
+		}
+
+		//判断2；projectname+s_winner+bidamount+buyer
+		query2 := elastic.NewBoolQuery().
+			Must(elastic.NewTermQuery("projectname", projectName)).
+			Must(elastic.NewTermQuery("s_winner", winner)).
+			Must(elastic.NewTermQuery("bidamount", money)).
+			Must(elastic.NewTermQuery("buyer", buyer))
+
+		//判断es 是否存在
+		ex, err, doc := checkBiddingData(client, query2)
+		if err != nil {
+			log.Println("checkBiddingData error", err)
+		}
+
+		//存在，已收录,判断是否推送
+		if ex {
+			f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "V", i+1), "是")
+			id := util.ObjToString(doc["id"])
+			f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "X", i+1), id)
+			where2 := map[string]interface{}{
+				"id":    id,
+				"appid": "jyGQ1XQQsEAwNeSENOFR9D",
+			}
+
+			datas, _ := Mgo.Find("usermail", where2, nil, nil, false, -1, -1)
+			isTui := false
+			for _, dd := range *datas {
+				if util.IntAll(dd["isOptimization"]) == 1 && util.IntAll(dd["earliestDay"]) == 0 {
+					isTui = true
+				}
+			}
+			//确实推送了
+			if isTui {
+				f.SetCellValue("Sheet2", fmt.Sprintf("%s%d", "Z", i+1), "是")
+			}
+		}
+	}
+}

xlsx/qyxy.go

@@ -254,7 +254,7 @@ func getAmountData(client *elastic.Client, index string, companyName string) (in
 
 // companyStatus 标记公司状态
 func companyStatus() {
-	f, err := excelize.OpenFile("./名单过滤.xlsx")
+	f, err := excelize.OpenFile("./惠聚打印List.xlsx")
 	if err != nil {
 		fmt.Println(err)
 		return
@@ -284,19 +284,85 @@ func companyStatus() {
 	MgoQY.InitPool()
 
 	for i := 1; i < len(rows); i++ {
-		name := rows[i][1]
+		name := rows[i][0]
 		if i%100 == 0 {
 			log.Println(i, name)
+			f.Save()
 		}
 
 		where2 := map[string]interface{}{
 			"company_name": name,
 		}
 		data, _ := MgoQY.FindOne("company_base", where2)
-		f.SetCellValue("Sheet1", fmt.Sprintf("D%v", i+1), (*data)["company_status"])
+		f.SetCellValue("Sheet1", fmt.Sprintf("B%v", i+1), (*data)["company_status"])
 	}
 
 	log.Println("2222222222222=======22222222222")
 	f.Save()
 
 }
+
+// getCompanyIndustry 获取凭安国标标签
+func getCompanyIndustry() {
+	//181 凭安库
+	MgoQY := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: "127.0.0.1:27001",
+		DbName:      "mixdata",
+		Size:        10,
+		UserName:    "",
+		Password:    "",
+		Direct:      true,
+	}
+	MgoQY.InitPool()
+
+	f, err := excelize.OpenFile("./业主打标签_企业.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("Sheet1")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	for i := 1; i < len(rows); i++ {
+		name := dealName(rows[i][0])
+		log.Println("iiii", i, name)
+		where := map[string]interface{}{
+			"company_name": name,
+		}
+		data, _ := MgoQY.FindOne("company_base", where)
+		companyID := util.ObjToString((*data)["company_id"])
+		if companyID != "" {
+			whereIndustry := map[string]interface{}{
+				"company_id": companyID,
+			}
+			industry, _ := MgoQY.FindOne("company_industry", whereIndustry)
+			if industry != nil && len(*industry) > 0 {
+				if util.ObjToString((*industry)["industry_l1_name"]) != "" {
+					f.SetCellValue("Sheet1", fmt.Sprintf("B%v", i+1), util.ObjToString((*industry)["industry_l1_name"]))
+				}
+				//二级名称
+				if util.ObjToString((*industry)["industry_l2_name"]) != "" {
+					f.SetCellValue("Sheet1", fmt.Sprintf("C%v", i+1), util.ObjToString((*industry)["industry_l2_name"]))
+				}
+				if util.ObjToString((*industry)["industry_l3_name"]) != "" {
+					f.SetCellValue("Sheet1", fmt.Sprintf("D%v", i+1), util.ObjToString((*industry)["industry_l3_name"]))
+				}
+				if util.ObjToString((*industry)["industry_l4_name"]) != "" {
+					f.SetCellValue("Sheet1", fmt.Sprintf("E%v", i+1), util.ObjToString((*industry)["industry_l4_name"]))
+				}
+			}
+		}
+	}
+
+	f.Save()
+	log.Println("数据处理完毕")
+}

xlsx/util.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"regexp"
+	"strings"
+)
+
+type TagMatching struct {
+	tagName       string        //标签名称
+	tagCode       string        //标签值(保存)
+	matchField    []string      //关键词匹配字段,title,detail
+	matchKey      string        //匹配的词语，多个使用逗号连接,"部队,国防,军事,军用"
+	matchKeyReg   []*RegexpInfo //关键词的正则
+	addField      []string      //附加词匹配字段
+	addKey        string        //附件词匹配关键词
+	addKeyReg     []*RegexpInfo //附加次的正则
+	excludeField  []string      //排除词匹配字段
+	excludeKey    string        //排除词匹配词
+	excludeKeyReg []*RegexpInfo //排除词正则
+	//clearKey      []string      //清理词匹配字段跟关键词一样
+}
+
+type RegexpInfo struct {
+	keyStr string
+	regs   *regexp.Regexp
+}
+
+// GetRegex 根据关键词或者对应正则
+func GetRegex(key string) []*RegexpInfo {
+	var infos []*RegexpInfo
+	for _, s := range strings.Split(key, ",") {
+		if strings.Contains(s, "&&") || strings.Contains(s, "&!") {
+			info := &RegexpInfo{
+				keyStr: s,
+				regs:   nil,
+			}
+			infos = append(infos, info)
+		} else {
+			info := &RegexpInfo{
+				keyStr: s,
+				regs:   regexp.MustCompile(".*(?i)" + s + ".*"),
+			}
+			infos = append(infos, info)
+		}
+
+	}
+	return infos
+}
+
+// TaskTags 根据数据和正则规则，验证数据标签
+func TaskTags(tmp map[string]interface{}, regs []TagMatching) (tags []string, keyWord, addWord, exWord string) {
+	// 在匹配字段里，比如标题满足了关键词，详情满足附加词,关键词的匹配字段含有标题，附加词的匹配字段含有详情；就符合条件
+Loop:
+	for _, v := range regs {
+		keyR := false // 关键词匹配结果
+		addR := false //附加次匹配结果
+		// 1.排除词
+		if len(v.excludeField) > 0 && len(v.excludeKeyReg) > 0 {
+			// 遍历排除词对应的tmp字段信息
+			for _, f := range v.excludeField {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					if rs, ex := getRegsResult(val, v.excludeKeyReg); rs {
+						//有排除词，直接判断下一个规则
+						exWord = ex
+						continue Loop
+					}
+				}
+			}
+		}
+
+		// 清理词；目的把 类似 fuck的单词替换为空字符串
+		//if len(v.clearKey) > 0 && len(v.matchField) > 0 {
+		//	for _, s := range v.clearKey {
+		//		for _, f := range v.matchField {
+		//			if val := util.ObjToString(tmp[f]); val != "" {
+		//				tmp[f] = strings.ReplaceAll(val, s, "")
+		//			}
+		//		}
+		//	}
+		//}
+
+		// 关键词
+		if len(v.matchField) > 0 && len(v.matchKeyReg) > 0 {
+			for _, f := range v.matchField {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					if rs, da := getRegsResult(val, v.matchKeyReg); rs {
+						keyR = true
+						keyWord = da
+						//log.Print("key", da)
+						break
+					}
+				}
+			}
+		}
+
+		// 附加词
+		if len(v.addField) > 0 && len(v.addKeyReg) > 0 && keyR {
+			for _, f := range v.addField {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					if rs, da := getRegsResult(val, v.addKeyReg); rs {
+						addR = true
+						addWord = da
+						//log.Println("add", da)
+						break
+					}
+				}
+			}
+		} else {
+			addR = true
+		}
+
+		// 满足 关键词和附加词条件
+		if keyR && addR {
+			// 去重相同标签
+			if !IsInStringArray(v.tagName, tags) {
+				tags = append(tags, v.tagName)
+			}
+		}
+	}
+
+	return
+}
+
+// getRegsResult 验证数据是否符合数组正则
+func getRegsResult(data string, regs []*RegexpInfo) (res bool, a string) {
+	for _, e1 := range regs {
+		if e1.regs != nil && e1.regs.MatchString(data) {
+			return true, e1.regs.String()
+		} else {
+			// && 特殊处理
+			if strings.Contains(e1.keyStr, "&&") {
+				flag := true
+				for _, s := range strings.Split(e1.keyStr, "&&") {
+					if !strings.Contains(data, s) {
+						flag = false
+						break
+					}
+				}
+				if flag {
+					return true, e1.keyStr
+				}
+			}
+			// 前面是必须有的关键词&!，后面是不能有的关键词;比如 军队&!点军队，
+			if strings.Contains(e1.keyStr, "&!") {
+				keys := strings.Split(e1.keyStr, "&!")
+				if strings.Contains(data, keys[0]) && !strings.Contains(data, keys[1]) {
+					return true, e1.keyStr
+				}
+			}
+		}
+	}
+	return false, ""
+}
+
+// dealName 处理名称中中英文括号
+func dealName(name string) string {
+	name = strings.Replace(name, "(", "（", -1)
+	name = strings.Replace(name, ")", "）", -1)
+	return name
+}

zhima/config.go

@@ -9,7 +9,9 @@ type BizContentForLabelQuery struct {
 
 // CreditLabel CreditLabel
 type CreditLabel struct {
-	ZhimaToptype string `json:"zhima_toptype"`
-	ZhimaSubtype string `json:"zhima_subtype"`
-	ZhimaName    string `json:"zhima_name"`
+	ZhimaToptype    string `json:"zhima_toptype"`
+	ZhimaSubtype    string `json:"zhima_subtype"`
+	ZhimaName       string `json:"zhima_name"`
+	ZhimaCode       string `json:"zhima_code"`
+	ZhimaDefinition string `json:"zhima_definition"`
 }

zhima/main.go

@@ -8,6 +8,7 @@ import (
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
+	"sync"
 )
 
 var (
@@ -46,26 +47,48 @@ func Init() {
 	}
 
 	//测试环境
+	//MgoT = &mongodb.MongodbSim{
+	//	MongodbAddr: "192.168.3.149:27102",
+	//	//MongodbAddr: "192.168.3.206:27002",
+	//	DbName:   "mixdata",
+	//	Size:     10,
+	//	UserName: "root",
+	//	Password: "root",
+	//	//Direct:      true,
+	//}
+	//MgoT.InitPool()
+	//
+	////测试环境es
+	//Es = &elastic.Elastic{
+	//	S_esurl: "http://192.168.3.149:9201",
+	//	//S_esurl:  "http://172.17.4.184:19805",
+	//	I_size:   10,
+	//	Username: "",
+	//	Password: "",
+	//}
+	//Es.InitElasticSize()
+
+	//正式环境
 	MgoT = &mongodb.MongodbSim{
-		MongodbAddr: "192.168.3.149:27102",
-		//MongodbAddr: "192.168.3.206:27002",
-		DbName:   "mixdata",
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
 		Size:     10,
-		UserName: "root",
-		Password: "root",
-		//Direct:      true,
+		DbName:   "mixdata",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:   true,
 	}
 	MgoT.InitPool()
 
-	//测试环境es
 	Es = &elastic.Elastic{
-		S_esurl: "http://192.168.3.149:9201",
-		//S_esurl:  "http://172.17.4.184:19805",
-		I_size:   5,
-		Username: "",
-		Password: "",
+		//S_esurl: "http://127.0.0.1:19908",
+		S_esurl:  "http://172.17.4.184:19908",
+		I_size:   10,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
 	}
 	Es.InitElasticSize()
+
 }
 
 func main() {
@@ -84,66 +107,55 @@ func main() {
 
 	fmt.Println("taskRun 开始")
 	count := 0
+	ch := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
 		if count%1000 == 0 {
-			log.Println("current", count, tmp["company_name"], tmp["credit_no"])
+			log.Println("current", count, tmp["company_name"], tmp["credit_no"], tmp["_id"])
 		}
+		// 存在芝麻标签，直接跳过
+		//if _, ok := tmp["zhima_labels"]; ok {
+		//	continue
+		//}
 
-		id := mongodb.BsonIdToSId(tmp["_id"])
-		epCertNo := util.ObjToString(tmp["credit_no"])
-		createLabels := getCreditLabel(appID, privateKeyPEM, epCertNo, sceneCode)
-		if len(createLabels) > 0 {
-			update := map[string]interface{}{
-				"zhima_labels": convertToInterfaceSlice(createLabels),
-			}
-			where := map[string]interface{}{
-				"_id": tmp["_id"],
-			}
-			MgoT.Update("qyxy_std", where, map[string]interface{}{"$set": update}, true, false)
-			//MgoT.UpdateById("qyxy_std", id, map[string]interface{}{"$set": update})
-			err := Es.UpdateDocument("qyxy", id, update)
-			if err != nil && err.Error() != "Document not updated: noop" {
-				log.Println("qyxy es update err", err, id)
-			}
+		if util.ObjToString(tmp["credit_no"]) == "" {
+			continue
 		}
 
-	}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			//
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			epCertNo := util.ObjToString(tmp["credit_no"])
+			createLabels := getCreditLabel(appID, privateKeyPEM, epCertNo, sceneCode)
+			if len(createLabels) > 0 {
+				updateEs := map[string]interface{}{
+					"zhima_labels": convertToInterfaceSliceEs(createLabels),
+				}
 
-	//bizbyte, _ := json.Marshal(BizContent)
-	//priKey, err := parsePrivateKey(privateKeyPEM)
-	//if err != nil {
-	//	log.Fatalln("parsePrivateKey err", err)
-	//}
-	//
-	//resMap, err := DoRequest(appID, string(bizbyte), curl, method, priKey)
-	//if err != nil {
-	//	log.Fatalln("request err", err)
-	//}
-	//responseData := util.ObjToMap((*resMap)["zhima_credit_ep_acceptance_label_query_response"])
-	//if util.ObjToString(((*responseData)["code"])) == "10000" && util.ObjToString(((*responseData)["msg"])) == "Success" {
-	//	if (*responseData)["label_content"] == nil {
-	//		log.Println("请求出错，没有返回企业标签内容模型", BizContent)
-	//	} else {
-	//		if labelContent, ok := (*responseData)["label_content"].([]interface{}); ok {
-	//			for _, v := range labelContent {
-	//				log.Println(v)
-	//				label := util.ObjToMap(v)
-	//				labelName := util.ObjToString((*label)["label_name"])
-	//				labelCategory := util.ObjToString((*label)["label_category"])
-	//				createLabel := CreditLabel{
-	//					LabelTop:  labelCategory,
-	//					LabelName: labelName,
-	//				}
-	//
-	//				createLabels = append(createLabels, createLabel)
-	//			}
-	//		}
-	//
-	//	}
-	//} else {
-	//	log.Println("请求失败", BizContent)
-	//}
+				updateMgo := map[string]interface{}{
+					"zhima_labels": convertToInterfaceSliceMgo(createLabels),
+				}
+				where := map[string]interface{}{
+					"_id": tmp["_id"],
+				}
+				MgoT.Update("qyxy_std", where, map[string]interface{}{"$set": updateMgo}, true, false)
+				err := Es.UpdateDocument("qyxy", id, updateEs)
+				if err != nil && err.Error() != "Document not updated: noop" {
+					log.Println("qyxy es update err", err, id)
+				}
+			}
 
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Println("企业芝麻标签结束")
 }
 
 // getCreditLabel 获取芝麻企业信用企业标签
@@ -171,15 +183,19 @@ func getCreditLabel(appID, privateKeyPEM, certNo, sceneCode string) (createLabel
 	responseData := util.ObjToMap((*resMap)["zhima_credit_ep_acceptance_label_query_response"])
 	if util.ObjToString(((*responseData)["code"])) == "10000" && util.ObjToString(((*responseData)["msg"])) == "Success" {
 		if (*responseData)["label_content"] == nil {
-			log.Println("请求出错，没有返回企业标签内容模型", BizContent)
+			//log.Println("请求出错，没有返回企业标签内容模型", BizContent)
 		} else {
 			if labelContent, ok := (*responseData)["label_content"].([]interface{}); ok {
 				for _, v := range labelContent {
 					label := util.ObjToMap(v)
 					labelName := util.ObjToString((*label)["label_name"])
 					labelCategory := util.ObjToString((*label)["label_category"])
+					labelCode := util.ObjToString((*label)["label_code"])
+					labelDe := util.ObjToString((*label)["label_definition"])
 
 					data := getLabel(labelCategory, labelName)
+					data.ZhimaCode = labelCode
+					data.ZhimaDefinition = labelDe
 					if data.ZhimaSubtype != "" {
 						createLabels = append(createLabels, data)
 					}
@@ -187,7 +203,7 @@ func getCreditLabel(appID, privateKeyPEM, certNo, sceneCode string) (createLabel
 			}
 		}
 	} else {
-		log.Println("请求失败", BizContent)
+		log.Println("请求失败", BizContent, (*responseData))
 	}
 
 	return
@@ -204,13 +220,28 @@ func getLabel(labelTop, labelName string) (data CreditLabel) {
 	return
 }
 
-func convertToInterfaceSlice(labels []CreditLabel) []map[string]interface{} {
+func convertToInterfaceSliceEs(labels []CreditLabel) []map[string]interface{} {
+	result := make([]map[string]interface{}, len(labels))
+	for i, label := range labels {
+		result[i] = map[string]interface{}{
+			"zhima_toptype":    label.ZhimaToptype,
+			"zhima_subtype":    label.ZhimaSubtype,
+			"zhima_name":       label.ZhimaName,
+			"zhima_definition": label.ZhimaDefinition,
+		}
+	}
+	return result
+}
+
+func convertToInterfaceSliceMgo(labels []CreditLabel) []map[string]interface{} {
 	result := make([]map[string]interface{}, len(labels))
 	for i, label := range labels {
 		result[i] = map[string]interface{}{
-			"zhima_toptype": label.ZhimaToptype,
-			"zhima_subtype": label.ZhimaSubtype,
-			"zhima_name":    label.ZhimaName,
+			"zhima_toptype":    label.ZhimaToptype,
+			"zhima_subtype":    label.ZhimaSubtype,
+			"zhima_name":       label.ZhimaName,
+			"zhima_code":       label.ZhimaCode,
+			"zhima_definition": label.ZhimaDefinition,
 		}
 	}
 	return result

zhima/zhima_test.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"log"
+	"testing"
+)
+
+func TestGetCreditLabel(T *testing.T) {
+
+	appID := "2021003111633206"
+	privateKeyPEM := "-----BEGIN RSA PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCaJnmz3N1Z43jxtmZJmPkInl3FbfudxjicThHj+Hkndp+muzZXlAPcWoYTFLAfVa6xg9xAc58fWTPUGOpouh0ZDEP8TGvD784AljDNwRXPI9+afjcxN8gzsNPlqA6KVkAu5u5uEI1HHQgtdyRoCEnXgQRXj6DqAs9olbx8HrTxHjZzjPd+HXv+v4na3zAvZsvPdhLB2ggpVW6uQN7ur8C+hVpB3ZO+gGVFHy3m1t08XgscljyOA9Vpk4T8TnKZqDvnI6ehWFCGeD11yoe3ipHJJdP/+u9/tXiKhm29mJvcM9C6gab6BOQABByQPb2nPFFyfPglbRYrYxgKp/eXUQTzAgMBAAECggEAO6BCVyZJoKlycNzC+Bjpy0Gmt+Vw/HheEWfJV8sW4gLqgk3MGVgxnhyN2/eWoKg96IA+w9d1Z2TTvWjd0yCafMPX0FGnJaUuo990hUH4s6UGqvXDV5BiKkxtylmVNkFilkHRXFWesDE0ppF9TZhXJ7rJ+Syfa6UmoSe6PrCKAPG4FD8MH1ODj5b42l/TRIFM948nTtTLFAFgd77P5ULBCKVfBB5V9XdKhLRdwG9RxugWwkxuCiMOtmk45DzA/PQgvkIzEViD/2NJX67/S88B0G63zdQizJrvxsLp9fThmjhlo/+CaF+a+nfIzg81TfQHlRuTjiYc7iE9n5K0sU82wQKBgQDydI9pEjkPH+ZEixEymdx8D4workBfa+GXgobJ2UwatqCJOCYqZJR0MhjTVeDJiiTGFiemNyZHT0WFQedAQ/O9mkRf0MRhzR6ZOiMnCG5pSSOKdmQXoqjOqcMzdgWBQRol0eLB/vto8M3bpt1VautR0qQOR7BrFsj57K3asofikQKBgQCiwwie5gyCAYB1gkp4QkLmQDSTJdZaTlzzx2oa2/ZB4Pg6wWWNJIxEfSwtuGVH4jGJl0Eer2YpHzaY7f2NfOWJCpNI/aRhM/RilQsFXTox/5bqlbAfspPMqSHnry2OgbZb9CRF3jH7kt65390SKde4A4JNE+jJ0Xb07Tzcr7+pQwKBgCDJ8KcWBs9r0flNEVtVgYLpm1vZDO2aqlySE2Zy0579OcK07Dxpc8YvDDpwb8DxxL1hO6V36d5U8yLtYbfIOMtYdIy8sAJF+jBgs2ZQ9pWYhYzxK+IiZk/kl3h7Ko1rX3zj4JW5xQhRIJXnACHQHtjm8rbXrR0ebLmX738XC9dxAoGAQfUDQ1E6dFyjb/syrnU7ow8VvyOwL7xd0rRW/TdyHdRLA5Rn0MCkzwRiqMx2zqCNLpLK4uJ1ZeiuOV8IRxP7+IL+mK+gs3tYZ7K6Wpwi/yvyPnIWg2Hcny1S/g5xSbSq76yKHn4ihy5Hb8+WSMwIDRvu68lLpq1ppWuWJGBcLP0CgYAnGtHlFY887zSI0aRdzV3EYm/sPnmYy0TWIUVLuGL8kkprcSIW2yRxx1GtpB+3A7cPu61WwRbiswJA6VmQHvn8f9ds81+aiO5KQLMk4MbrTwWhWLDpP4BxjWANVzPr19ZwmDJcYkh7YUI2iLqH7dXho5k62OYCM1T53xlql6HDcg==\n-----END RSA PRIVATE KEY-----"
+	//epCertNo := "914400007211976169" //对应企业库 credit_no
+	sceneCode := "JIANYU_LABEL"
+
+	epCertNo := util.ObjToString("9111000071093123XX")
+	createLabels := getCreditLabel(appID, privateKeyPEM, epCertNo, sceneCode)
+	log.Println(createLabels)
+
+}