update

bidding_test/ai.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+// sendAi 调用大模型招标分类
+func sendAi(data map[string]interface{}, url string) (res map[string]interface{}) {
+	// 设置 2 秒的超时
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
+	//url := "http://192.168.3.109:16688"
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		fmt.Println("JSON marshal error:", err)
+		return
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		fmt.Println("Request error:", err)
+		return
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	// 将请求与上下文关联
+	req = req.WithContext(ctx)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		// 使用 errors.Is 检查错误是否是超时错误
+		if errors.Is(err, context.DeadlineExceeded) {
+			fmt.Println("Request timed out")
+			return
+		}
+		fmt.Println("Request error:", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	err = json.NewDecoder(resp.Body).Decode(&res)
+	if err != nil {
+		fmt.Println("Response decoding error:", err)
+		return
+	}
+	//map[result:[结果-中标] status:200]
+
+	return
+}

bidding_test/bidd.go

@@ -2,26 +2,29 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
 	"net/http"
+	"strings"
+	"sync"
+	"time"
 )
 
-func getBidding() {
+// getBidding2 获取bidding 1.3日无二级分类数据
+func getBidding2() {
 	//2024-1-3日数据
 	where := map[string]interface{}{
 		"comeintime": map[string]interface{}{
 			"$gte": 1704211200,
 			"$lte": 1704297600,
 		},
-		"toptype": map[string]interface{}{
-			"$exists": 1,
-		},
 		"subtype": map[string]interface{}{
-			"$exists": 0,
+			"$exists": 1,
 		},
 	}
 
@@ -29,31 +32,180 @@ func getBidding() {
 	defer MgoB.DestoryMongoConn(sess)
 
 	//texts := make([]string, 0)
-	query := sess.DB("qfw").C("bidding").Find(where).Select(map[string]interface{}{"title": 1, "toptype": 1, "href": 1, "detail": 1}).Iter()
+	query := sess.DB("qfw").C("bidding").Find(where).Select(map[string]interface{}{"title": 1, "toptype": 1, "subtype": 1, "href": 1, "detail": 1, "channel": 1}).Iter()
 	count := 0
+
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
 		if count%1000 == 0 {
 			log.Println("current:", count)
 		}
+
+		if util.IntAll(tmp["extracttype"]) == -1 {
+			continue
+		}
+
+		id := mongodb.BsonIdToSId(tmp["_id"])
+		tmp["jyhref"] = GetJyURLByID(id)
+		MgoB.SaveByOriID("wcc_bidding_20240103_subtype_exists", tmp)
+
+		tmp = make(map[string]interface{})
+	}
+
+	log.Println("over")
+
+}
+
+// callAi 调用大模型
+func callAi() {
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	//where := map[string]interface{}{
+	//	"subtype_a": map[string]interface{}{
+	//		"$exists": 0,
+	//	},
+	//}
+	query := sess.DB("qfw_data").C("wcc_bidding_20240103_subtype_exists").Find(nil).Select(nil).Iter()
+	count := 0
+
+	//ch := make(chan bool, 1)
+	//wg := &sync.WaitGroup{}
+
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("--------  current:", count, tmp["_id"], "  ---------")
+		}
+
+		//ch <- true
+		//wg.Add(1)
+		//go func(tmp map[string]interface{}) {
+		//	defer func() {
+		//		<-ch
+		//		wg.Done()
+		//	}()
+
 		id := mongodb.BsonIdToSId(tmp["_id"])
 		title := util.ObjToString(tmp["title"])
 		detail := util.ObjToString(tmp["detail"])
 
-		data := title + "+" + detail
-		tmp["bidding_id"] = id
-
+		data := map[string]interface{}{
+			"title":  title,
+			"detail": detail,
+		}
 		reqData := map[string]interface{}{
-			"texts": []string{"title+detail  " + data},
+			"texts": []interface{}{data},
 		}
 
+		now := time.Now()
 		res := send(reqData)
+		log.Println(time.Since(now).Seconds(), tmp["_id"])
+
 		subtype := res["result"].([]interface{})
-		tmp["subtype"] = subtype[0]
+		result := subtype[0]
+		types := strings.Split(util.ObjToString(result), "-")
+		update := make(map[string]interface{})
+		if len(types) == 2 {
+			update["toptype_ai"] = types[0]
+			update["subtype_ai"] = types[1]
+			//没有内容
+			if detail == "" {
+				update["data_type"] = 1
+			} else {
+				update["data_type"] = 0
+			}
+
+			MgoB.UpdateById("wcc_bidding_20240103_subtype_exists", id, map[string]interface{}{"$set": update})
+		}
+
+		//}(tmp)
+		tmp = make(map[string]interface{})
+	}
+
+	//wg.Wait()
+	log.Println("over")
+
+}
+
+// getBidding 调用分类大模型
+func getBidding() {
+	//2024-1-3日数据
+	where := map[string]interface{}{
+		"comeintime": map[string]interface{}{
+			"$gte": 1704211200,
+			"$lte": 1704297600,
+		},
+		"subtype": map[string]interface{}{
+			"$exists": 0,
+		},
+	}
+
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	//texts := make([]string, 0)
+	query := sess.DB("qfw").C("bidding").Find(where).Select(map[string]interface{}{"title": 1, "toptype": 1, "href": 1, "detail": 1}).Iter()
+	count := 0
+
+	ch := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count)
+		}
+
+		if util.IntAll(tmp["extracttype"]) == -1 {
+			continue
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			title := util.ObjToString(tmp["title"])
+			detail := util.ObjToString(tmp["detail"])
+
+			tmp["bidding_id"] = id
+
+			data := map[string]interface{}{
+				"title":  title,
+				"detail": detail,
+			}
+			reqData := map[string]interface{}{
+				"texts": []interface{}{data},
+			}
 
-		MgoB.Save("wcc_0110", tmp)
-		tmp = map[string]interface{}{}
+			res := SendAi(reqData)
+			subtype := res["result"].([]interface{})
+			result := subtype[0]
+			types := strings.Split(util.ObjToString(result), "-")
+			if len(types) == 2 {
+				tmp["new_toptype"] = types[0]
+				tmp["new_subtype"] = types[1]
+			}
+
+			tmp["jyhref"] = GetJyURLByID(id)
+
+			//没有内容
+			if detail == "" {
+				tmp["data_type"] = 1
+			} else {
+				tmp["data_type"] = 0
+			}
+
+			MgoB.Save("wcc_20240103-2", tmp)
+
+		}(tmp)
+		tmp = make(map[string]interface{})
 	}
 
+	wg.Wait()
+	log.Println("over")
+
 }
 
 func send(data map[string]interface{}) (res map[string]interface{}) {
@@ -88,3 +240,49 @@ func send(data map[string]interface{}) (res map[string]interface{}) {
 
 	return
 }
+
+// SendAi 调用大模型招标分类
+func SendAi(data map[string]interface{}) (res map[string]interface{}) {
+	// 设置 2 秒的超时
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
+	url := "http://192.168.3.109:16688"
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		fmt.Println("JSON marshal error:", err)
+		return
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		fmt.Println("Request error:", err)
+		return
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	// 将请求与上下文关联
+	req = req.WithContext(ctx)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		// 使用 errors.Is 检查错误是否是超时错误
+		if errors.Is(err, context.DeadlineExceeded) {
+			fmt.Println("Request timed out")
+			return
+		}
+		fmt.Println("Request error:", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	err = json.NewDecoder(resp.Body).Decode(&res)
+	if err != nil {
+		fmt.Println("Response decoding error:", err)
+		return
+	}
+
+	return
+}

bidding_test/bidding_test.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"strings"
+	"testing"
+)
+
+// TestUpdateBiddingEs 更新bidding,site 乌鲁木齐地铁 错误标讯数据，删除es
+func TestUpdateBiddingEs(t *testing.T) {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	//es
+	Es := &elastic.Elastic{
+		S_esurl: "http://127.0.0.1:19805",
+		//S_esurl:  "http://172.17.4.184:19805",
+		I_size:   5,
+		Username: "es_all",
+		Password: "TopJkO2E_d1x",
+	}
+	Es.InitElasticSize()
+
+	//es 新集群
+	//EsNew := &elastic.Elastic{
+	//	S_esurl: "http://127.0.0.1:19905",
+	//	//S_esurl:  "http://172.17.4.184:19905",
+	//	I_size:   5,
+	//	Username: "jybid",
+	//	Password: "Top2023_JEB01i@31",
+	//}
+	//EsNew.InitElasticSize()
+
+	where := map[string]interface{}{
+		"site": "乌鲁木齐地铁",
+	}
+
+	query := sess.DB("qfw").C("bidding").Find(&where).Select(nil).Sort("-_id").Iter()
+	count := 0
+
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%100 == 0 {
+			log.Println("current:", count)
+		}
+
+		if util.IntAll(tmp["extracttype"]) == -1 {
+			continue
+		}
+
+		if util.IntAll(tmp["infoformat"]) != 1 {
+			continue
+		}
+
+		id := mongodb.BsonIdToSId(tmp["_id"])
+		title := util.ObjToString(tmp["title"])
+
+		if strings.Contains(title, "工作人员捡到") || strings.Contains(title, "捡到") {
+			log.Println(id)
+			//update := map[string]interface{}{
+			//	"extracttype": -1,
+			//}
+
+			//Mgo.UpdateById("bidding", id, map[string]interface{}{"$set": update})
+			//
+			//err := Es.DeleteByID("bidding", id)
+			//if err != nil {
+			//	log.Println("es delete err", err)
+			//}
+			//err = EsNew.DeleteByID("bidding", id)
+			//if err != nil {
+			//	log.Println("esNew delete err", err)
+			//}
+		}
+
+	}
+
+}

bidding_test/go.mod

@@ -2,21 +2,26 @@ module bidding_test
 
 go 1.20
 
-require jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231206053835-8820a8d9b44e
+require (
+	go.mongodb.org/mongo-driver v1.10.1
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16
+)
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.0 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/dchest/captcha v1.0.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.mongodb.org/mongo-driver v1.10.1 // indirect
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect

bidding_test/go.sum

@@ -19,6 +19,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
@@ -51,6 +52,7 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -59,10 +61,12 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -190,5 +194,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231206053835-8820a8d9b44e h1:MmuG5yx7pJ5iN/wp5ectaofXf+hIlqpYOqJkgbmtl/M=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231206053835-8820a8d9b44e/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16 h1:K6hILkGcqNRmTO1nsrpZMqG2iupJETmnmZmKt77SsNY=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

bidding_test/main.go

@@ -15,26 +15,89 @@ var (
 )
 
 func InitMgo() {
-	MgoB = &mongodb.MongodbSim{
-		//MongodbAddr: "172.17.189.140:27080",
-		MongodbAddr: "127.0.0.1:27083",
-		Size:        10,
-		DbName:      "qfw",
-		UserName:    "SJZY_RWbid_ES",
-		Password:    "SJZY@B4i4D5e6S",
-		Direct:      true,
-	}
-	MgoB.InitPool()
+	//MgoB = &mongodb.MongodbSim{
+	//	MongodbAddr: "172.17.189.140:27080",
+	//	//MongodbAddr: "127.0.0.1:27083",
+	//	Size:     10,
+	//	DbName:   "qfw",
+	//	UserName: "SJZY_RWbid_ES",
+	//	Password: "SJZY@B4i4D5e6S",
+	//	//Direct:   true,
+	//}
+	//MgoB.InitPool()
+
+	// 本地数据库
+	//MgoB = &mongodb.MongodbSim{
+	//	//MongodbAddr: "172.17.189.140:27080",
+	//	MongodbAddr: "127.0.0.1:27017",
+	//	Size:        10,
+	//	DbName:      "wcc",
+	//	//UserName:    "SJZY_RWbid_ES",
+	//	//Password:    "SJZY@B4i4D5e6S",
+	//	//Direct:      true,
+	//}
+	//MgoB.InitPool()
+
+	// 测试环境
+	//MgoB = &mongodb.MongodbSim{
+	//	MongodbAddr: "192.168.3.206:27002",
+	//	//MongodbAddr: "127.0.0.1:27017",
+	//	Size:     10,
+	//	DbName:   "qfw_data",
+	//	UserName: "root",
+	//	Password: "root",
+	//	//Direct:      true,
+	//}
+	//MgoB.InitPool()
 }
 
 func main() {
-	InitMgo()
-	getBidding()
-	//go updateMethod()
-	//dealNavColumn()
-	log.Println("over")
+	//InitMgo()
+	//getBidding2()
+	//callAi()
+
+	//updateTop()
+	//findData()
+	//getCompanyName()
+	//log.Println("over")
 	//select {}
 
+	//SpecialData()
+
+	StdData()
+	// ==============================================
+	////1.调用分类大模型
+	//title := "松江南站大型居住社区C18-26-01号地块动迁安置房（智能化工程）"
+	//detail := "<br><ul ><li>招标项目编号:</li><li >e3100000151006395007</li></ul><ul ><li>相关标段（包）编号:</li><li >e3100000151006395007001</li></ul><ul ><li>公示标题:</li><li >松江南站大型居住社区C18-26-01号地块动迁安置房（智能化工程）</li></ul><ul ><li>公示内容:</li><li >第一中标候选人：上海浩臣机电科技有限公司，投标价格：1228.1990，得分/投票：合格；第二中标候选人：上海格瑞特机电系统工程有限公司，投标价格：1230.1015，得分/投票：合格；</li></ul><ul ><li>公示发布时间:</li><li >2024-01-03</li></ul><ul ><li>公示发布媒介:</li><li >http://zjw.sh.gov.cn</li></ul><ul ><li>公示源URL:</li><li ><br>https://ciac.zjw.sh.gov.cn/XMJYPTInterWeb/Tender/PrinttoPdf?zbgcid=65693</li></ul><ul ><li>公示类型:</li><li >正常</li></ul><ul ><li>公示开始时间:</li><li >2024-01-03</li></ul><ul ><li>公示结束时间:</li><li >2024-01-05</li></ul>"
+	//
+	//data := map[string]interface{}{
+	//	"title":  title,
+	//	"detail": detail,
+	//}
+	//reqData := map[string]interface{}{
+	//	"texts": []interface{}{data},
+	//}
+	//
+	//url := "http://172.17.162.35:24401"
+	//now := time.Now()
+	//res := sendAi(reqData, url)
+	////log.Println("res", res)
+	//log.Println("time seconds", time.Since(now).Seconds())
+	//log.Println("lens", len(res))
+	//if len(res) > 0 {
+	//	resa := res["result"]
+	//	log.Println("resa", resa)
+	//	if dataa, ok := resa.([]interface{}); ok {
+	//		log.Println(222)
+	//		da := dataa[0]
+	//		if len(util.ObjToString(da)) > 0 {
+	//			cs := strings.Split(util.ObjToString(da), "-")
+	//			log.Println("toptype", cs[0])
+	//			log.Println("subtype", cs[1])
+	//		}
+	//	}
+	//}
+	// ==============================================
 }
 
 func dealNavColumn() {

bidding_test/qyxy.go

@@ -0,0 +1,222 @@
+package main
+
+import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+)
+
+// findData  找出企业信息中，company_type 不等于合体工商户的企业数据，然后写入一个临时表
+func findData() {
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "mixdata",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	//where := map[string]interface{}{
+	//	"company_type": map[string]interface{}{
+	//		"$ne": "个体工商户",
+	//	},
+	//}
+	query := sess.DB("mixdata").C("qyxy_std").Find(nil).Select(map[string]interface{}{"company_name": 1, "company_type": 1, "company_status": 1, "use_flag": 1}).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current:", count)
+		}
+		companyType := util.ObjToString(tmp["company_type"])
+		if companyType == "个体工商户" {
+			continue
+		}
+
+		company_name := util.ObjToString(tmp["company_name"])
+		whereN := map[string]interface{}{
+			"company_name": company_name,
+		}
+		num := Mgo.Count("qyxy_std", whereN)
+		if num > 1 {
+			Mgo.Save("wcc_qyxy_20240311", tmp)
+		}
+		tmp = make(map[string]interface{})
+	}
+
+	log.Println("结束")
+
+}
+
+// getCompanyName
+func getCompanyName() {
+	// 找出wcc_qyxy_20240311表中，公司名称有多个，并且一个use_flag=0,一个use_flag=10的数据
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "mixdata",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+	companyMap := make(map[string]bool)
+
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	query := sess.DB("mixdata").C("wcc_qyxy_20240311").Find(nil).Select(map[string]interface{}{"company_name": 1, "use_flag": 1}).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current:", count)
+		}
+
+		name := util.ObjToString(tmp["company_name"])
+		if companyMap[name] {
+			continue
+		} else {
+			companyMap[name] = true
+		}
+
+		where := map[string]interface{}{
+			"company_name": name,
+		}
+		res, _ := Mgo.Find("wcc_qyxy_20240311", where, nil, nil, false, -1, -1)
+		flaga := false
+		flagb := false
+		for _, v := range *res {
+			if util.Int64All(v["use_flag"]) == 0 {
+				flaga = true
+			} else if util.Int64All(v["use_flag"]) == 10 {
+				flagb = true
+			}
+		}
+		// 存在0和10 二个状态
+		if flaga && flagb {
+			Mgo.Save("wcc_qyxy_name_0325", map[string]interface{}{"company_name": name})
+		}
+
+		tmp = make(map[string]interface{})
+	}
+
+	log.Println("结束")
+}
+
+// SpecialData 处理特殊企业数据，更新 qyxy_std 表use_flag
+func SpecialData() {
+	// 处理 特殊企业 数据,
+	tables := []string{"special_enterprise", "special_foundation", "special_law_office", "special_social_organ", "special_trade_union"}
+	//1.先处理 special_enterprise 表数据，循环数据，根据company_name 去查询 qyxy_std 表，如果
+	// 数据只有一条，并且 special_enterprise.company_id 等于 qyxy_std._id；就更新 qyxy_std表的 use_flag 字段
+	// qyxy_std 表
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "mixdata",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+	// 181 凭安库
+	Mgo2 := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.181:27001",
+		//MongodbAddr: "127.0.0.1:27001",
+		DbName:   "mixdata",
+		Size:     10,
+		UserName: "",
+		Password: "",
+		//Direct:      true,
+	}
+	Mgo2.InitPool()
+
+	sess := Mgo2.GetMgoConn()
+	defer Mgo2.DestoryMongoConn(sess)
+
+	for _, v := range tables {
+		query := sess.DB("mixdata").C(v).Find(nil).Select(map[string]interface{}{"company_name": 1, "use_flag": 1, "company_id": 1}).Iter()
+		count := 0
+		for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+			if count%10000 == 0 {
+				log.Println("current:", count, "table - ", v, tmp["company_name"])
+			}
+
+			if _, ok := tmp["company_id"]; !ok {
+				continue
+			}
+			where := map[string]interface{}{
+				"_id": tmp["company_id"],
+			}
+			//update := map[string]interface{}{
+			//	"use_flag": tmp["use_flag"],
+			//}
+			//Mgo.Update("qyxy_std", where, map[string]interface{}{"$set": update}, true, false)
+			//name := util.ObjToString(tmp["company_name"])
+
+			//where := map[string]interface{}{
+			//	"company_name": name,
+			//}
+			// 如果 ID 相同，std 表use_flag =10,更新
+			datas, _ := Mgo.FindOne("qyxy_std", where)
+			if len(*datas) == 0 {
+				continue
+			}
+			//if util.Int64All((*datas)["use_flag"]) == util.Int64All(tmp["use_flag"]) {
+			//	continue
+			//}
+
+			//Mgo.Update("qyxy_std", where, map[string]interface{}{"$set": update}, true, false)
+			data := *datas
+			data["use_flag"] = tmp["use_flag"]
+			Mgo.SaveByOriID("wcc_std_0411", data)
+		}
+	}
+
+	log.Println("结束")
+
+}
+
+func StdData() {
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "mixdata",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	query := sess.DB("mixdata").C("wcc_std_0410").Find(nil).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current:", count)
+		}
+		autoid := util.Int64All(tmp["autoid"])
+		if autoid == 0 {
+			continue
+		}
+
+		where := map[string]interface{}{
+			"autoid": autoid,
+		}
+		datas, _ := Mgo.FindOne("qyxy_std", where)
+		if len(*datas) == 0 {
+			continue
+		}
+		Mgo.SaveByOriID("wcc_std_0411", datas)
+	}
+
+	log.Println("over")
+}

bidding_test/tools.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"sort"
 	"strings"
@@ -158,3 +159,11 @@ func TagBidding(tmp map[string]interface{}) (res map[string]interface{}) {
 
 	return
 }
+
+// GetJyURLByID 获取剑鱼地址
+func GetJyURLByID(id string) string {
+	var Url = "https://www.jianyu360.com/article/content/%s.html"
+	url := fmt.Sprintf(Url, util.CommonEncodeArticle("content", id))
+
+	return url
+}

bidding_test/updateBidding.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+)
+
+// updateBidding 更新bidding数据
+func updateBidding() {
+	//mongodb
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "qfw",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+
+	//es
+	Es := &elastic.Elastic{
+		//S_esurl: "http://127.0.0.1:19805",
+		S_esurl:  "http://172.17.4.184:19805",
+		I_size:   5,
+		Username: "es_all",
+		Password: "TopJkO2E_d1x",
+	}
+	Es.InitElasticSize()
+
+	//es 新集群
+	EsNew := &elastic.Elastic{
+		//S_esurl: "http://127.0.0.1:19905",
+		S_esurl:  "http://172.17.4.184:19905",
+		I_size:   5,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	EsNew.InitElasticSize()
+	//
+
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	selected := map[string]interface{}{"contenthtml": 0, "detail": 0}
+	it := sess.DB("qfw").C("bidding").Find(nil).Select(&selected).Iter()
+
+	count := 0
+	log.Println("开始")
+	// 拟建 爬虫代码
+	njCodes := []string{"a_qgjsxmhjxxgspt_ysbggs", "a_qgjsxmhjxxgspt_hpspqgs", "a_qgjsxmhjxxgspt_hpbggs"}
+	//产权 爬虫代码
+	cqCodes := []string{"sh_shsggzyjyzx_nyys", "sh_shsggzyjyzx_sfpm", "gd_dgsncjtzcglw_djyxx_lbfbzc", "gd_dgsncjtzcglw_djyxx_djyzc", "sc_scsggzyjyfwpt_jygk_qt", "qg_zhrmghgzrzyb_tdsccrgg", "ln_lnsggzyjypt_gycq", "a_gjggzyjypt_qt"}
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		id := mongodb.BsonIdToSId(tmp["_id"])
+		spidercode := util.ObjToString(tmp["spidercode"])
+		if count%10000 == 0 {
+			log.Println("current:", count, id)
+		}
+
+		if IsInStringArray(spidercode, njCodes) {
+			//拟建数据
+			update := map[string]interface{}{
+				"toptype": "拟建",
+				"subtype": "拟建",
+			}
+			Mgo.UpdateById("bidding", id, map[string]interface{}{"$set": update})
+
+			err := Es.UpdateDocument("bidding", id, update)
+			if err != nil {
+				log.Println("es 更新失败", id, err)
+			}
+			err = EsNew.UpdateDocument("bidding", id, update)
+			if err != nil {
+				log.Println("EsNew 更新失败", id, err)
+			}
+			//产权数据
+		} else if IsInStringArray(spidercode, cqCodes) {
+			update := map[string]interface{}{
+				"toptype": "产权",
+				"subtype": "产权",
+			}
+			Mgo.UpdateById("bidding", id, map[string]interface{}{"$set": update})
+			err := Es.DeleteByID("bidding", id)
+			if err != nil {
+				log.Println("es delete ", id, err)
+			}
+			err = EsNew.DeleteByID("bidding", id)
+			if err != nil {
+				log.Println("EsNew delete ", id, err)
+			}
+		}
+
+	}
+
+	log.Println("结束 ---------- over ")
+
+}
+
+// updateTop 更新一级分类
+func updateTop() {
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "qfw",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	query := sess.DB("qfw").C("wcc_bidding_20240103_subtype_exists").Find(nil).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current:", count)
+		}
+
+		id := mongodb.BsonIdToSId(tmp["_id"])
+		bidding, _ := Mgo.FindById("bidding", id, nil)
+		update := map[string]interface{}{
+			"toptype":    (*bidding)["toptype"],
+			"subtype":    (*bidding)["subtype"],
+			"infoformat": (*bidding)["infoformat"],
+		}
+
+		Mgo.UpdateById("wcc_bidding_20240103_subtype_exists", id, map[string]interface{}{"$set": update})
+		tmp = make(map[string]interface{})
+	}
+
+	log.Println("over")
+}

export_bidding/bank_poc.go

@@ -1,15 +1,12 @@
 package main
 
 import (
-	"context"
 	"fmt"
+	"github.com/xuri/excelize/v2"
 	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
-	"net/url"
 	"sort"
 	"strings"
 	"sync"
@@ -216,7 +213,7 @@ func bankPOC() {
 
 }
 
-// bankWinnerStatistic 统计企业信息
+// bankWinnerStatistic 统计企业信息;企业出现数量
 func bankWinnerStatistic() {
 	Mgo := &mongodb.MongodbSim{
 		MongodbAddr: "172.17.189.140:27080",
@@ -275,7 +272,68 @@ func bankWinnerStatistic() {
 	log.Println("保存结束")
 }
 
-// exportPocN 导出合作渠道银行POC需求;龚文华 需求
+// getPengWinner 根据 龚文华需求， 碰撞提供的企业名单，获取相关标讯信息
+func getPengWinner() {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+
+	f, err := excelize.OpenFile("政采测试100户名单0118.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		f.Save()
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	// 获取 Sheet1 上所有单元格
+	rows, err := f.GetRows("Sheet1")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for i := 1; i < len(rows); i++ {
+		name := rows[i][0]
+		where := map[string]interface{}{
+			"winner": name,
+		}
+		log.Println(name)
+		datas, _ := Mgo.Find("wcc_bank_winner", where, nil, nil, false, -1, -1)
+		//没有找到中标单位 标讯信息
+		if len(*datas) == 0 {
+			f.SetCellValue("Sheet1", fmt.Sprintf("C%v", i+1), "无")
+		} else {
+			f.SetCellValue("Sheet1", fmt.Sprintf("C%v", i+1), "有")
+		}
+
+		//if len(*datas) > 0 {
+		//	for k, _ := range *datas {
+		//		biddingID := util.ObjToString((*datas)[k]["bidding_id"])
+		//		biddingData, _ := Mgo.FindById("bidding", biddingID, nil)
+		//		//fmt.Println(biddingData)
+		//		Mgo.SaveByOriID("wcc_bank_winner_bidding", biddingData)
+		//	}
+		//}
+	}
+
+	log.Println("over")
+
+}
+
+/**
+// exportPocN 导出合作渠道银行POC需求;龚文华 需求;函数已作废
 func exportPocN() {
 	//InitMgo()
 	username := "SJZY_RWbid_ES"
@@ -386,7 +444,7 @@ func exportPocN() {
 	log.Println("over")
 }
 
-// staticBank 统计中标单位中标项目数量
+// staticBank 统计中标单位中标项目数量；函数已作废
 func staticBank() {
 	username := "SJZY_RWbid_ES"
 	password := "SJZY@B4i4D5e6S"
@@ -501,7 +559,7 @@ func staticBank() {
 
 }
 
-// exportPoc 导出合作渠道银行POC需求;龚文华 需求
+// exportPoc 导出合作渠道银行POC需求;龚文华 需求;函数作废
 func exportPoc() {
 	//InitMgo()
 	username := "SJZY_RWbid_ES"
@@ -621,6 +679,8 @@ func exportPoc() {
 	log.Println("over")
 }
 
+**/
+
 // IsInStringArray 判断数组中是否存在字符串
 func IsInStringArray(str string, arr []string) bool {
 	// 先对字符串数组进行排序

export_bidding/go.mod

@@ -3,6 +3,7 @@ module export_bidding
 go 1.20
 
 require (
+	github.com/xuri/excelize/v2 v2.8.0
 	go.mongodb.org/mongo-driver v1.10.1
 	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231223031213-3f719e173cb5
 )
@@ -13,15 +14,20 @@ require (
 	github.com/dchest/captcha v1.0.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/richardlehane/mscfb v1.0.4 // indirect
+	github.com/richardlehane/msoleps v1.0.3 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca // indirect
+	github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/crypto v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 )

export_bidding/go.sum

@@ -60,6 +60,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
@@ -71,14 +73,22 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
+github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
+github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM=
+github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
 github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -87,9 +97,16 @@ github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
 github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca h1:uvPMDVyP7PXMMioYdyPH+0O+Ta/UO1WFfNYMO3Wz0eg=
+github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
+github.com/xuri/excelize/v2 v2.8.0 h1:Vd4Qy809fupgp1v7X+nCS/MioeQmYVVzi495UCTqB7U=
+github.com/xuri/excelize/v2 v2.8.0/go.mod h1:6iA2edBTKxKbZAa7X5bDhcCg51xdOn1Ar5sfoXRGrQg=
+github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a h1:Mw2VNrNNNjDtw68VsEj2+st+oCSn4Uz7vZw6TbhcV1o=
+github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
@@ -102,14 +119,20 @@ go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo=
+golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -117,17 +140,24 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -138,13 +168,24 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -152,6 +193,8 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -190,7 +233,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c h1:fgH3lMOi2jG3KCejzC3d0otkJB6mJ9eaq1HecMoHG4c=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
 jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231223031213-3f719e173cb5 h1:IJlZ+JTn7UvVeHyALb+yWacmtE94TW2XvBIRgTyRmzU=
 jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231223031213-3f719e173cb5/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

export_bidding/main.go

@@ -3,7 +3,11 @@ package main
 import "log"
 
 func main() {
-	bankPOC()
+	//bankPOC()
 	//bankWinnerStatistic()
+	//getPengWinner()
+
+	//dealXmData() //处理厦门数据
+	CountBidamount()
 	log.Println("over ---- over")
 }

export_bidding/xiamen.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"strings"
+	"sync"
+	"time"
+	"unicode/utf8"
+)
+
+// dealXmData 处理中标企业注册地为厦门的数据;陈伟铭 需求
+/**
+中标方企业注册地为厦门的企业数量一共是多少，可以分别看下22年度，23年度的两个数据总量
+*/
+func dealXmData() {
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "qfw",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+
+	MgoC := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "mixdata",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	MgoC.InitPool()
+
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	pool := make(chan bool, 10) //处理协程
+	wg := &sync.WaitGroup{}
+
+	// 设置查询条件
+	filter := bson.D{
+		{"publishtime", bson.M{"$gte": 1640966400, "$lt": 1704038400}},
+		{"subtype", bson.M{"$in": []string{"中标", "单一", "成交", "合同"}}},
+	}
+
+	selected := map[string]interface{}{
+		"contenthtml":      0, // 0表示不返回该字段
+		"attach_text":      0, // 0表示不返回该字段
+		"detail":           0, // 0表示不返回该字段
+		"purchasingsource": 0, // 0表示不返回该字段
+		"jsondata":         0, // 0表示不返回该字段
+		"package":          0, // 0表示不返回该字段
+	}
+
+	it := sess.DB("qfw").C("bidding").Find(&filter).Select(&selected).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current :", count)
+		}
+
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+
+			if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
+				tmp = make(map[string]interface{})
+				return
+			}
+			// 针对存量数据，重复数据不进索引
+			if util.IntAll(tmp["extracttype"]) == -1 {
+				return
+			}
+
+			swinner := util.ObjToString(tmp["s_winner"])
+			if swinner == "" {
+				return
+			}
+
+			if utf8.RuneCountInString(swinner) < 4 {
+				return
+			}
+
+			publishtime := util.Int64All(tmp["publishtime"])
+			projectName := util.ObjToString(tmp["projectname"])
+
+			if strings.Contains(swinner, ",") {
+				winners := strings.Split(swinner, ",")
+				for _, v := range winners {
+					if utf8.RuneCountInString(v) < 4 {
+						continue
+					} else {
+						da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": v})
+						if da == nil || (*da)["credit_no"] == nil || util.ObjToString((*da)["credit_no"]) == "" {
+							continue
+						}
+						city := util.ObjToString((*da)["company_city"])
+						if city != "厦门市" {
+							continue
+						}
+
+						insert := map[string]interface{}{
+							"winner":         v,
+							"credit_no":      (*da)["credit_no"],
+							"bidding_id":     mongodb.BsonIdToSId(tmp["_id"]),
+							"projectname":    projectName,
+							"company_type":   (*da)["company_type"],
+							"company_status": (*da)["company_status"],
+							"company_area":   (*da)["company_area"],
+							"company_city":   (*da)["company_city"],
+							"year":           time.Unix(publishtime, 0).Year(),
+						}
+						Mgo.Save("wcc_xiamen_winner", insert)
+					}
+				}
+
+			} else {
+				da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": swinner})
+				if da == nil || (*da)["credit_no"] == nil || util.ObjToString((*da)["credit_no"]) == "" {
+					return
+				}
+
+				city := util.ObjToString((*da)["company_city"])
+				if city != "厦门市" {
+					return
+				}
+
+				insert := map[string]interface{}{
+					"winner":         swinner,
+					"credit_no":      (*da)["credit_no"],
+					"bidding_id":     mongodb.BsonIdToSId(tmp["_id"]),
+					"projectname":    projectName,
+					"company_type":   (*da)["company_type"],
+					"company_status": (*da)["company_status"],
+					"company_area":   (*da)["company_area"],
+					"company_city":   (*da)["company_city"],
+					"year":           time.Unix(publishtime, 0).Year(),
+				}
+				Mgo.Save("wcc_xiamen_winner", insert)
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+
+	}
+
+	wg.Wait()
+	log.Println("结束")
+}
+
+// CountBidamount 统计厦门中标单位，中标金额总数
+func CountBidamount() {
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "qfw",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	var BidMap = make(map[string]bool)
+	it := sess.DB("qfw").C("wcc_xiamen_winner").Find(nil).Select(nil).Iter()
+	count := 0
+
+	var total = float64(0)
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		biddingID := util.ObjToString(tmp["bidding_id"])
+		if BidMap[biddingID] {
+			continue
+		}
+		data, _ := Mgo.FindById("bidding", biddingID, nil)
+		BidMap[biddingID] = true
+		bid := util.Float64All((*data)["bidamount"])
+
+		total += bid
+		tmp = make(map[string]interface{})
+	}
+
+	log.Println("total", total)
+}

getEs/clickhouse.go

@@ -122,7 +122,7 @@ func click() {
 	sess := Mgo.GetMgoConn()
 	defer Mgo.DestoryMongoConn(sess)
 
-	query := sess.DB("top").C("wcc_beijing_0110").Find(nil).Select(map[string]interface{}{"list": 0}).Iter()
+	query := sess.DB("top").C("wcc_beijing_2023Q4").Find(nil).Select(map[string]interface{}{"list": 0}).Iter()
 	count := 0
 	ch := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
@@ -204,7 +204,7 @@ func click() {
 				indusName := util.ObjToString((*indus)[0]["industry_l1_name"])
 				beijing.IndustryName = &indusName
 			}
-			_, err := g.DB().Insert(gctx.New(), "wcc_beijing_2013", beijing)
+			_, err := g.DB().Insert(gctx.New(), "wcc_beijing_2024Q1", beijing)
 			if err != nil {
 				log.Println("clickhouse 写入失败;", err, companyName, companyID)
 			}

getEs/config.yaml

@@ -1,5 +1,6 @@
 database:
   default:
-    link: "clickhouse@tcp(127.0.0.1:8123)/default?dial_timeout=2000ms&max_execution_time=60"
-#    link: "clickhouse:jybi_admin:Da#jy20230825@tcp(cc-2zelp3xmkmsrtjhgp.public.clickhouse.ads.aliyuncs.com:8123)/top_test?dial_timeout=2000ms&max_execution_time=60"
+#    link: "clickhouse@tcp(127.0.0.1:8123)/default?dial_timeout=2000ms&max_execution_time=60"
+    link: "clickhouse:jybi_admin:Da#jy20230825@tcp(127.0.0.1:8123)/top_test?dial_timeout=2000ms&max_execution_time=60"
+    #    link: "clickhouse:jybi_admin:Da#jy20230825@tcp(cc-2zelp3xmkmsrtjhgp.public.clickhouse.ads.aliyuncs.com:8123)/top_test?dial_timeout=2000ms&max_execution_time=60"
     debug: true

getEs/diff.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	elasticsearch7 "github.com/elastic/go-elasticsearch/v7"
+	"log"
+	"time"
+)
+
+// getDiff 获取es差异数据
+func getDiff() {
+	cfg1 := elasticsearch7.Config{
+		Addresses: []string{
+			"127.0.0.1:19805",
+		},
+		Username: "es_all",
+		Password: "TopJkO2E_d1x",
+	}
+	es, err := elasticsearch7.NewClient(cfg1)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// 执行滚动搜索
+	scrollID := ""
+	scrollSize := 1000 // 每次滚动返回的文档数量
+	for {
+		res, err := es.Search(
+			es.Search.WithContext(context.Background()),
+			es.Search.WithIndex("buyer_v2"),
+			es.Search.WithSize(scrollSize),
+			es.Search.WithScroll(time.Minute), // 设置滚动过期时间
+
+		)
+		if err != nil {
+			log.Fatalf("Error searching: %s", err)
+		}
+		defer res.Body.Close()
+
+		var result map[string]interface{}
+		if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
+			log.Fatalf("Error parsing the response body: %s", err)
+		}
+
+		// 处理结果，比较数据
+		hits, found := result["hits"].(map[string]interface{})["hits"].([]interface{})
+		if !found || len(hits) == 0 {
+			break // 没有更多结果，退出循环
+		}
+		for _, hit := range hits {
+			doc := hit.(map[string]interface{})["_source"]
+			// 在这里，您可以根据需要进行数据对比
+			// 例如，检查该文档是否存在于 buyer_v3 中
+			// 如果不存在，则输出相关信息
+			fmt.Println(doc)
+		}
+
+		// 获取滚动 ID
+		if scrollID == "" {
+			scrollID, _ = result["_scroll_id"].(string)
+		}
+	}
+
+	// 清除滚动
+	if scrollID != "" {
+		res, err := es.ClearScroll(
+			es.ClearScroll.WithContext(context.Background()),
+			es.ClearScroll.WithScrollID(scrollID),
+		)
+		if err != nil {
+			log.Printf("Error clearing scroll: %s", err)
+		} else {
+			defer res.Body.Close()
+			var response map[string]interface{}
+			if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
+				log.Printf("Error parsing the response body: %s", err)
+			} else {
+				fmt.Println("Scroll cleared successfully")
+			}
+		}
+	}
+
+}

getEs/es_test.go

@@ -7,7 +7,11 @@ import (
 	"github.com/olivere/elastic/v7"
 	"io"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
+	"regexp"
+	"sort"
+	"strconv"
 	"strings"
 	"testing"
 )
@@ -16,6 +20,123 @@ func TestEs(T *testing.T) {
 
 	s := "aa"
 	fmt.Println(strings.Contains(s, ","))
+
+	//re := regexp.MustCompile(`包\d+`) // 标题只有一个包2
+	//re := regexp.MustCompile(`\d+批包(\d+(?:、\d+)*)`)
+	// 定义正则表达式
+	//re3 := regexp.MustCompile(`(?:包)?(\d+(?:、\d+)*)包?`) //标题含有多个包；包10、12、14、17、18、19
+	//re3 := regexp.MustCompile(`包(\d+(?:、\d+)*)`) //标题含有多个包；包10、12、14、17、18、19
+	////re3 := regexp.MustCompile(`\d+包`)//冀中股份2023年12月标准件计划框架协议采购2包12.7
+	////re2 := regexp.MustCompile(`标包\d+`)//2024一季度水火电一般工序类中小金工件（标包1）-招标公告
+	//text := "中国绿发投资集团有限公司直属项目公司2023年第20批集中采购非招标项目（包10、12、14、17、18、19"
+	//matches := re.FindAllString(util.ObjToString(text), -1)
+	//log.Println(matches)
+
+	//title := "中电科青岛科技产业园一期设计施工总承包1标段"
+	//rea := regexp.MustCompile(`总承包\d标段`) //总承包1标段
+	//maches := rea.FindAllString(title, -1)
+	//log.Println("rea", maches)
+	//reb := regexp.MustCompile(`包\d[-~]\d`) //包1-6
+	//title = "2023年度分布式光伏发电项目主要设备(包1~6)设备采购合格供应商"
+	//matches := reb.FindAllString(title, -1)
+	//log.Println("reb", matches)
+
+	//re := regexp.MustCompile(`\d+批包(\d+(?:、\d+)*)`)
+	re := regexp.MustCompile(`包?\d{1,2}[-~、和](包)?\d{1,2}包?`) //1-6包；01-06包；01、02包;包1、包2
+	title := "中国绿发投资集团有限公司直属项目公司2023年第20批集中标段1采购非招标项目（包10、12、14、17、18、19"
+	matches := re.FindAllString(title, -1)
+	log.Println("rec", matches)
+	//
+	//re2 := regexp.MustCompile(`包?\d{1,2}`) // 标题只有一个包2
+	//re3 := regexp.MustCompile(`\d{1,2}包`) // 标题只有一个包2
+	re3 := regexp.MustCompile(`(标段[1-9一二三四五六七八九]|包[1-9一二三四五六七八九]?[0-9]|[1-9一二三四五六七八九]?[0-9]包|[a-kA-K]包)`) // 标题只有一个包2
+	title = "济南市历下标段3区东关中心幼儿园超市包21、22、23、24"
+	matches = re3.FindAllString(title, -1)
+	log.Println("aaa", matches)
+
+	//ree := regexp.MustCompile(`第\d{1,2}[批]|\d{1,2}标段|标段\d{1,2}`)
+	//ree2 := regexp.MustCompile(`(第)?\d{1,2}[标段\d{1,2}]`)
+
+	//log.Println(ree2.FindAllString(title, -1))
+	// 原始数据
+	data := []string{"包1", "包1-2", "包4", "包4-5", "包3", "包5", "包9", "包11", "包12", "11包", "标段1", "包13"}
+	// 调用去重函数
+	uniquePackages := removeDuplicates(data)
+	// 输出去重后的结果
+	fmt.Println("去重后的数据:", uniquePackages)
+
+}
+
+func removeDuplicates(data []string) []string {
+	// 存储已存在的包号
+	existingPackages := make(map[int]bool)
+
+	// 存储包含包号信息的字符串
+	packages := make(map[int]string)
+
+	// 匹配包号的正则表达式
+	re := regexp.MustCompile(`(包)(\d+)(?:-(\d+))?`)
+
+	noexists := make([]string, 0)
+	// 遍历数据
+	for _, item := range data {
+		// 提取包号信息
+		matches := re.FindStringSubmatch(item)
+		if len(matches) < 3 {
+			noexists = append(noexists, item)
+			continue
+		}
+
+		// 解析包号
+		start, _ := strconv.Atoi(matches[2])
+		end := start
+		if len(matches[3]) > 0 {
+			end, _ = strconv.Atoi(matches[3])
+		}
+
+		// 添加到已存在的包号中
+		for i := start; i <= end; i++ {
+			existingPackages[i] = true
+		}
+
+		// 将包含包号信息的字符串存储到 packages 中
+		packages[start] = matches[0]
+	}
+
+	// 从 map 中提取去重后的包号并排序
+	var uniquePackages []int
+	for packageNum := range existingPackages {
+		uniquePackages = append(uniquePackages, packageNum)
+	}
+	sort.Ints(uniquePackages)
+
+	// 将连续的包号转换为包含范围的字符串
+	var result []string
+	var start, end int
+	for i, num := range uniquePackages {
+		if i == 0 {
+			start = num
+			end = num
+		} else if num == end+1 {
+			end = num
+		} else {
+			if start == end {
+				result = append(result, packages[start])
+			} else {
+				result = append(result, fmt.Sprintf("包%d-%d", start, end))
+			}
+			start = num
+			end = num
+		}
+	}
+	if start == end {
+		result = append(result, packages[start])
+	} else {
+		result = append(result, fmt.Sprintf("包%d-%d", start, end))
+	}
+
+	result = append(result, noexists...)
+	return result
 }
 
 // syncEs 同步es 数据道信集群
@@ -107,3 +228,125 @@ func TestSyncEs(T *testing.T) {
 	fmt.Println("结束~~~~~~~~~~~~~~~")
 
 }
+
+func TestGetP(T *testing.T) {
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+
+	//url := "http://172.17.4.184:19805"
+	url := "http://127.0.0.1:19805"
+	username := "es_all"
+	password := "TopJkO2E_d1x"
+	index := "projectset" //索引名称
+	// 创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(url),
+		elastic.SetBasicAuth(username, password),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+
+	rangeQuery := elastic.NewRangeQuery("pici").Gt("1672502400").Lte("1704038400")
+	//termQ := elastic.NewTermQuery("multipackage", 0)
+	//rangeQuery := elastic.NewRangeQuery("id").Gt("657b08556977356f5578cb25").Lte("657b08556977356f5578cb26")
+	query := elastic.NewBoolQuery().Must(rangeQuery)
+
+	ctx := context.Background()
+	//开始滚动搜索
+	scrollID := ""
+	scroll := "1m"
+	searchSource := elastic.NewSearchSource().
+		Query(query).
+		Size(1000).
+		Sort("_doc", true) //升序排序
+	//Sort("_doc", false) //降序排序
+	searchService := client.Scroll(index).
+		Size(1000).
+		Scroll(scroll).
+		SearchSource(searchSource)
+
+	res, err := searchService.Do(ctx)
+
+	if err != nil {
+		if err == io.EOF {
+			fmt.Println("没有数据")
+		} else {
+			panic(err)
+		}
+
+	}
+	defer client.ClearScroll().ScrollId(scrollID).Do(ctx) // 在退出时清理资源
+	fmt.Println("总数是：", res.TotalHits())
+	total := 0
+	for len(res.Hits.Hits) > 0 {
+		for _, hit := range res.Hits.Hits {
+			var doc map[string]interface{}
+			err := json.Unmarshal(hit.Source, &doc)
+			if err != nil {
+				log.Printf("解析文档失败：%s", err)
+				continue
+			}
+			//id := util.ObjToString(doc["id"])
+			//log.Println(id)
+			var matchWords = make([]string, 0)
+			var matchList = make([]interface{}, 0)
+			if list, ok := doc["list"].([]interface{}); ok {
+				for _, v := range list {
+					if da, ok := v.(map[string]interface{}); ok {
+						if util.ObjToString(da["toptype"]) != "招标" {
+							continue
+						}
+						title := util.ObjToString(da["title"])
+						// 使用正则表达式进行匹配
+						matches := GetMatches(title)
+						if len(matches) > 0 {
+							matchList = append(matchList, da)
+						}
+						matchWords = append(matchWords, matches...)
+					}
+				}
+			}
+
+			insert := make(map[string]interface{})
+			insert["project_id"] = doc["id"]
+			insert["_id"] = doc["id"]
+			insert["multipackage"] = doc["multipackage"]
+			insert["list"] = doc["list"]
+			insert["projectname"] = doc["projectname"]
+			insert["sourceinfourl"] = doc["sourceinfourl"]
+			if len(matchWords) > 0 {
+				insert["matchList"] = matchList
+				insert["package_name"] = util.ObjToString(doc["projectname"]) + "-" + strings.Join(matchWords, "、")
+				insert["type"] = 1
+			}
+
+			MgoB.SaveByOriID("wcc_project_20240304", insert)
+
+		}
+		total = total + len(res.Hits.Hits)
+		scrollID = res.ScrollId
+		res, err = client.Scroll().ScrollId(scrollID).Scroll(scroll).Do(ctx)
+		log.Println("current count:", total)
+		if err != nil {
+			if err == io.EOF {
+				// 滚动到最后一批数据，退出循环
+				break
+			}
+			log.Printf("滚动搜索失败：%s", err)
+			break // 处理错误时退出循环
+		}
+	}
+
+	fmt.Println("结束~~~~~~~~~~~~~~~")
+
+}

getEs/forecast.go

@@ -0,0 +1,194 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	"io"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"sync"
+)
+
+// getForecast 获取测试环境数据
+func getForecast() {
+	url := "http://192.168.3.149:9201"
+	//url := "http://127.0.0.1:19805"
+	username := ""
+	password := ""
+	index := "forecast" //索引名称
+	// 创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(url),
+		elastic.SetBasicAuth(username, password),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+	//测试环境
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "127.0.0.1:27080",
+		MongodbAddr: "192.168.3.206:27002",
+		DbName:      "qfw_data",
+		Size:        10,
+		UserName:    "root",
+		Password:    "root",
+		//Direct: true,
+	}
+	Mgo.InitPool()
+
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+
+	//2023年01-01  2023-10-01，,1-3季度
+	//areaTermsQuery := elastic.NewTermsQuery("area", "江苏", "安徽", "上海", "天津", "河北", "浙江", "天津市", "上海市", "河北省", "安徽省", "江苏省", "浙江省")
+	//rangeQuery := elastic.NewRangeQuery("firsttime").Gte(1696089600).Lt(1704038400)
+	//query := elastic.NewBoolQuery().
+	//	Must(areaTermsQuery).
+	//	Must(rangeQuery)
+
+	ctx := context.Background()
+	//开始滚动搜索
+	scrollID := ""
+	scroll := "10m"
+	searchSource := elastic.NewSearchSource().
+		Query(nil).
+		Size(10000).
+		Sort("_doc", true) //升序排序
+	//Sort("_doc", false) //降序排序
+
+	searchService := client.Scroll(index).
+		Size(10000).
+		Scroll(scroll).
+		SearchSource(searchSource)
+
+	res, err := searchService.Do(ctx)
+
+	if err != nil {
+		if err == io.EOF {
+			fmt.Println("没有数据")
+		} else {
+			panic(err)
+		}
+
+	}
+	//defer client.ClearScroll().ScrollId(scrollID).Do(ctx) // 在退出时清理资源
+	fmt.Println("总数是：", res.TotalHits())
+	total := 0
+	for len(res.Hits.Hits) > 0 {
+		for _, hit := range res.Hits.Hits {
+			var doc map[string]interface{}
+			err := json.Unmarshal(hit.Source, &doc)
+			if err != nil {
+				log.Printf("解析文档失败：%s", err)
+				continue
+			}
+
+			//infoid := util.ObjToString(doc["infoid"])
+			//
+			//bidding, _ := MgoB.FindById("bidding", infoid, nil)
+			////存入新表
+			//rs := Mgo.SaveByOriID("bidding", bidding)
+			//if !rs {
+			//	log.Println("保存出错", infoid)
+			//}
+
+			Mgo.Save("wcc_forecast", doc)
+
+		}
+
+		total = total + len(res.Hits.Hits)
+		scrollID = res.ScrollId
+		res, err = client.Scroll().ScrollId(scrollID).Scroll(scroll).Do(ctx)
+		log.Println("current count:", total)
+		if err != nil {
+			if err == io.EOF {
+				// 滚动到最后一批数据，退出循环
+				break
+			}
+			log.Println("滚动搜索失败：", err, res)
+			break // 处理错误时退出循环
+		}
+	}
+	// 在循环外调用 ClearScroll
+	_, err = client.ClearScroll().ScrollId(scrollID).Do(ctx)
+	if err != nil {
+		log.Printf("清理滚动搜索失败：%s", err)
+	}
+
+	fmt.Println("结束~~~~~~~~~~~~~~~")
+}
+
+func dealData() {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "127.0.0.1:27080",
+		MongodbAddr: "192.168.3.206:27002",
+		DbName:      "qfw_data",
+		Size:        10,
+		UserName:    "root",
+		Password:    "root",
+		//Direct: true,
+	}
+	Mgo.InitPool()
+
+	// 163 正式环境
+	MgoB := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	MgoB.InitPool()
+
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	query := sess.DB("qfw_data").C("wcc_forecast").Find(nil).Select(map[string]interface{}{"infoid": 1}).Iter()
+	count := 0
+
+	ch := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count)
+		}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			infoid := util.ObjToString(tmp["infoid"])
+			bidding, _ := MgoB.FindById("bidding", infoid, nil)
+			//存入新表
+			rs := Mgo.SaveByOriID("bidding", bidding)
+			if !rs {
+				log.Println("保存出错", infoid)
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+
+	wg.Wait()
+	log.Println("over")
+
+}

getEs/go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/clbanning/mxj/v2 v2.7.0 // indirect
 	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-faster/city v1.0.1 // indirect

getEs/go.sum

@@ -994,6 +994,8 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
+github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=

getEs/main.go

@@ -16,6 +16,9 @@ import (
 func main() {
 	//getData()
 	click()
+	//dealData()
+
+	//getProject()
 	log.Println("over ------------------ over")
 }
 
@@ -57,7 +60,9 @@ func getData() {
 	//MgoB.InitPool()
 
 	//2023年01-01  2023-10-01，,1-3季度
-	areaTermsQuery := elastic.NewTermsQuery("area", "江苏", "安徽", "上海", "天津", "河北", "浙江", "天津市", "上海市", "河北省", "安徽省", "江苏省", "浙江省")
+	//2024-1 - 2024-4;1704038400-1711900800
+	//2023-10-1 2024-1-1;1696089600-1704038400
+	areaTermsQuery := elastic.NewTermsQuery("area", "江苏", "安徽", "上海", "天津", "河北", "浙江", "天津市", "上海市", "河北省", "安徽省", "江苏省", "浙江省", "北京", "北京市")
 	rangeQuery := elastic.NewRangeQuery("firsttime").Gte(1696089600).Lt(1704038400)
 	query := elastic.NewBoolQuery().
 		Must(areaTermsQuery).
@@ -117,7 +122,7 @@ func getData() {
 				insert := doc
 				insert["s_winner"] = v
 				//存入新表
-				err = Mgo.InsertOrUpdate("top", "wcc_beijing_0110", insert)
+				err = Mgo.InsertOrUpdate("top", "wcc_beijing_2023Q4", insert)
 				if err != nil {
 					log.Println("error", doc["id"])
 				}

getEs/project.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	"io"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"regexp"
+	"strings"
+)
+
+// getProject 获取项目数据
+func getProject() {
+	MgoB := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "qfw",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:   true,
+	}
+	MgoB.InitPool()
+
+	url := "http://172.17.4.184:19805"
+	//url := "http://127.0.0.1:19805"
+	username := "es_all"
+	password := "TopJkO2E_d1x"
+	index := "projectset" //索引名称
+	// 创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(url),
+		elastic.SetBasicAuth(username, password),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+
+	rangeQuery := elastic.NewRangeQuery("pici").Gt("1685548800").Lte("1704038400")
+	//termQ := elastic.NewTermQuery("multipackage", 0)
+	//rangeQuery := elastic.NewRangeQuery("id").Gt("657b08556977356f5578cb25").Lte("657b08556977356f5578cb26")
+	query := elastic.NewBoolQuery().Must(rangeQuery)
+
+	ctx := context.Background()
+	//开始滚动搜索
+	scrollID := ""
+	scroll := "1m"
+	searchSource := elastic.NewSearchSource().
+		Query(query).
+		Size(100).
+		Sort("_doc", true) //升序排序
+	//Sort("_doc", false) //降序排序
+	searchService := client.Scroll(index).
+		Size(100).
+		Scroll(scroll).
+		SearchSource(searchSource)
+
+	res, err := searchService.Do(ctx)
+
+	if err != nil {
+		if err == io.EOF {
+			fmt.Println("没有数据")
+		} else {
+			panic(err)
+		}
+
+	}
+	defer client.ClearScroll().ScrollId(scrollID).Do(ctx) // 在退出时清理资源
+	fmt.Println("总数是：", res.TotalHits())
+	total := 0
+	for len(res.Hits.Hits) > 0 {
+		for _, hit := range res.Hits.Hits {
+			var doc map[string]interface{}
+			err := json.Unmarshal(hit.Source, &doc)
+			if err != nil {
+				log.Printf("解析文档失败：%s", err)
+				continue
+			}
+			//id := util.ObjToString(doc["id"])
+			//log.Println(id)
+			var matchWords = make([]string, 0)
+			var matchList = make([]interface{}, 0)
+			if list, ok := doc["list"].([]interface{}); ok {
+				for _, v := range list {
+					if da, ok := v.(map[string]interface{}); ok {
+						if util.ObjToString(da["toptype"]) != "招标" {
+							continue
+						}
+						title := util.ObjToString(da["title"])
+						// 使用正则表达式进行匹配
+						matches := GetMatches(title)
+						if len(matches) > 0 {
+							matchList = append(matchList, da)
+						}
+						matchWords = append(matchWords, matches...)
+					}
+				}
+			}
+
+			insert := make(map[string]interface{})
+			insert["project_id"] = doc["id"]
+			insert["_id"] = doc["id"]
+			insert["multipackage"] = doc["multipackage"]
+			insert["list"] = doc["list"]
+			insert["projectname"] = doc["projectname"]
+			insert["sourceinfourl"] = doc["sourceinfourl"]
+			if len(matchWords) > 0 {
+				insert["matchList"] = matchList
+				insert["package_name"] = util.ObjToString(doc["projectname"]) + "-" + strings.Join(matchWords, "、")
+				insert["type"] = 1
+			}
+
+			MgoB.SaveByOriID("wcc_project_20240304", insert)
+
+		}
+		total = total + len(res.Hits.Hits)
+		scrollID = res.ScrollId
+		res, err = client.Scroll().ScrollId(scrollID).Scroll(scroll).Do(ctx)
+		log.Println("current count:", total)
+		if err != nil {
+			if err == io.EOF {
+				// 滚动到最后一批数据，退出循环
+				break
+			}
+			log.Printf("滚动搜索失败：%s", err)
+			break // 处理错误时退出循环
+		}
+	}
+
+	fmt.Println("结束~~~~~~~~~~~~~~~")
+
+}
+
+func GetMatches(title string) (res []string) {
+	// 编译正则表达式
+	re := regexp.MustCompile(`包\d+`) // 标题只有一个包2
+	// 定义正则表达式
+	re2 := regexp.MustCompile(`包(\d+(?:、\d+)*)`) //标题含有多个包；包10、12、14、17、18、19
+	re3 := regexp.MustCompile(`\d+包`)            //冀中股份2023年12月标准件计划框架协议采购2包12.7
+	re4 := regexp.MustCompile(`标包\d+`)           //2024一季度水火电一般工序类中小金工件（标包1）-招标公告
+	//re4 := regexp.MustCompile(`\d+标段`)
+
+	//text := "中国绿发投资集团有限公司直属项目公司2023年第20批集中采购非招标项目（包10、12、14、17、18、19"
+	//matches := re3.FindAllString(util.ObjToString(text), -1)
+	matches := re2.FindAllString(util.ObjToString(title), -1)
+	if len(matches) > 0 {
+		return matches
+	}
+
+	matches = re4.FindAllString(util.ObjToString(title), -1)
+	if len(matches) > 0 {
+		return matches
+	}
+
+	matches = re3.FindAllString(util.ObjToString(title), -1)
+	if len(matches) > 0 {
+		return matches
+	}
+
+	matches = re.FindAllString(util.ObjToString(title), -1)
+	return matches
+}

ldx/go.mod

@@ -0,0 +1,11 @@
+module ldx
+
+go 1.16
+
+require (
+	github.com/stretchr/testify v1.8.0 // indirect
+	go.mongodb.org/mongo-driver v1.11.0
+	golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 // indirect
+	golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16
+)

ldx/go.sum

@@ -0,0 +1,201 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.11.0 h1:FZKhBSTydeuffHj9CBjXlR8vQLee1cQyTWYPA6/tqiE=
+go.mongodb.org/mongo-driver v1.11.0/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c=
+golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E=
+golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16 h1:K6hILkGcqNRmTO1nsrpZMqG2iupJETmnmZmKt77SsNY=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

ldx/main.go

@@ -0,0 +1,506 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"math"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	MatchArr []TagMatching
+	BrandArr []Brand
+	KeyRegs  []*RegexpInfo //关键词正则
+	KeyWords = "课桌椅,办公桌,会议桌,讲桌,办公桌椅,木质课桌,电脑桌,折叠桌,课桌凳,学生桌,教师桌,乒乓球桌,幼儿桌子,午休椅,讲桌,木制桌,休闲桌椅,多功能桌,多功能椅,教室桌,培训桌,桌椅板凳,午休卓,办公椅,礼堂椅,会客椅"
+)
+
+type Brand struct {
+	Name string   `json:"name" bson:"name"`
+	Tags []string `json:"tags" bson:"tags"`
+}
+
+var (
+	MgoBid   *mongodb.MongodbSim
+	Thread   int //配置项线程数
+	saveSize int
+
+	savePool chan map[string]interface{}
+	saveSp   chan bool
+
+	saveTmpPool chan map[string]interface{} //存储itemname过滤的数据，存在临时表
+	saveTmpSp   chan bool
+)
+
+func InitDB() *mongo.Client {
+	clientOptions := options.Client().ApplyURI("mongodb://127.0.0.1:27017")
+	MongoDBClient, err := mongo.Connect(context.TODO(), clientOptions) // 连接数据库
+	if err != nil {
+		fmt.Println("err =>", err)
+	}
+	err = MongoDBClient.Ping(context.TODO(), nil) // 检查数据库是否连接成功
+	if err != nil {
+		fmt.Println("err =>", err)
+	}
+	fmt.Println("链接success")
+
+	return MongoDBClient
+
+}
+
+func InitMgo() {
+	MgoBid = &mongodb.MongodbSim{
+		MongodbAddr: "127.0.0.1:27017",
+		DbName:      "ldx",
+		Size:        10,
+		UserName:    "",
+		Password:    "",
+	}
+	MgoBid.InitPool()
+
+	Thread = 1
+	saveSize = 200
+
+	savePool = make(chan map[string]interface{}, 1000)
+	saveSp = make(chan bool, 4)
+
+	saveTmpPool = make(chan map[string]interface{}, 1000)
+	saveTmpSp = make(chan bool, 2)
+}
+
+func main() {
+	//client := InitDB()
+	InitMgo()
+	InitRule()
+	InitBrand()
+
+	go saveMethod()
+	go saveTmpMethod()
+	taskRun()
+
+	c := make(chan bool, 1)
+	<-c
+
+	//
+	//f, err := excelize.OpenFile("ldx-tmp.xlsx")
+	//if err != nil {
+	//	fmt.Println(err)
+	//	return
+	//}
+	//defer func() {
+	//	if err := f.Close(); err != nil {
+	//		fmt.Println(err)
+	//	}
+	//}()
+	//
+	//datas := []interface{}{}
+	//// 获取 Sheet1 上所有单元格
+	//rows, err := f.GetRows("ldx-tmp")
+	//if err != nil {
+	//	fmt.Println(err)
+	//	return
+	//}
+	//
+	//for k, row := range rows[1:] {
+	//	if len(row) < 3 {
+	//		continue
+	//	}
+	//	pinpai := PinPai{
+	//		Name: strings.TrimSpace(row[1]),
+	//		Tags: strings.Split(row[2], "/"),
+	//	}
+	//	fmt.Println("k ==>", k)
+	//
+	//	datas = append(datas, pinpai)
+	//	//client.Database("ldx").Collection("pinpai").InsertOne(context.TODO(), pinpai)
+	//
+	//}
+	//client.Database("ldx").Collection("pinpai").InsertMany(context.TODO(), datas)
+	//
+	//fmt.Println("success~~~~~~")
+
+}
+
+// InitRule 初始化规则
+func InitRule() {
+	info, _ := MgoBid.Find("productclass_rule", nil, `{"_id": 1}`, nil, false, -1, -1)
+	for _, m := range *info {
+		tag := TagMatching{}
+		tag.tagName = util.ObjToString(m["label_name"])
+		tag.tagCode = util.ObjToString(m["label_code"])
+		tag.buyerclass = util.ObjToString(m["buyerclass"])
+
+		// 关键词
+		if f := util.ObjToString(m["match_keyword"]); f != "" {
+			tag.matchField = strings.Split(f, ",")
+
+			if v := util.ObjToString(m["keyword"]); v != "" {
+				tag.matchKey = util.ObjToString(m["keyword"])
+				tag.matchKeyReg = GetRegex(util.ObjToString(m["keyword"]))
+			}
+		}
+		// 附件词
+		if f := util.ObjToString(m["match_fjword"]); f != "" {
+			tag.addField = strings.Split(f, ",")
+
+			if v := util.ObjToString(m["fjword"]); v != "" {
+				tag.addKey = util.ObjToString(m["fjword"])
+				tag.addKeyReg = GetRegex(util.ObjToString(m["fjword"]))
+			}
+		}
+		// 排除词
+		if f := util.ObjToString(m["match_pcword"]); f != "" {
+			tag.excludeField = strings.Split(f, ",")
+
+			if v := util.ObjToString(m["pcword"]); v != "" {
+				tag.excludeKey = util.ObjToString(m["pcword"])
+				tag.excludeKeyReg = GetRegex(util.ObjToString(m["pcword"]))
+			}
+		}
+		// 清理词
+		if v := util.ObjToString(m["qlword"]); v != "" {
+			tag.clearKey = strings.Split(util.ObjToString(m["qlword"]), ",")
+		}
+		MatchArr = append(MatchArr, tag)
+	}
+	fmt.Println("len InitRule ==>", len(MatchArr))
+	//log.Info("InitRule", zap.Any("MatchArr", len(MatchArr)))
+}
+
+func InitBrand() {
+	info, _ := MgoBid.Find("brand_name", nil, `{"_id": 1}`, nil, false, -1, -1)
+	brands, err := json.Marshal(*info)
+	if err != nil {
+		fmt.Println("InitBrand Marshal err =>", err)
+	}
+
+	err = json.Unmarshal(brands, &BrandArr)
+	if err != nil {
+		fmt.Println("InitBrand Unmarshal err =>", err)
+	}
+
+	fmt.Println("len brand ==>", len(BrandArr))
+	//log.Info("InitBrand", zap.Int("BrandArr", len(BrandArr)))
+
+	KeyRegs = GetRegex(KeyWords)
+	fmt.Println("len(KeyRegs)->", len(KeyRegs))
+}
+
+func taskRun() {
+	sess := MgoBid.GetMgoConn()
+	defer MgoBid.DestoryMongoConn(sess)
+
+	ch := make(chan bool, Thread)
+	wg := &sync.WaitGroup{}
+
+	//query := sess.DB("ldx").C("20221122Ldx_kzy").FindId("637c7cd709f6fc229744825b").Select(nil).Iter()
+	f := bson.M{"purchasinglist": 1, "area": 1, "city": 1, "district": 1, "buyerclass": 1, "winner": 1, "buyer": 1,
+		"agency": 1, "matchkey": 1, "publishtime": 1, "projectname": 1, "budget": 1, "bidamount": 1, "id": 1}
+	query := sess.DB("ldx").C("20221122Ldx_kzy").Find(nil).Select(f).Iter()
+	count := 0
+	//tmp, _ := MgoBid.FindById("20221122Ldx_kzy", "637c7cd709f6fc2297448265", nil)
+
+	//go func(tmp map[string]interface{}) {
+	//	defer func() {
+	//		<-ch
+	//		wg.Done()
+	//	}()
+	//	tag := taskFuc(tmp) //打标签, 返回 对应TagMatching，
+	//	//处理数据
+	//	start := time.Now()
+	//	dealTmp(tmp, tag)
+	//	elapsed := time.Since(start)
+	//	fmt.Println("dealTmp -- 该函数执行完成耗时：", elapsed)
+	//
+	//}(*tmp)
+
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%saveSize == 0 {
+			//log.Info(fmt.Sprintf("current --- %d", count))
+			log.Println("current ----->", count)
+			//fmt.Println(fmt.Sprintf("current --- %d", count))
+		}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			tag := TaskFuc(tmp) //打标签, 返回 对应TagMatching，
+			//处理数据
+			//start := time.Now()
+			dealTmp(tmp, tag)
+			//elapsed := time.Since(start)
+			//fmt.Println("dealTmp -- 该函数执行完成耗时：", elapsed)
+
+		}(tmp)
+
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	fmt.Println("over ---> ", count)
+}
+
+func TaskFuc(tmp map[string]interface{}) (tag TagMatching) {
+
+L:
+
+	for _, v := range MatchArr {
+		// 同个类型的标签如果存在，就不需要再打
+		if v.buyerclass == "" || (v.buyerclass != "" && strings.Contains(v.buyerclass, util.ObjToString(tmp["buyerclass"]))) {
+			// 排除词
+			if len(v.excludeField) > 0 && len(v.excludeKeyReg) > 0 {
+				for _, f := range v.excludeField {
+					if val := util.ObjToString(tmp[f]); val != "" {
+						for _, e1 := range v.excludeKeyReg {
+							if e1.regs != nil && e1.regs.MatchString(val) {
+								break L
+							} else {
+								// && 特殊处理
+								if strings.Contains(e1.keyStr, "&&") {
+									flag := true
+									for _, s := range strings.Split(e1.keyStr, "&&") {
+										if !strings.Contains(val, s) {
+											flag = false
+											break
+										}
+									}
+									if flag {
+										break L
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+			// 清理词
+			if len(v.clearKey) > 0 && len(v.matchField) > 0 {
+				for _, s := range v.clearKey {
+					for _, f := range v.matchField {
+						if val := util.ObjToString(tmp[f]); val != "" {
+							tmp[f] = strings.ReplaceAll(val, s, "")
+						}
+					}
+				}
+			}
+			// 关键词
+			if len(v.matchField) > 0 && len(v.matchKeyReg) > 0 {
+				for _, f := range v.matchField {
+					if val := util.ObjToString(tmp[f]); val != "" {
+						for _, r1 := range v.matchKeyReg {
+							if r1.regs.MatchString(val) {
+								if len(v.addField) > 0 && len(v.addKeyReg) > 0 {
+									// 匹配附加词
+									isCt := false
+									for _, f1 := range v.addField {
+										if v1 := util.ObjToString(tmp[f1]); v1 != "" {
+											for _, r2 := range v.addKeyReg {
+												if r2.regs != nil && r2.regs.MatchString(v1) {
+													isCt = true
+												} else {
+													// && 特殊处理
+													if strings.Contains(r2.keyStr, "&&") {
+														flag := true
+														for _, s := range strings.Split(r2.keyStr, "&&") {
+															if !strings.Contains(v1, s) {
+																flag = false
+																break
+															}
+														}
+														if flag {
+															isCt = true
+														}
+													}
+												}
+											}
+										}
+									}
+									if isCt {
+										return v
+									}
+								} else {
+									return v
+
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return tag
+}
+
+// dealTmp 处理tmp 新增字段，拆分purchasinglist,补充里面单价数量
+func dealTmp(tmp map[string]interface{}, tag TagMatching) {
+	tmp["infoid"] = util.ObjToString(tmp["id"])
+	//满足打标签，添加新字段
+	if tag.tagName != "" {
+		tmp["productclass_val"] = tag.tagName
+		tmp[tag.tagName] = tag.tagCode
+	}
+
+	//包含二个字段时，进行计算补充第三个字段
+	fields := []string{"number", "totalprice", "unitprice"}
+
+	// 1.循环purchasinglist，匹配里面的brandname 值，然后替换
+	//2.计算补充里 数量总价
+	if purchasinglist, ok := tmp["purchasinglist"]; ok {
+		if list, ok1 := purchasinglist.([]interface{}); ok1 {
+			if len(list) > 0 {
+				for _, item := range list {
+					newTmp := util.DeepCopy(tmp).(map[string]interface{})
+					skipTmp := util.DeepCopy(tmp).(map[string]interface{})
+
+					var values []string
+					item1 := item.(map[string]interface{})
+
+					if brandname := util.ObjToString(item1["brandname"]); brandname != "" {
+						values = append(values, util.ObjToString(brandname))
+					}
+
+					if itemname := util.ObjToString(item1["itemname"]); itemname != "" {
+						if !MatchField(strings.Split(KeyWords, ","), []string{itemname}) {
+							for kk, vv := range item1 {
+								skipTmp[kk] = vv
+							}
+							delete(skipTmp, "purchasinglist")
+							delete(skipTmp, "_id")
+							saveTmpPool <- skipTmp
+							continue
+						}
+
+						values = append(values, util.ObjToString(itemname))
+					}
+					if model := util.ObjToString(item1["model"]); model != "" {
+						values = append(values, util.ObjToString(model))
+					}
+					var matched bool
+					//匹配 brandname,匹配到就替换原来的值
+					for _, brand := range BrandArr {
+						matched = MatchField(brand.Tags, values)
+						if matched {
+							item1["brandname"] = brand.Name
+							break
+						}
+					}
+					//匹配不上，直接用其他
+					if !matched && util.ObjToString(item1["brandname"]) != "" {
+						item1["brandname"] = "其他"
+					}
+
+					// 判断补充 purchasinglist 节点里的数量关系
+					if SumFields(fields, item1) > 1 {
+						number, okn := item1["number"]
+						unitprice, oku := item1["unitprice"]
+						totalprice, okt := item1["totalprice"]
+
+						if okn && oku && !okt { //只有数量和单价
+							item1["totalprice"] = number.(float64) * unitprice.(float64)
+						} else if okn && okt && !oku { // 只有数量和总价
+							item1["unitprice"] = totalprice.(float64) / number.(float64)
+						} else if okt && oku && !okn { //只有总价和单价
+							number = totalprice.(float64) / unitprice.(float64)
+							number = math.Ceil(number.(float64))
+							item1["number"] = number
+						}
+					}
+
+					for kk, vv := range item1 {
+						newTmp[kk] = vv
+					}
+
+					delete(newTmp, "purchasinglist")
+					delete(newTmp, "_id")
+					savePool <- newTmp
+				}
+			}
+
+		}
+	}
+
+}
+
+func saveMethod() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MgoBid.SaveBulk("20221122Ldx_kzy_new", arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MgoBid.SaveBulk("20221122Ldx_kzy_new", arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+// saveTmpMethod 过滤的数据进一个临时表
+func saveTmpMethod() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveTmpPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveTmpSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveTmpSp
+					}()
+					MgoBid.SaveBulk("20221122Ldx_kzy_skip", arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveTmpSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveTmpSp
+					}()
+					MgoBid.SaveBulk("20221122Ldx_kzy_skip", arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

ldx/read.txt

@@ -0,0 +1,6 @@
+处理立达信  项目代码
+
+1.读取并存储品牌信息
+2.根据 bidding表的标的物信息，匹配到对应的品牌，更新品牌缺失字段以及标的物中的 单价、数量、总价信息。
+3.针对数据进行标签匹配，匹配到就追加相应的字段信息
+

ldx/util.go

@@ -0,0 +1,450 @@
+package main
+
+import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type TagMatching struct {
+	tagName       string        //标签名称
+	tagCode       string        //标签值(保存)
+	matchField    []string      //关键词匹配字段,title,detail
+	matchKey      string        //匹配的词语，多个使用逗号连接,"部队,国防,军事,军用"
+	matchKeyReg   []*RegexpInfo //关键词的正则
+	matchOutKey   string        //匹配关键词的同时，排除关键词中的特殊词
+	matchOutReg   []*RegexpInfo //关键词中排除词的正则
+	addField      []string      //附加词匹配字段
+	addKey        string        //附件词匹配关键词
+	addKeyReg     []*RegexpInfo //附加次的正则
+	addOutKey     string        //满足附加词同时，需要排除的关键词，排除字段还是 addKey
+	addOutReg     []*RegexpInfo //满足附加词中排除词的正则表达式
+	excludeField  []string      //全局排除词
+	excludeKey    string        //全局排除词匹配词
+	excludeKeyReg []*RegexpInfo
+	clearKey      []string //清理词匹配字段跟关键词一样
+	buyerclass    string   //采购单位类型字段
+}
+
+type RegexpInfo struct {
+	keyStr string
+	regs   *regexp.Regexp
+}
+
+// GetRegex 根据关键词或者对应正则
+func GetRegex(key string) []*RegexpInfo {
+	var infos []*RegexpInfo
+	for _, s := range strings.Split(key, ",") {
+		if strings.Contains(s, "&&") || strings.Contains(s, "&!") {
+			info := &RegexpInfo{
+				keyStr: s,
+				regs:   nil,
+			}
+			infos = append(infos, info)
+		} else {
+			info := &RegexpInfo{
+				keyStr: s,
+				regs:   regexp.MustCompile(".*(?i)" + s + ".*"),
+			}
+			infos = append(infos, info)
+		}
+
+	}
+	return infos
+}
+
+// TaskTags 根据数据和正则规则，验证数据标签
+func TaskTags(tmp map[string]interface{}, regs []TagMatching) (tags []string) {
+	for _, v := range regs {
+		// 1.排除词
+		if len(v.excludeField) > 0 && len(v.excludeKeyReg) > 0 {
+			// 遍历排除词对应的tmp字段信息
+			for _, f := range v.excludeField {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					if getRegsResult(val, v.excludeKeyReg) {
+						return
+					}
+				}
+			}
+		}
+
+		// 清理词；目的把 类似 fuck的单词替换为空字符串
+		if len(v.clearKey) > 0 && len(v.matchField) > 0 {
+			for _, s := range v.clearKey {
+				for _, f := range v.matchField {
+					if val := util.ObjToString(tmp[f]); val != "" {
+						tmp[f] = strings.ReplaceAll(val, s, "")
+					}
+				}
+			}
+		}
+
+		// 关键词
+		if len(v.matchField) > 0 && len(v.matchKeyReg) > 0 {
+			for _, f := range v.matchField {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					//判断关键词字段的排除情况,含关键词的排除词直接退出
+					if len(v.matchOutReg) > 0 && getRegsResult(val, v.matchOutReg) {
+						return
+					}
+					// 符合关键词条件
+					if getRegsResult(val, v.matchKeyReg) {
+						// 附加词含有排除词时
+						if len(v.addOutReg) > 0 && getRegsResult(val, v.addOutReg) {
+							return
+						}
+						if len(v.addField) > 0 && len(v.addKeyReg) > 0 {
+							// 不满足附加词，直接返回
+							if !getRegsResult(val, v.addKeyReg) {
+								continue
+							}
+						}
+						tags = append(tags, v.tagName)
+					}
+				}
+			}
+		}
+	}
+
+	return
+}
+
+// getRegsResult 验证数据是否符合数组正则
+func getRegsResult(data string, regs []*RegexpInfo) (res bool) {
+	for _, e1 := range regs {
+		if e1.regs != nil && e1.regs.MatchString(data) {
+			return true
+		} else {
+			// && 特殊处理
+			if strings.Contains(e1.keyStr, "&&") {
+				flag := true
+				for _, s := range strings.Split(e1.keyStr, "&&") {
+					if !strings.Contains(data, s) {
+						flag = false
+						break
+					}
+				}
+				if flag {
+					return true
+				}
+			}
+			// 前面是必须有的关键词&!，后面是不能有的关键词;比如 军队&!点军队，
+			if strings.Contains(e1.keyStr, "&!") {
+				keys := strings.Split(e1.keyStr, "&!")
+				if strings.Contains(data, keys[0]) && !strings.Contains(data, keys[1]) {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+var (
+	regNumFloat, _  = regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
+	regStrUnit, _   = regexp.Compile(`[元|万|亿]`)
+	contentUnit, _  = regexp.Compile(`(万元|单位/万)`)
+	numCapitals, _  = regexp.Compile(`([〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]{4,40})`)
+	regStrChar      = `[〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
+	moneyRegChar, _ = regexp.Compile(regStrChar)
+	regQianw, _     = regexp.Compile(`\d{1,2}千万`)
+
+	cutAllSpace, _ = regexp.Compile(`\s*`)
+	spaces         = []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n"}
+
+	moneyChar = map[string]interface{}{ //"〇": "0", "零": "0",
+		"一": float64(1), "壹": float64(1), "二": float64(2), "贰": float64(2), "三": float64(3), "叁": float64(3), "四": float64(4), "肆": float64(4), "五": float64(5), "伍": float64(5),
+		"六": float64(6), "陆": float64(6), "七": float64(7), "柒": float64(7), "八": float64(8), "捌": float64(8), "九": float64(9), "玖": float64(9), "十": float64(10), "拾": float64(10),
+		"百": float64(100), "佰": float64(100), "千": float64(1000), "仟": float64(1000), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000),
+		"零": float64(0), "点": ".", "角": float64(0.1), "分": float64(0.01),
+	}
+	moneyUnit = map[string]float64{
+		"元": float64(1), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000), //单位
+	}
+)
+
+var currencyItem = map[string]string{
+	"人民币": "人民币",
+	"rmb": "人民币",
+	"RMB": "人民币",
+	"$":   "美元",
+	"＄":   "美元",
+	"美元":  "美元",
+	"港元":  "港币",
+	"港币":  "港币",
+	"澳币":  "澳币",
+	"澳元":  "澳币",
+}
+
+// 获取币种
+func GetCurrency(text string) (currency string) {
+	if text == "" {
+		return
+	}
+	currency = "人民币"
+	for k, v := range currencyItem {
+		if strings.Contains(text, k) {
+			currency = v
+			return
+		}
+	}
+	return
+}
+
+// 金额转换
+func ObjToMoney(text string) float64 {
+	isfindUnit := true
+	ret := capitalMoney(text)
+	if ret < float64(10000) || ret > float64(50000000000) {
+		ret2, b := numMoney(text)
+		isfindUnit = b
+		if ret2 > ret {
+			ret = ret2
+		}
+	}
+	f, _ := strconv.ParseFloat(strconv.FormatFloat(ret, 'f', 4, 64), 64)
+	// if f < 1 {
+	// 	f = 0
+	// }
+	//如果金额小于50，全文检索单位：万
+	if f < 50 && f > 0 && isfindUnit {
+		rep := contentUnit.FindAllStringIndex(text, -1)
+		if len(rep) > 0 {
+			f = f * 10000
+		}
+	}
+	return f
+}
+
+func capitalMoney(text string) float64 {
+	nodes := []float64{}
+	node := float64(0)
+	tmp := float64(0)
+	decimals := 0.0
+	ishaspoint := false //是否含小数点
+	fnum := float64(0)
+	end := false
+	//str := fmt.Sprint(data[0])
+	//提取第一个大写信息
+	strmatch := numCapitals.FindAllStringSubmatch(text, -1)
+	if len(strmatch) > 0 {
+		text = strmatch[0][0]
+	}
+	suffixUnit := float64(1)
+	if strings.HasSuffix(text, "万") || strings.HasSuffix(text, "万元") || strings.HasSuffix(text, "万元整") {
+		index := strings.LastIndex(text, "万")
+		text = text[0:index]
+		suffixUnit = float64(10000)
+	}
+	moneyRegChar.ReplaceAllStringFunc(text, func(key string) string {
+		if key == "元" || key == "圆" || key == "点" {
+			ishaspoint = true
+		}
+		if v, ok := moneyChar[key].(float64); ok && !end {
+			if ishaspoint && v > 10 { //排除后面有其他的单位
+				return ""
+			}
+			//fmt.Println(key, v, fnum)
+			if v < 10 && v >= 0 {
+				if ishaspoint { //小数部分
+					if v >= 1 {
+						fnum = v
+					} else if v < 1 && v > 0 {
+						decimals += fnum * v
+					}
+				} else {
+					if tmp != float64(0) {
+						node += tmp
+					}
+					tmp = float64(v)
+				}
+			} else if v == 10000 || v == 100000000 { //单位万、亿
+				if tmp != float64(0) {
+					node += tmp
+					tmp = float64(0)
+				}
+				nodes = append(nodes, node*float64(v))
+				node = float64(0)
+			} else {
+				if v == 10 && tmp == 0 {
+					tmp = 1
+				}
+				tmp = tmp * float64(v)
+				node += tmp
+				tmp = float64(0)
+			}
+		}
+		if key == "整" || key == "正" || key == "分" {
+			end = true
+		}
+		return ""
+	})
+	nodes = append(nodes, node, tmp)
+	ret := float64(0)
+	for _, v := range nodes {
+		ret += v
+	}
+	return (ret + decimals) * suffixUnit
+}
+
+// 数字金额转换
+func numMoney(text string) (moneyFloat float64, flag bool) {
+	//tmp := fmt.Sprintf("%f", data[0])
+	repUnit := float64(1)
+	if regQianw.MatchString(text) {
+		text = strings.Replace(text, "千万", "万", -1)
+		repUnit = float64(1000)
+	}
+	text = replaceSymbol(text, []string{",", "，", "（", "）", "(", ")", "：", "\n"})
+	text = replaceString(text, []string{"万元", "亿元", "．"}, []string{"万", "亿", "."})
+	text = CutAllSpace(text)
+	rets := regNumFloat.FindAllString(text, -1)
+	fnums := []float64{}
+	unitstrs := []string{}
+	if len(rets) > 0 {
+		pindex := 0 //单位前置
+		for k, v := range rets {
+			f, err := strconv.ParseFloat(v, 64)
+			if err == nil {
+				fnums = append(fnums, f)
+				index := strings.Index(text, v)
+				//单位后置
+				start := index + len(v)
+				end := start + 3
+				//log.Println("vvv", tmp, v, pindex, index, start)
+				if k > 0 {
+					if start >= pindex+3 {
+						pstart := pindex + 3
+						if pstart >= index {
+							pstart = index
+						}
+						if len(text) > end {
+							unitstrs = append(unitstrs, text[pstart:index]+text[start:end])
+						} else {
+							unitstrs = append(unitstrs, text[pstart:index]+text[start:])
+						}
+					} else {
+						if len(text) > end {
+							unitstrs = append(unitstrs, text[start:end])
+						} else {
+							unitstrs = append(unitstrs, text[start:])
+						}
+					}
+				} else {
+					if len(text) > end {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, text[index-3:index]+text[start:end])
+						} else {
+							unitstrs = append(unitstrs, text[start:end])
+						}
+					} else {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, text[index-3:index]+text[start:])
+						} else {
+							unitstrs = append(unitstrs, text[start:])
+						}
+					}
+				}
+				pindex = start
+			}
+		}
+	}
+	//log.Println("unitstrs", fnums, unitstrs)
+	unit := float64(0)
+	fnum := float64(0)
+	for k, v := range fnums {
+		fnum = v
+		units := regStrUnit.FindAllString(unitstrs[k], -1)
+		for _, v := range units {
+			if moneyUnit[v] != 0 {
+				unit = moneyUnit[v]
+				break
+			}
+		}
+		if unit != float64(0) { //取第一个
+			break
+		}
+	}
+	fnum = fnum * repUnit
+	if unit == float64(0) {
+		moneyFloat = fnum
+	} else {
+		moneyFloat = fnum * unit
+	}
+	if unit == 10000 {
+		flag = false
+	} else {
+		flag = true
+	}
+	return
+}
+
+// 清理所有空白符
+func CutAllSpace(text string) string {
+	tmp := cutAllSpace.ReplaceAllString(text, "")
+	tmp = replaceSymbol(tmp, spaces)
+	return tmp
+}
+
+// 符号替换
+func replaceString(con string, ret, rep []string) string {
+	for k, v := range ret {
+		if len(rep) > k {
+			con = strings.Replace(con, v, rep[k], -1)
+		}
+	}
+	return con
+}
+
+// 过滤符号
+func replaceSymbol(con string, rep []string) string {
+	for _, v := range rep {
+		con = strings.Replace(con, v, "", -1)
+	}
+	return con
+}
+
+// MatchField 判断valus 是否包含key 字符串
+func MatchField(keys []string, values []string) (ok bool) {
+	var matchKeyRegs []*RegexpInfo
+
+	if len(keys) > 0 {
+		for _, key := range keys {
+			KeyReg := GetRegex(key)
+			matchKeyRegs = append(matchKeyRegs, KeyReg...)
+		}
+
+		for _, reg := range matchKeyRegs {
+			for _, val := range values {
+				if reg.regs.MatchString(val) {
+					return true
+				}
+			}
+		}
+	}
+
+	return
+}
+
+// SumFields 计算map 包含的字段个数
+func SumFields(keys []string, val map[string]interface{}) (res int) {
+	for _, v := range keys {
+		if _, ok := val[v]; ok {
+			res++
+		}
+	}
+
+	return
+
+}
+
+// containsChinese  识别含有中文
+func containsChinese(str string) bool {
+	result, _ := regexp.MatchString(`[\x{4e00}-\x{9fa5}]+`, str)
+	if result {
+		return true
+	}
+
+	return false
+}

mobile_tag/go.mod

@@ -0,0 +1,32 @@
+module mobile_tag
+
+go 1.20
+
+require (
+	github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.10.1 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+)

mobile_tag/go.sum

@@ -0,0 +1,202 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16 h1:K6hILkGcqNRmTO1nsrpZMqG2iupJETmnmZmKt77SsNY=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

mobile_tag/main.go

@@ -0,0 +1,184 @@
+package main
+
+import (
+	"fmt"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"sync"
+	"time"
+)
+
+var (
+	Mgo        *mongodb.MongodbSim
+	Es         *elastic.Elastic
+	EsNew      *elastic.Elastic
+	Es3        *elastic.Elastic
+	MatchArr   = make([]TagMatching, 0) // 存放标签规则
+	globalRegs = make([]TagMatching, 0) //关键词规则，是标签规则大前提
+	//更新es
+	updateEsPool = make(chan []map[string]interface{}, 5000)
+	updateEsSp   = make(chan bool, 5) //保存协程
+	// 更新mongo
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp   = make(chan bool, 5)
+)
+
+func main() {
+	go updateEsMethod() // 更新es
+	go updateMethod()
+	Init()
+	//oss.InitOss()
+	InitRule()
+	taskRun()
+	log.Println("over")
+	c := make(chan bool, 1)
+	<-c
+}
+
+// taskRun 执行
+func taskRun() {
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	ch := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+
+	//查询条件
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			//"$gte": mongodb.StringTOBsonId("65d73ba366cf0db42aca6e2f"),
+			//"$lte": mongodb.StringTOBsonId("6612323266cf0db42a75789f"),
+			"$lte": mongodb.StringTOBsonId("5f00e67e52c1d9fbf8367996"), //测试环境
+		},
+	}
+	count := 0
+	selected := map[string]interface{}{"title": 1, "detail": 1, "projectname": 1, "purchasing": 1, "buyer": 1, "attach_text": 1}
+	it := sess.DB("qfw_data").C("bidding").Find(q).Select(selected).Sort("-_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%5000 == 0 {
+			log.Println("current:", count, tmp["_id"])
+		}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			// 存在附件时
+			//if atta, ok := tmp["attach_text"]; ok && atta != nil {
+			//	text := getFileText(tmp) // 附件信息
+			//	if text != "" {
+			//		tmp["filetext"] = text
+			//	}
+			//}
+
+			gs, _, _ := TaskTags(tmp, globalRegs)
+			if len(gs) > 0 {
+				tags, match, add := TaskTags(tmp, MatchArr)
+				if len(tags) > 0 {
+					update := map[string]interface{}{
+						"mobile_tag": tags,
+					}
+					log.Println("id--", mongodb.BsonIdToSId(tmp["_id"]), match, add)
+					//更新MongoDB
+					updatePool <- []map[string]interface{}{
+						{"_id": tmp["_id"]},
+						{"$set": update},
+					}
+
+					//Mgo.UpdateById("bidding", mongodb.BsonIdToSId(tmp["_id"]), map[string]interface{}{"$set": update})
+					//更新es
+					updateEsPool <- []map[string]interface{}{
+						{"_id": mongodb.BsonIdToSId(tmp["_id"])},
+						update,
+					}
+				}
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	fmt.Println("over ---> ", count)
+}
+
+// updateMethod 更新MongoDB
+func updateMethod() {
+	arru := make([][]map[string]interface{}, 50)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 50 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					Mgo.UpdateBulk("bidding", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 50)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					Mgo.UpdateBulk("bidding", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 50)
+				indexu = 0
+			}
+		}
+	}
+}
+
+// updateEsMethod 更新es
+func updateEsMethod() {
+	arru := make([][]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-updateEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk("bidding", arru...)
+					//EsNew.UpdateBulk("bidding", arru...)
+					//Es3.UpdateBulk("bidding", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk("bidding", arru...)
+					//Es3.UpdateBulk("bidding", arru...)
+					//EsNew.UpdateBulk("bidding", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}

mobile_tag/oss/oss_test.go

@@ -0,0 +1,12 @@
+package oss
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestOssGetObject(t *testing.T) {
+	InitOss()
+	res := OssGetObject("ed2d1b08-cdb3-11ee-a4b9-0242ac120006", "65d0e4f966cf0db42abe69b1")
+	fmt.Println("65d0e4f966cf0db42abe69b1->res:", res)
+}

mobile_tag/oss/ossclient.go

@@ -0,0 +1,110 @@
+package oss
+
+import (
+	"fmt"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"io/ioutil"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"os"
+	"strconv"
+)
+
+var (
+	ossEndpoint = "oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
+	//ossEndpoint        = "oss-cn-beijing.aliyuncs.com" //测试环境：oss-cn-beijing.aliyuncs.com
+	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
+	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
+	ossBucketName      = "topjy"
+	ossclient          *oss.Client
+)
+
+func InitOss() {
+	client, err := oss.New(ossEndpoint, ossAccessKeyId, ossAccessKeySecret)
+	if err != nil {
+		fmt.Println("Error:", err)
+		os.Exit(-1)
+	}
+	ossclient = client
+}
+
+func OssGetObject(objectName, id string) string {
+	util.Catch()
+	// 获取存储空间。
+	bucket, err := ossclient.Bucket(ossBucketName)
+	if err != nil {
+		return ""
+	}
+
+	// 下载文件到流。
+	body, err := bucket.GetObject(objectName)
+	if err != nil {
+		//log.Info("OssGetObject", zap.String("key", objectName), zap.String("id", id))
+		//log.Info("OssGetObject", zap.Error(err))
+		return ""
+	}
+	defer body.Close()
+	data, err := ioutil.ReadAll(body)
+	if err != nil {
+		fmt.Println("ReadAll", err)
+		return ""
+	}
+	return string(data)
+}
+
+func OssObjExists(bname, fid string) bool {
+	util.Catch()
+	// 获取存储空间。
+	bucket, err := ossclient.Bucket(bname)
+	if err != nil {
+		fmt.Println("Error:", err)
+	}
+	// 判断文件是否存在。
+	isExist, err := bucket.IsObjectExist(fid)
+	if err != nil {
+		fmt.Println("Error:", err)
+	}
+	return isExist
+}
+
+// OssFileSize 获取文件大小
+func OssFileSize(name string) bool {
+	util.Catch()
+	// 获取存储空间。
+	bucket, err := ossclient.Bucket(ossBucketName)
+	if err != nil {
+		fmt.Println("OssFileSize,Bucket err", err)
+	}
+	// 获取文件的元信息
+	props, err := bucket.GetObjectDetailedMeta(name)
+	if err != nil {
+		fmt.Println("Error getting object metadata:", err)
+	}
+
+	// 提取文件大小信息
+	fileSizeStr := props.Get("Content-Length")
+	fmt.Printf("%T,%+v \n", fileSizeStr, fileSizeStr)
+	// 将文件大小字符串解析为float64类型
+	fileSize, err := parseFileSize(fileSizeStr)
+	fmt.Println("fileSize", fileSize)
+	if err != nil {
+		fmt.Println("Error parsing file size,", err)
+	}
+	// 判断文件大小是否超过20兆
+	if fileSize > 20*1024*1024 { // 将20兆转换为字节
+		fmt.Println("文件大小超过20兆")
+		return true
+	} else {
+		fmt.Println("文件大小未超过20兆")
+		return false
+	}
+
+}
+
+// 解析文件大小字符串，将其转换为 int 类型
+func parseFileSize(sizeStr string) (int, error) {
+	fileSize, err := strconv.Atoi(sizeStr)
+	if err != nil {
+		return 0, err
+	}
+	return fileSize, nil
+}

mobile_tag/readme.md

@@ -0,0 +1,4 @@
+
+需求：P513中国移动定制招标采购搜索，根据规则打标记
+
+

mobile_tag/util.go

@@ -0,0 +1,185 @@
+package main
+
+import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"mobile_tag/oss"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+type TagMatching struct {
+	tagName       string        //标签名称
+	tagCode       string        //标签值(保存)
+	matchField    []string      //关键词匹配字段,title,detail
+	matchKey      string        //匹配的词语，多个使用逗号连接,"部队,国防,军事,军用"
+	matchKeyReg   []*RegexpInfo //关键词的正则
+	addField      []string      //附加词匹配字段
+	addKey        string        //附件词匹配关键词
+	addKeyReg     []*RegexpInfo //附加次的正则
+	excludeField  []string      //排除词匹配字段
+	excludeKey    string        //排除词匹配词
+	excludeKeyReg []*RegexpInfo //排除词正则
+	//clearKey      []string      //清理词匹配字段跟关键词一样
+}
+
+type RegexpInfo struct {
+	keyStr string
+	regs   *regexp.Regexp
+}
+
+// GetRegex 根据关键词或者对应正则
+func GetRegex(key string) []*RegexpInfo {
+	var infos []*RegexpInfo
+	for _, s := range strings.Split(key, ",") {
+		if strings.Contains(s, "&&") || strings.Contains(s, "&!") {
+			info := &RegexpInfo{
+				keyStr: s,
+				regs:   nil,
+			}
+			infos = append(infos, info)
+		} else {
+			info := &RegexpInfo{
+				keyStr: s,
+				regs:   regexp.MustCompile(".*(?i)" + s + ".*"),
+			}
+			infos = append(infos, info)
+		}
+
+	}
+	return infos
+}
+
+// TaskTags 根据数据和正则规则，验证数据标签
+func TaskTags(tmp map[string]interface{}, regs []TagMatching) (tags []string, keyWord, addWord string) {
+	// 在匹配字段里，比如标题满足了关键词，详情满足附加词,关键词的匹配字段含有标题，附加词的匹配字段含有详情；就符合条件
+Loop:
+	for _, v := range regs {
+		keyR := false // 关键词匹配结果
+		addR := false //附加次匹配结果
+		// 1.排除词
+		if len(v.excludeField) > 0 && len(v.excludeKeyReg) > 0 {
+			// 遍历排除词对应的tmp字段信息
+			for _, f := range v.excludeField {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					if rs, _ := getRegsResult(val, v.excludeKeyReg); rs {
+						//有排除词，直接判断下一个规则
+						continue Loop
+					}
+				}
+			}
+		}
+
+		// 清理词；目的把 类似 fuck的单词替换为空字符串
+		//if len(v.clearKey) > 0 && len(v.matchField) > 0 {
+		//	for _, s := range v.clearKey {
+		//		for _, f := range v.matchField {
+		//			if val := util.ObjToString(tmp[f]); val != "" {
+		//				tmp[f] = strings.ReplaceAll(val, s, "")
+		//			}
+		//		}
+		//	}
+		//}
+
+		// 关键词
+		if len(v.matchField) > 0 && len(v.matchKeyReg) > 0 {
+			for _, f := range v.matchField {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					if rs, da := getRegsResult(val, v.matchKeyReg); rs {
+						keyR = true
+						keyWord = da
+						//log.Print("key", da)
+						break
+					}
+				}
+			}
+		}
+
+		// 附加词
+		if len(v.addField) > 0 && len(v.addKeyReg) > 0 && keyR {
+			for _, f := range v.addField {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					if rs, da := getRegsResult(val, v.addKeyReg); rs {
+						addR = true
+						addWord = da
+						//log.Println("add", da)
+						break
+					}
+				}
+			}
+		} else {
+			addR = true
+		}
+
+		// 满足 关键词和附加词条件
+		if keyR && addR {
+			// 去重相同标签
+			if !IsInStringArray(v.tagName, tags) {
+				tags = append(tags, v.tagName)
+			}
+		}
+	}
+
+	return
+}
+
+// getRegsResult 验证数据是否符合数组正则
+func getRegsResult(data string, regs []*RegexpInfo) (res bool, a string) {
+	for _, e1 := range regs {
+		if e1.regs != nil && e1.regs.MatchString(data) {
+			return true, e1.regs.String()
+		} else {
+			// && 特殊处理
+			if strings.Contains(e1.keyStr, "&&") {
+				flag := true
+				for _, s := range strings.Split(e1.keyStr, "&&") {
+					if !strings.Contains(data, s) {
+						flag = false
+						break
+					}
+				}
+				if flag {
+					return true, e1.keyStr
+				}
+			}
+			// 前面是必须有的关键词&!，后面是不能有的关键词;比如 军队&!点军队，
+			if strings.Contains(e1.keyStr, "&!") {
+				keys := strings.Split(e1.keyStr, "&!")
+				if strings.Contains(data, keys[0]) && !strings.Contains(data, keys[1]) {
+					return true, e1.keyStr
+				}
+			}
+		}
+	}
+	return false, ""
+}
+
+// IsInStringArray 判断数组中是否存在字符串
+func IsInStringArray(str string, arr []string) bool {
+	// 先对字符串数组进行排序
+	sort.Strings(arr)
+	// 使用二分查找算法查找字符串
+	pos := sort.SearchStrings(arr, str)
+	// 如果找到了则返回 true，否则返回 false
+	return pos < len(arr) && arr[pos] == str
+}
+
+// getFileText 获取附件内容
+func getFileText(tmp map[string]interface{}) (filetext string) {
+	if attchMap, ok := tmp["attach_text"].(map[string]interface{}); attchMap != nil && ok {
+		for _, tmpData1 := range attchMap {
+			if tmpData2, ok := tmpData1.(map[string]interface{}); tmpData2 != nil && ok {
+				for _, result := range tmpData2 {
+					if resultMap, ok := result.(map[string]interface{}); resultMap != nil && ok {
+						if attach_url := util.ObjToString(resultMap["attach_url"]); attach_url != "" {
+							bs := oss.OssGetObject(attach_url, mongodb.BsonIdToSId(tmp["_id"])) //oss读数据
+							filetext += bs
+						}
+					}
+				}
+			}
+		}
+	}
+	return
+}

project_package/go.mod

@@ -0,0 +1,35 @@
+module project_package
+
+go 1.20
+
+require (
+	github.com/RoaringBitmap/roaring v1.9.0
+	github.com/cespare/xxhash/v2 v2.2.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/bits-and-blooms/bitset v1.12.0 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/mschoch/smat v0.2.0 // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.10.1 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+)

project_package/go.sum

@@ -0,0 +1,212 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/RoaringBitmap/roaring v1.9.0 h1:lwKhr90/j0jVXJyh5X+vQN1VVn77rQFfYnh6RDRGCcE=
+github.com/RoaringBitmap/roaring v1.9.0/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
+github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
+github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16 h1:K6hILkGcqNRmTO1nsrpZMqG2iupJETmnmZmKt77SsNY=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40 h1:xTeRmpFgwOdu+NbWg/YntX3MnQpttm7jj33C1+JdBTk=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

project_package/main.go

@@ -0,0 +1,415 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/RoaringBitmap/roaring"
+	"github.com/cespare/xxhash/v2"
+	"io/ioutil"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"os"
+	"os/signal"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+var (
+	dbfile      = flag.String("dbfile", "./db", "数据库文件")
+	cache       = roaring.NewBitmap()
+	cacheModify = false    //控制10秒 定时写入文件
+	mutex       sync.Mutex // 互斥锁，用于保护 cache 的并发写入操作
+)
+
+func init() {
+	_, err := os.Stat(*dbfile)
+	if !os.IsNotExist(err) {
+		bs, err := ioutil.ReadFile(*dbfile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if len(bs) > 0 {
+			cache.FromBuffer(bs)
+		}
+	}
+	//监听，写入文件保存
+	go func() {
+		for {
+			time.Sleep(10 * time.Second)
+			if cacheModify {
+				saveDb()
+				cacheModify = false
+			}
+		}
+	}()
+}
+
+func main() {
+	MgoB := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.85:27080",
+		//MongodbAddr: "192.168.3.206:27002",
+		Size:   10,
+		DbName: "qfw",
+		//UserName:    "root",
+		//Password:    "root",
+	}
+	MgoB.InitPool()
+
+	Es := &elastic.Elastic{
+		//S_esurl: "http://192.168.3.149:9201",
+		S_esurl:  "http://172.17.4.184:19805",
+		I_size:   5,
+		Username: "es_all",
+		Password: "TopJkO2E_d1x",
+	}
+	Es.InitElasticSize()
+
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+	//
+
+	where := map[string]interface{}{
+		"pici": map[string]interface{}{
+			//"$gte": 0,
+			//"$lte": 1710381363,
+			"$gt":  1710381363,
+			"$lte": 1710468406,
+		},
+	}
+	query := sess.DB("qfw").C("projectset_20230904").Find(where).Select(nil).Iter()
+	count := 0
+	lastID := ""
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		id := mongodb.BsonIdToSId(tmp["_id"])
+		lastID = id
+		if count%10000 == 0 {
+			log.Println("current:", count, id)
+		}
+
+		name := getNewName(tmp)
+		if name != "" {
+			update := make(map[string]interface{})
+			update["subtitle_projectname"] = name
+			MgoB.UpdateById("projectset_20230904", id, map[string]interface{}{"$set": update})
+			//Es.UpdateDocument("projectset", id, update)
+		}
+
+		tmp = make(map[string]interface{})
+	}
+	log.Println("结束", lastID)
+
+	//监听异常退出信号；及时保存数据
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
+	<-signalChan
+	log.Println("程序退出")
+	saveDb()
+}
+
+// getNewName 获取新的不重复名称
+func getNewName(tmp map[string]interface{}) string {
+	projectName := util.ObjToString(tmp["projectname"])
+	projectCode := util.ObjToString(tmp["projectcode"])
+	buyer := util.ObjToString(tmp["buyer"])
+	firsttime := util.Int64All(tmp["firsttime"])
+	createtime := util.Int64All(tmp["createtime"])
+	var projectDate, createDate string
+	if firsttime > 0 {
+		projectDate = time.Unix(firsttime, 0).Format("2006-01-02")
+	}
+	if createtime > 0 {
+		createDate = time.Unix(createtime, 0).Format("2006-01-02")
+	}
+
+	var matchWords = make([]string, 0)
+	if list, ok := tmp["list"].([]interface{}); ok {
+		if len(list) > 0 {
+			for _, v := range list {
+				if da, ok := v.(map[string]interface{}); ok {
+					title := util.ObjToString(da["title"])
+					// 使用正则表达式进行匹配
+					matches := GetPackages(title)
+					for _, v := range matches {
+						if !IsInStringArray(v, matchWords) {
+							matchWords = append(matchWords, v)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	//pks := removeDuplicates(matchWords)
+	packages := strings.Join(matchWords, "、")
+	return RenameProjectName(projectName, projectCode, packages, projectDate, buyer, createDate)
+}
+
+// saveDb 文件写入
+func saveDb() {
+	mutex.Lock()
+	defer mutex.Unlock()
+	// 如果 cache 为空，则无需执行写入操作
+	if cache.GetCardinality() == 0 {
+		return
+	}
+	fo, err := os.OpenFile(*dbfile, os.O_CREATE|os.O_RDWR|os.O_SYNC|os.O_TRUNC, 0777)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fo.Close()
+	cache.WriteTo(fo)
+}
+
+// hash 计算hash
+func hash(src string) uint64 {
+	return xxhash.Sum64String(src)
+}
+
+// RenameProjectName 获取新的不重复的项目名称
+func RenameProjectName(projectName, projectCode, packages, projectDate, buyer, createDate string) (newName string) {
+	//TODO 1.判断项目名称是否重复
+	var id uint64
+	defer func() {
+		if id > 0 && newName != "" {
+			cache.Add(uint32(id))
+			cacheModify = true
+		}
+	}()
+
+	//1.项目名称
+	if projectName != "" {
+		id = hash(projectName)
+		if !cache.Contains(uint32(id)) {
+			newName = projectName
+			return projectName
+		}
+	}
+	//TODO 2.1	项目名称+项目编码
+	if projectCode != "" {
+		newName = projectName + "_" + projectCode
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 2.2	项目名称+分包信息
+	if packages != "" {
+		newName = projectName + "_" + packages
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 2.3	项目名称+项目时间
+	if projectDate != "" {
+		newName = projectName + "_" + projectDate
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+
+	//TODO 2.4	项目名称+采购单位名称
+	if buyer != "" {
+		newName = projectName + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 3.1 项目名称+项目编码+分包信息
+	if projectCode != "" && packages != "" {
+		newName = projectName + "_" + projectCode + "_" + packages
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 3.2 项目名称+项目编码+项目时间
+	if projectCode != "" && projectDate != "" {
+		newName = projectName + "_" + projectCode + "_" + projectDate
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 3.3 项目名称+项目编码+采购单位
+	if projectCode != "" && buyer != "" {
+		newName = projectName + "_" + projectCode + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+
+	//TODO 3.4 项目名称+分包+项目时间
+	if packages != "" && projectDate != "" {
+		newName = projectName + "_" + packages + "_" + projectDate
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 3.5 项目名称+分包+采购单位
+	if packages != "" && buyer != "" {
+		newName = projectName + "_" + packages + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 3.6 项目名称+项目时间+采购单位
+	if projectDate != "" && buyer != "" {
+		newName = projectName + "_" + projectDate + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+
+	//TODO 4.1 项目名称+项目编码+分包信息+项目时间
+	if projectCode != "" && packages != "" && projectDate != "" {
+		newName = projectName + "_" + projectCode + "_" + packages + "_" + projectDate
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 4.2 项目名称+项目编码+分包信息+采购单位
+	if projectCode != "" && packages != "" && buyer != "" {
+		newName = projectName + "_" + projectCode + "_" + packages + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+
+	//TODO 5 项目名称+项目编码+分包信息+项目时间+采购单位
+	if projectCode != "" && packages != "" && projectDate != "" && buyer != "" {
+		newName = projectName + "_" + projectCode + "_" + packages + "_" + projectDate + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	} else {
+		newName = projectName + "_" + projectCode + "_" + packages + "_" + projectDate + "_" + buyer + "_" + createDate
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		} else {
+			newName = ""
+		}
+	}
+
+	return
+}
+
+// GetPackages 获取对应的分包
+func GetPackages(title string) (res []string) {
+	// 定义正则表达式
+	rea := regexp.MustCompile(`包\d{1,2}[-~、]\d{1,2}|\d{1,2}[-~、]\d{1,2}包`) //1-6包；01-06包；01、02包;包1、包2
+	//text := "中国绿发投资集团有限公司直属项目公司2023年第20批集中采购非招标项目（包10、12、14、17、18、19"
+	packages := rea.FindAllString(util.ObjToString(title), -1) //匹配的包
+
+	if len(packages) > 0 {
+		res = append(res, packages...)
+	}
+
+	reb := regexp.MustCompile(`(标段[1-9一二三四五六七八九]|[1-9一二三四五六七八九]标段|包[1-9一二三四五六七八九]?[0-9]|[1-9一二三四五六七八九]?[0-9]包|[a-kA-K]包)`) // 标题只有一个包2
+	pgs := reb.FindAllString(title, -1)
+	if len(pgs) > 0 {
+		for _, v := range pgs {
+			if !IsInStringArray(v, res) {
+				res = append(res, v)
+			}
+		}
+	}
+	return res
+}
+
+// IsInStringArray 判断数组中是否存在字符串
+func IsInStringArray(str string, arr []string) bool {
+	// 先对字符串数组进行排序
+	sort.Strings(arr)
+	// 使用二分查找算法查找字符串
+	pos := sort.SearchStrings(arr, str)
+	// 如果找到了则返回 true，否则返回 false
+	return pos < len(arr) && arr[pos] == str
+}
+
+// removeDuplicates 去除数据中重复包，并合并连续数字包
+func removeDuplicates(data []string) []string {
+	// 存储已存在的包号
+	existingPackages := make(map[int]bool)
+
+	// 存储包含包号信息的字符串
+	packages := make(map[int]string)
+
+	// 匹配包号的正则表达式
+	re := regexp.MustCompile(`(包)(\d+)(?:-(\d+))?`)
+
+	noexists := make([]string, 0)
+	// 遍历数据
+	for _, item := range data {
+		// 提取包号信息
+		matches := re.FindStringSubmatch(item)
+		if len(matches) < 3 {
+			noexists = append(noexists, item)
+			continue
+		}
+		// 解析包号
+		start, _ := strconv.Atoi(matches[2])
+		end := start
+		if len(matches[3]) > 0 {
+			end, _ = strconv.Atoi(matches[3])
+		}
+		// 添加到已存在的包号中
+		for i := start; i <= end; i++ {
+			existingPackages[i] = true
+		}
+		// 将包含包号信息的字符串存储到 packages 中
+		packages[start] = matches[0]
+	}
+
+	// 从 map 中提取去重后的包号并排序
+	var uniquePackages []int
+	for packageNum := range existingPackages {
+		uniquePackages = append(uniquePackages, packageNum)
+	}
+	sort.Ints(uniquePackages)
+
+	// 将连续的包号转换为包含范围的字符串
+	var result []string
+	var start, end int
+	for i, num := range uniquePackages {
+		if i == 0 {
+			start = num
+			end = num
+		} else if num == end+1 {
+			end = num
+		} else {
+			if start == end {
+				result = append(result, packages[start])
+			} else {
+				result = append(result, fmt.Sprintf("包%d-%d", start, end))
+			}
+			start = num
+			end = num
+		}
+	}
+	if start == end {
+		result = append(result, packages[start])
+	} else {
+		result = append(result, fmt.Sprintf("包%d-%d", start, end))
+	}
+
+	result = append(result, noexists...)
+	return result
+}

project_package/pn_test.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestGetPackages(t *testing.T) {
+	title := "汕头市潮阳区贵屿镇北港河底泥修复工程监理及验收服务(采购项目编号：440513-201811-112-0010)招标公告,包2、包4、包2"
+	fmt.Println(GetPackages(title))
+	//fmt.Println(removeDuplicates(GetPackages(title)))
+}

project_package/readme.txt

@@ -0,0 +1,3 @@
+
+为了解决项目名称重复问题，需要从分包、项目编号、日期时间、采购单位 四个维度去补充完善，
+然后拼接在项目名称后面，存储一个新字段。

renamepn/README.md

@@ -0,0 +1,51 @@
+剑鱼项目重命名服务
+
+根据提供项目信息，返回一个和已有项目不重复的项目名称
+
+默认端口：8088；若需要更改代码，启动时 附带参数 `-p :8888`
+
+默认数据存储文件：  `./db`;当前路径下 `db`文件，若需更改启动时附带参数 `-dbfile ./dbfile`，
+
+
+请求参数示例：
+```json
+{
+  "project_name":"东关中心幼儿园食堂桌椅采购",
+  "project_code":"2023-01-01",
+  "buyer":"济南市历下区东关中心幼儿园",
+  "project_date":"2021-12-11",
+  "packages":"分包1、分包3"
+}
+```
+参数解释：
+
+|     字段名称     | 字段类型   | 备注                                         |
+|:------------:|--------|--------------------------------------------|
+| project_name | string | 原有的项目名称                                    |
+| project_code | string | 项目编码                                       |
+|    buyer     | string | 采购单位                                       |
+| project_date | string | 年月日 格式的字符串;"2022-11-12"                    |
+|   packages   | string | 项目中的分包信息；例如"分包1","分包3、分包8","包1-3","1-3包" 等 |
+
+
+
+返回示例：
+
+```json
+{
+  "code": 0,
+  "data": "东关中心幼儿园食堂桌椅采购_2023-01-01",
+  "msg": "请求成功"
+}
+```
+
+失败样例:
+
+```json
+{
+    "code": -1,
+    "data": "",
+    "msg": "提供参数，无法获取到唯一不重复项目名称"
+}
+```
+

renamepn/go.mod

@@ -0,0 +1,44 @@
+module renamepn
+
+go 1.19
+
+require (
+	github.com/RoaringBitmap/roaring v1.9.0
+	github.com/cespare/xxhash/v2 v2.2.0
+	github.com/gin-gonic/gin v1.9.1
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/bits-and-blooms/bitset v1.12.0 // indirect
+	github.com/bytedance/sonic v1.11.2 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
+	github.com/chenzhuoyu/iasm v0.9.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.19.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mschoch/smat v0.2.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	golang.org/x/arch v0.7.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16 // indirect
+)

renamepn/go.sum

@@ -0,0 +1,257 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/RoaringBitmap/roaring v1.9.0 h1:lwKhr90/j0jVXJyh5X+vQN1VVn77rQFfYnh6RDRGCcE=
+github.com/RoaringBitmap/roaring v1.9.0/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
+github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
+github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
+github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A=
+github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
+github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
+github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
+github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA=
+github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
+github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0=
+github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
+github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=
+github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
+github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
+github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
+github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
+github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
+github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc=
+golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16 h1:K6hILkGcqNRmTO1nsrpZMqG2iupJETmnmZmKt77SsNY=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

renamepn/main.go

@@ -0,0 +1,260 @@
+package main
+
+import (
+	"flag"
+	"github.com/gin-gonic/gin"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/RoaringBitmap/roaring"
+	"github.com/cespare/xxhash/v2"
+)
+
+var (
+	dbfile      = flag.String("dbfile", "./db", "数据库文件")
+	addr        = flag.String("p", ":8088", "监听地址")
+	cache       = roaring.NewBitmap()
+	cacheModify = false //控制10秒 定时写入文件
+)
+
+type ProjectData struct {
+	ProjectName string `json:"project_name"`
+	ProjectCode string `json:"project_code"`
+	Buyer       string `json:"buyer"`
+	ProjectDate string `json:"project_date"`
+	Packages    string `json:"packages"`
+}
+
+// init 初始化
+func init() {
+	flag.Parse()
+	_, err := os.Stat(*dbfile)
+	if !os.IsNotExist(err) {
+		bs, err := ioutil.ReadFile(*dbfile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if len(bs) > 0 {
+			cache.FromBuffer(bs)
+		}
+	}
+
+	//监听，写入文件保存
+	go func() {
+		for {
+			time.Sleep(10 * time.Second)
+			if cacheModify {
+				saveDb()
+				cacheModify = false
+			}
+		}
+	}()
+}
+
+func main() {
+	// 设置 Gin 模式为发布模式，关闭每次路由请求打印
+	gin.SetMode(gin.ReleaseMode)
+	gin.DefaultWriter = io.Discard
+
+	router := gin.Default()
+	router.POST("/rename", getRename)
+	log.Println("程序监听端口", *addr)
+
+	go func() {
+		err := router.Run(*addr)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}()
+
+	//监听异常退出信号；及时保存数据
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
+	<-signalChan
+	log.Println("程序退出")
+	saveDb()
+}
+
+// saveDb 文件写入
+func saveDb() {
+	fo, err := os.OpenFile(*dbfile, os.O_CREATE|os.O_RDWR|os.O_SYNC|os.O_TRUNC, 0777)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fo.Close()
+	cache.WriteTo(fo)
+}
+
+// hash 计算hash
+func hash(src string) uint64 {
+	return xxhash.Sum64String(src)
+}
+
+// RenameProjectName 获取新的不重复的项目名称
+func RenameProjectName(projectName, projectCode, packages, projectDate, buyer string) string {
+	//TODO 1.判断项目名称是否重复
+	var id uint64
+	defer func() {
+		cache.Add(uint32(id))
+		cacheModify = true
+	}()
+
+	//1.项目名称
+	if projectName != "" {
+		id = hash(projectName)
+		if !cache.Contains(uint32(id)) {
+			return projectName
+		}
+	}
+	//TODO 2.1	项目名称+项目编码
+	if projectCode != "" {
+		newName := projectName + "_" + projectCode
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 2.2	项目名称+分包信息
+	if packages != "" {
+		newName := projectName + "_" + packages
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 2.3	项目名称+项目时间
+	if projectDate != "" {
+		newName := projectName + "_" + projectDate
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 2.4	项目名称+采购单位名称
+	if buyer != "" {
+		newName := projectName + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 3.1 项目名称+项目编码+分包信息
+	if projectCode != "" && packages != "" {
+		newName := projectName + "_" + projectCode + "_" + packages
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 3.2 项目名称+项目编码+项目时间
+	if projectCode != "" && projectDate != "" {
+		newName := projectName + "_" + projectCode + "_" + projectDate
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 3.3 项目名称+项目编码+采购单位
+	if projectCode != "" && buyer != "" {
+		newName := projectName + "_" + projectCode + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+
+	//TODO 3.4 项目名称+分包+项目时间
+	if packages != "" && projectDate != "" {
+		newName := projectName + "_" + packages + "_" + projectDate
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 3.5 项目名称+分包+采购单位
+	if packages != "" && buyer != "" {
+		newName := projectName + "_" + packages + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 3.6 项目名称+项目时间+采购单位
+	if projectDate != "" && buyer != "" {
+		newName := projectName + "_" + projectDate + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+
+	//TODO 4.1 项目名称+项目编码+分包信息+项目时间
+	if projectCode != "" && packages != "" && projectDate != "" {
+		newName := projectName + "_" + projectCode + "_" + packages + "_" + projectDate
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+	//TODO 4.2 项目名称+项目编码+分包信息+采购单位
+	if projectCode != "" && packages != "" && buyer != "" {
+		newName := projectName + "_" + projectCode + "_" + packages + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+
+	//TODO 5 项目名称+项目编码+分包信息+项目时间+采购单位
+	if projectCode != "" && packages != "" && projectDate != "" && buyer != "" {
+		newName := projectName + "_" + projectCode + "_" + packages + "_" + projectDate + "_" + buyer
+		id = hash(newName)
+		if !cache.Contains(uint32(id)) {
+			return newName
+		}
+	}
+
+	now := time.Now().Format("2006-01-02")
+	newName := projectName + "_" + projectCode + "_" + packages + "_" + projectDate + "_" + buyer + "_" + now
+	id = hash(newName)
+	if !cache.Contains(uint32(id)) {
+		return newName
+	}
+	return ""
+}
+
+// getRename 获取新的不重复名称
+func getRename(c *gin.Context) {
+	var data = ProjectData{}
+	err := c.ShouldBindJSON(&data)
+	if err != nil {
+		log.Println("ShouldBindJSON", err)
+		c.JSON(200, map[string]interface{}{
+			"code": 0,
+			"msg":  "参数识别错误，请检查参数",
+			"data": nil,
+		})
+		return
+	}
+
+	newName := RenameProjectName(data.ProjectName, data.ProjectCode, data.Packages, data.ProjectDate, data.Buyer)
+	if newName != "" {
+		c.JSON(200, map[string]interface{}{
+			"code": 0,
+			"msg":  "请求成功",
+			"data": newName,
+		})
+	} else {
+		c.JSON(200, map[string]interface{}{
+			"code": -1,
+			"msg":  "提供参数，无法获取到唯一不重复项目名称",
+			"data": newName,
+		})
+	}
+
+}

test/go.mod

@@ -0,0 +1,3 @@
+module test
+
+go 1.20

test/main.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"log"
+	"strings"
+)
+
+func main() {
+	name := "宣威市西泽乡社会保障服务中心"
+	rs := ruleBuyer(name)
+	log.Println("rs", rs)
+}
+
+// ruleBuyer 判断采购单位正确,名称错误返回true，名称正确返回false
+func ruleBuyer(input string) (res bool) {
+	//开头的单词
+	preWords := []string{"转发", "省", "会议室", "某", "某单位", "某公司", "某学校", "测试", "某大学", "某医院", "拟建", "机械", "县", "市", "区", "巡察", "人民路", "对", "楼", "家长", "春季", "对", "度", "初级中学", "州", "总部", "一号", "含", "育场", "选择", "第包", "医院",
+		"共赢资产", "从", "实验中学", "公共", "乘用车", "部与", "一直", "一体化", "现对", "现为", "之", "份", "半坡", "下属", "程", "信息", "带", "工务", "技术", "于", "号", "辖属", "造", "计财", "实验", "原", "器械", "是", "师", "雨山区", "云城区", "阜涣", "公联", "批",
+		"总承包", "姓市", "秋季学期", "哈巴河", "保险", "开发区", "尘设资", "团", "物业", "农村", "部", "二", "后勤", "位于", "选定", "依法", "口", "代表", "这", "逝沮省", "上半年", "第一", "门市", "配套", "八月", "七月", "集回", "维修", "遴选", "但", "增加",
+		"取消", "采购", "一直", "一体化", "现对", "现为", "之", "份", "取消", "转发", "省", "会议室", "某", "某单位", "某公司", "某学校", "测试", "某大学", "某医院", "拟建", "机械", "县", "市", "区", "巡察", "人民", "对", "楼", "家长", "春季", "于", "度", "初级中学", "州", "总部", "一号", "含", "育场", "选择", "第包", "医院",
+		"共赢资产", "从", "实验中学", "公共", "乘用车", "部与", "一直", "一体化", "现对", "现为", "之", "份", "半坡", "下属", "程", "信息", "带", "工务", "技术", "于", "号", "辖属", "造", "计财", "实验", "原", "器械", "是", "师", "阜涣", "公联", "批",
+		"总承包", "姓市", "秋季学期", "哈巴河", "保险", "开发区", "尘设资", "团", "物业", "农村", "部", "东省", "于", "章", "董", "分局", "再", "微", "明", "建", "公司", "审计", "满足", "中学", "届", "大学", "按摩", "中心", "附属", "总", "教育局", "中学", "小学", "学院",
+		"中医院", "人医院", "学校", "输", "十六", "有限", "节能", "二公司", "分公司", "子公司", "公司", "审计", "河河", "地公司", "六公司", "三公司", "十六公司", "节能", "股份", "运输公司", "堇", "家纺", "港区", "检查"}
+	//开头关键词
+	for _, word := range preWords {
+		if strings.HasPrefix(input, word) {
+			res = true
+			log.Println(word)
+			return res
+		}
+	}
+
+	//结尾关键词
+	suffixes := []string{"院院", "局局", "场场", "小学室", "和社", "人屡政府", "谈话室", "小区局", "今作社", "点场", "人昆政府",
+		"年度室", "分行银行", "人政府", "人民矢院", "教学楼局", "笺理局", "地场", "人民唉院", "瞥理局", "所院", "农业衣村局",
+		"民丢局", "委员会老干部", "办非处", "等支局", "监督站局", "停车库局", "检查员", "办直处", "进行局", "楼局", "等局", "人民政府社",
+		"模块局", "人民政俯", "人民医标院", "农业农局", "园局", "规则局", "人乓政府", "人事条局", "箐理所", "在政府", "重点局", "大学入场",
+		"入场", "发也站", "赘源局", "计划生育服务中心政府", "第一人民吹院", "有限公司室", "教标育局", "一民政府", "场部", "埋局", "大学大学",
+		"分公司分公司", "医院医院", "测试", "公司", "市", "显微镜", "段", "现场局", "租赁"}
+	for _, word := range suffixes {
+		if strings.HasSuffix(input, word) {
+			res = true
+			log.Println(word)
+			return res
+		}
+	}
+
+	//包含的关键词
+	specials := []string{"丿", "艹", "[ ", "【", "?", "亻", "#", "~", "^", "亻", "*", "$", "、", "/", "*",
+		"<>", "[ ", "【", "?", "某", "我", "第一轮", "第一次", "第一词", "第一季", "各学校", "第一批", "完全学校",
+		"一致同意", "X", "保体障局", "人们政府", "上设局", "开发茎", "场场", "设保", "武鳖", "集回", "项日", "代理银行",
+		"辽宁省省会", "菖处", "口国", "人民汰院", "肉州市", "火车站社", "自然资源和规局", "萎员会", "经济技术开发茎",
+		"（）", "()", "有限公司厂", "测试", "建设建设", "建设规模", "满足", "郑州郑州", "学校学校", "&", ";", "；",
+		"就医院", "酒医院", "至医院", "对", "所需", "提升", "提高", "学期", "学年", "规模", "必须", "建成", "各公司",
+		"养护", "拿", "艹", "关于", "要求", "举办", "显示所", "右阴公司", "发射", "加装", "扩建", "道路维修", "等所",
+		"维稳中心", "异议", "任务", "分包", "其他", "行为", "中标人", "教室", "安置", "暂行", "选聘", "合同", "转发",
+		"勿救", "既有", "装置", "开展", "依据", "指定", "一般用", "采供", "修缮", "招标单位", "联系方式", "沟通",
+		"该项目", "安装", "造价", "改造", "样品", "课室", "水毁道路", "闲置资产", "某单位", "某公司", "某学校", "测试", "某大学", "某医院", "地产鲨", "项目组",
+		"公国司", "物资", "评估与医院", "年室", "招标", "设有", "复限公司", "服务", "罩套", "有阴公", "人队", "报告",
+		"显微镜", "现场局", "租赁", "镇", "市中心", "勐阿糖厂", "展厅", "等医院", "广场室", "水利顷", "市室", "交界处", "自助银行",
+		"若源局", "布局", "联合体", "气室", "活动场", "包装公司", "投资", "有限", "等", "设备", "急件", "核实", "测汞仪", "黑名单", "制作类",
+		"大学大学", "学院学院", "大学场", "学校室", "部队部队", "内部", "部分支公司", "短管", "投标文件", "汇编", "收购"}
+
+	for _, v := range specials {
+		if strings.Contains(input, v) {
+			res = true
+			log.Println(v)
+			return res
+		}
+	}
+
+	return res
+
+}

test/test_test.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestA(T *testing.T) {
+	values := []int{1, 2, 3, 4, 5, 6, 7, 8, 9}
+	ch := make(chan int)
+	for _, v := range values {
+		go func(v int) {
+			ch <- v
+		}(v)
+	}
+	for i := 0; i < len(values); i++ {
+		fmt.Println(<-ch)
+	}
+
+}
+
+func TestB(T *testing.T) {
+	CollArr := []string{"company_base", "company_employee", "company_history_name", "company_partner", "annual_report_base", "annual_report_website",
+		"special_enterprise", "special_foundation", "special_gov_unit", "special_hongkong_company", "special_law_office", "special_social_organ", "special_trade_union"}
+
+	for _, v := range CollArr {
+		switch v {
+		case "company_base":
+			fmt.Println("company_base")
+		case "company_employee":
+			fmt.Println("company_employee")
+		case "special_enterprise", "special_foundation", "special_gov_unit", "special_hongkong_company", "special_law_office", "special_social_organ", "special_trade_union":
+			fmt.Println("special")
+		}
+	}
+}

updateBidding/go.mod

@@ -4,7 +4,7 @@ go 1.18
 
 require (
 	github.com/wcc4869/common_utils v0.0.0-20230418011649-f77e1011c822
-	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230712115659-b418d6181de3
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16
 )
 
 require (

updateBidding/go.sum

@@ -670,6 +670,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230712115659-b418d6181de3 h1:kgtSaRR/hRunxM6Kxi66REk7f2PqN1u56j/V+8FfPW8=
 jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230712115659-b418d6181de3/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16 h1:K6hILkGcqNRmTO1nsrpZMqG2iupJETmnmZmKt77SsNY=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

updateBidding/main.go

@@ -1,20 +1,25 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"github.com/wcc4869/common_utils/log"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo/options"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
-	"sync"
 	"time"
 )
 
 var (
 	Mgo      *mongodb.MongodbSim
+	MgoT     *mongodb.MongodbSim //测试环境链接
+	MgoR     *mongodb.MongodbSim
 	saveSize = 50
 	Es       *elastic.Elastic
 	EsNew    *elastic.Elastic
+	EsT      *elastic.Elastic
 
 	// 更新mongo
 	updatePool = make(chan []map[string]interface{}, 5000)
@@ -27,16 +32,48 @@ var (
 
 func main() {
 	//mongodb
-	Mgo = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.189.140:27080",
-		//MongodbAddr: "127.0.0.1:27083",
-		DbName:   "qfw",
-		Size:     10,
-		UserName: "SJZY_RWbid_ES",
-		Password: "SJZY@B4i4D5e6S",
-		//Direct:      true,
+	//Mgo = &mongodb.MongodbSim{
+	//	//MongodbAddr: "172.17.189.140:27080",
+	//	MongodbAddr: "127.0.0.1:27083",
+	//	DbName:      "qfw",
+	//	Size:        10,
+	//	UserName:    "SJZY_RWbid_ES",
+	//	Password:    "SJZY@B4i4D5e6S",
+	//	Direct:      true,
+	//}
+	//Mgo.InitPool()
+
+	//85
+	MgoR = &mongodb.MongodbSim{
+		//MongodbAddr: "127.0.0.1:27080",
+		MongodbAddr: "172.17.4.85:27080",
+		DbName:      "qfw",
+		Size:        10,
+		//Direct: true,
 	}
-	Mgo.InitPool()
+	MgoR.InitPool()
+
+	////测试环境MongoDB
+	//MgoT = &mongodb.MongodbSim{
+	//	//MongodbAddr: "172.17.189.140:27080",
+	//	MongodbAddr: "192.168.3.206:27002",
+	//	DbName:      "qfw_data",
+	//	Size:        10,
+	//	UserName:    "root",
+	//	Password:    "root",
+	//	//Direct:      true,
+	//}
+	//MgoT.InitPool()
+
+	// 测试环境es
+	//Es = &elastic.Elastic{
+	//	S_esurl: "http://192.168.3.149:9201",
+	//	//S_esurl:  "http://172.17.4.184:19805",
+	//	I_size:   5,
+	//	Username: "",
+	//	Password: "",
+	//}
+	//Es.InitElasticSize()
 
 	//es
 	Es = &elastic.Elastic{
@@ -48,21 +85,21 @@ func main() {
 	}
 	Es.InitElasticSize()
 
-	//es 新集群
-	EsNew = &elastic.Elastic{
-		//S_esurl: "http://127.0.0.1:19905",
-		S_esurl:  "http://172.17.4.184:19905",
-		I_size:   5,
-		Username: "jybid",
-		Password: "Top2023_JEB01i@31",
-	}
-	EsNew.InitElasticSize()
+	// es 新集群
+	//EsNew = &elastic.Elastic{
+	//	S_esurl: "http://127.0.0.1:19905",
+	//	//S_esurl:  "http://172.17.4.184:19905",
+	//	I_size:   5,
+	//	Username: "jybid",
+	//	Password: "Top2023_JEB01i@31",
+	//}
+	//EsNew.InitElasticSize()
 
-	go updateMethod()   //更新mongodb
+	//go updateMethod()   //更新mongodb
 	go updateEsMethod() //更新es
 
 	taskRun()
-
+	//dealDataTest()// 测试环境数据处理
 	fmt.Println(111)
 	c := make(chan bool, 1)
 	<-c
@@ -71,100 +108,411 @@ func main() {
 // taskRun 更新es 省市区三个字段
 func taskRun() {
 	defer util.Catch()
-	sess := Mgo.GetMgoConn()
-	defer Mgo.DestoryMongoConn(sess)
+	sess := MgoR.GetMgoConn()
+	defer MgoR.DestoryMongoConn(sess)
 
-	pool := make(chan bool, 10) //处理协程
-	wg := &sync.WaitGroup{}
+	//pool := make(chan bool, 2) //处理协程
+	//wg := &sync.WaitGroup{}
 
 	//查询条件
 	q := map[string]interface{}{
-		//"_id": map[string]interface{}{
-		//	"$gt":  mongodb.StringTOBsonId("652423800000000000000000"),
-		//	"$lte": mongodb.StringTOBsonId("6543c7800000000000000000"),
-		//},
-		"comeintime": map[string]interface{}{
-			"$gt": 1669824000,
-			//"$lte": 1669864950,
-			"$lte": 1702265941,
+		"_id": map[string]interface{}{
+			//"$gt":  mongodb.StringTOBsonId("5a862f0640d2d9bbe88e3cea"),
+			//"$lte": mongodb.StringTOBsonId("65d73ba366cf0db42aca6e2f"),
+
+			//"$gte": mongodb.StringTOBsonId("65d73ba366cf0db42aca6e2f"),
+			"$lte": mongodb.StringTOBsonId("624949c64f7bde5444ed7c6c"),
 		},
-		"site": "国家能源e购",
+		//"comeintime": map[string]interface{}{
+		//	"$gt": 1669824000,
+		//	//"$lte": 1669864950,
+		//	"$lte": 1702265941,
+		//},
+		//"site": "国家能源e购",
 	}
-	selected := map[string]interface{}{"contenthtml": 0, "detail": 0}
-	it := sess.DB("qfw").C("bidding").Find(&q).Select(&selected).Iter()
 
-	fmt.Println("开始")
+	//selected := map[string]interface{}{"contenthtml": 0, "detail": 0}
+	it := sess.DB("qfw").C("projectset_20230904").Find(q).Select(nil).Iter()
+
+	fmt.Println("taskRun 开始")
 	count := 0
-	realNum := 0
+	//realNum := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
 		if count%10000 == 0 {
 			log.Info("current", log.Int("count", count), log.Any("_id", tmp["_id"]))
 		}
 
-		if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
-			tmp = make(map[string]interface{})
+		esUpdate := make(map[string]interface{})
+		if subtitle_projectname, ok := tmp["subtitle_projectname"]; ok && subtitle_projectname != nil {
+			esUpdate["subtitle_projectname"] = subtitle_projectname
+		}
+
+		//if len(esUpdate) > 0 {
+		//	id := mongodb.BsonIdToSId(tmp["_id"])
+		//	err := Es.UpdateDocument("projectset", id, esUpdate)
+		//	if err != nil {
+		//		if strings.Contains(err.Error(), "Document not updated:") {
+		//			continue
+		//		} else {
+		//			fmt.Println(err, mongodb.BsonIdToSId(tmp["_id"]))
+		//		}
+		//	}
+		//}
+
+		if len(esUpdate) > 0 {
+			// 更新es
+			updateEsPool <- []map[string]interface{}{
+				{"_id": mongodb.BsonIdToSId(tmp["_id"])},
+				esUpdate,
+			}
+		}
+
+		//if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
+		//	tmp = make(map[string]interface{})
+		//	continue
+		//}
+		//// 针对存量数据，重复数据不进索引
+		//if util.IntAll(tmp["extracttype"]) == -1 {
+		//	continue
+		//}
+		//
+		////针对产权数据，暂时不入es 索引库
+		//if util.IntAll(tmp["infoformat"]) == 3 {
+		//	continue
+		//}
+		//只有 紧急直接零星采购公告 栏目的数据，需要改成 结果-成交
+		//channel := util.ObjToString(tmp["channel"])
+		//if channel != "紧急直接零星采购公告" {
+		//	continue
+		//}
+
+		//realNum++
+		//pool <- true
+		//wg.Add(1)
+		//go func(tmp map[string]interface{}) {
+		//	defer func() {
+		//		<-pool
+		//		wg.Done()
+		//	}()
+
+		////1.更新MongoDB
+		//update := map[string]interface{}{
+		//	"tag_set":            tmp["tag_set"],
+		//	"tag_topinformation": tmp["tag_topinformation"],
+		//	"tag_subinformation": tmp["tag_subinformation"],
+		//}
+		//if len(update) > 0 {
+		//	//更新MongoDB
+		//	updatePool <- []map[string]interface{}{
+		//		{"_id": tmp["_id"]},
+		//		{"$set": update},
+		//	}
+		//}
+
+		//2.es 更新字段
+		//esUpdate := make(map[string]interface{})
+		//if autoid, ok := tmp["autoid"]; ok && autoid != nil {
+		//	esUpdate["autoid"] = autoid
+		//}
+		//
+		//if len(esUpdate) > 0 {
+		//	err := Es.UpdateDocument("bidding", mongodb.BsonIdToSId(tmp["_id"]), esUpdate)
+		//	if err != nil {
+		//		if strings.Contains(err.Error(), "Document not updated:") {
+		//			return
+		//		} else {
+		//			fmt.Println(err, mongodb.BsonIdToSId(tmp["_id"]))
+		//		}
+		//	}
+		//}
+
+		//if err != nil {
+		//	fmt.Println(err, mongodb.BsonIdToSId(tmp["_id"]))
+		//}
+		//if tag_set, ok := tmp["tag_set"]; ok && tag_set != nil {
+		//	esUpdate["tag_set"] = tag_set
+		//}
+		//
+		//if tag_topinformation, ok := tmp["tag_topinformation"]; ok && tag_topinformation != nil {
+		//	esUpdate["tag_topinformation"] = tag_topinformation
+		//}
+		//
+		//if tag_subinformation, ok := tmp["tag_subinformation"]; ok && tag_subinformation != nil {
+		//	esUpdate["tag_subinformation"] = tag_subinformation
+		//}
+
+		//if len(esUpdate) > 0 {
+		//	// 更新es
+		//	updateEsPool <- []map[string]interface{}{
+		//		{"_id": mongodb.BsonIdToSId(tmp["_id"])},
+		//		esUpdate,
+		//	}
+		//}
+
+		//}(tmp)
+		//tmp = make(map[string]interface{})
+	}
+	//wg.Wait()
+
+	log.Info("Run Over...Count:", log.Int("count", count))
+
+}
+
+// dealData 正式环境，同步合同期限
+func dealData() {
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	//where := map[string]interface{}{
+	//	"_id": mongodb.StringTOBsonId("65c5a36a66cf0db42ab9c1ef"),
+	//}
+	selected := map[string]interface{}{"signaturedate": 1, "contractperiod": 1, "expiredate": 1}
+	it := sess.DB("qfw").C("zktest_quanliang_0210_fbs").Find(nil).Select(&selected).Iter()
+	count := 0
+	realNum := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			log.Info("current", log.Int("count", count), log.Any("_id", tmp["_id"]))
+		}
+
+		idStr := mongodb.BsonIdToSId(tmp["_id"])
+		update := make(map[string]interface{})
+		if tmp["signaturedate"] != nil {
+			update["signaturedate"] = tmp["signaturedate"]
+		}
+		if tmp["contractperiod"] != nil {
+			update["contractperiod"] = tmp["contractperiod"]
+		}
+		if tmp["expiredate"] != nil {
+			update["expiredate"] = tmp["expiredate"]
+		}
+
+		if len(update) == 0 {
 			continue
 		}
+
+		//bidding 表
+		if idStr > "5a862e7040d2d9bbe88e3b1f" {
+			bidding, _ := Mgo.FindById("bidding", idStr, map[string]interface{}{"extracttype": 1})
+			data := *bidding
+			Mgo.UpdateById("bidding", idStr, map[string]interface{}{"$set": update})
+
+			// 针对存量数据，重复数据不进索引
+			if util.IntAll(data["extracttype"]) == -1 {
+				tmp = make(map[string]interface{})
+				continue
+			}
+
+		} else {
+			//bidding_back
+			bidding, _ := Mgo.FindById("bidding_back", idStr, map[string]interface{}{"extracttype": 1})
+			data := *bidding
+			Mgo.UpdateById("bidding_back", idStr, map[string]interface{}{"$set": update})
+			// 针对存量数据，重复数据不进索引
+			if util.IntAll(data["extracttype"]) == -1 {
+				tmp = make(map[string]interface{})
+				continue
+			}
+		}
+
+		realNum++
+
+		//2.es 更新字段
+		esUpdate := update
+		esUpdate["id"] = idStr
+		if len(esUpdate) > 0 {
+			// 更新es
+			updateEsPool <- []map[string]interface{}{
+				{"_id": mongodb.BsonIdToSId(tmp["_id"])},
+				esUpdate,
+			}
+		}
+
+		tmp = make(map[string]interface{})
+	}
+
+	log.Info("Run Over...Count:", log.Int("count", count), log.Int("realNum", realNum))
+
+}
+
+// dealResult 查询抽取表，更新合同周期字段;是dealData的后面遗漏数据
+func dealResult() {
+	defer util.Catch()
+	sess := MgoR.GetMgoConn()
+	defer MgoR.DestoryMongoConn(sess)
+
+	where := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": mongodb.StringTOBsonId("5a4909cf40d2d9bbe8ab329c"),
+			"$lte": mongodb.StringTOBsonId("5a4ad94d40d2d9bbe8ae0183"),
+		},
+		"subtype": "合同",
+	}
+	selected := map[string]interface{}{"signaturedate": 1, "contractperiod": 1, "expiredate": 1}
+	it := sess.DB("qfw").C("result_20220219").Find(where).Select(&selected).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%1000 == 0 {
+			log.Info("current", log.Int("count", count), log.Any("_id", tmp["_id"]))
+		}
+
+		idStr := mongodb.BsonIdToSId(tmp["_id"])
+		update := make(map[string]interface{})
+		if tmp["signaturedate"] != nil {
+			update["signaturedate"] = tmp["signaturedate"]
+		}
+		if tmp["contractperiod"] != nil {
+			update["contractperiod"] = tmp["contractperiod"]
+		}
+		if tmp["expiredate"] != nil {
+			update["expiredate"] = tmp["expiredate"]
+		}
+
+		if len(update) == 0 {
+			continue
+		}
+		bidding, _ := Mgo.FindById("bidding", idStr, map[string]interface{}{"extracttype": 1})
+		data := *bidding
+		Mgo.UpdateById("bidding", idStr, map[string]interface{}{"$set": update})
+
 		// 针对存量数据，重复数据不进索引
-		if util.IntAll(tmp["extracttype"]) == -1 {
+		if util.IntAll(data["extracttype"]) == -1 {
+			tmp = make(map[string]interface{})
 			continue
 		}
 
-		//针对产权数据，暂时不入es 索引库
-		if util.IntAll(tmp["infoformat"]) == 3 {
+		//2.es 更新字段
+		esUpdate := update
+		esUpdate["id"] = idStr
+		if len(esUpdate) > 0 {
+			// 更新es
+			updateEsPool <- []map[string]interface{}{
+				{"_id": mongodb.BsonIdToSId(tmp["_id"])},
+				esUpdate,
+			}
+		}
+
+		tmp = make(map[string]interface{})
+	}
+
+	log.Info("Run Over...Count:", log.Int("count", count))
+}
+
+// dealDataTest 处理测试环境数据
+func dealDataTest() {
+	defer util.Catch()
+	sess := MgoT.GetMgoConn()
+	defer MgoT.DestoryMongoConn(sess)
+
+	where := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": mongodb.StringTOBsonId("635051528aea8786d196e24a"),
+			"$lte": mongodb.StringTOBsonId("639356ca8aea8786d1995c4b"),
+		},
+	}
+
+	//where := map[string]interface{}{
+	//	"_id": mongodb.StringTOBsonId("639356ca8aea8786d1995c4b"),
+	//}
+
+	ctx := context.Background()
+	coll := sess.M.C.Database("qfw_data").Collection("bidding")
+	find := options.Find().SetBatchSize(1000).SetSort(bson.D{bson.E{"_id", -1}}).SetProjection(bson.M{"_id": 1, "title": 1, "subtype": 1})
+	cur, err := coll.Find(ctx, where, find)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	///////
+	selected := map[string]interface{}{"signaturedate": 1, "contractperiod": 1, "expiredate": 1}
+	//it := sess.DB("qfw_data").C("bidding").Find(where).Select(nil).Iter()
+	count := 0
+	realNum := 0
+	for tmp := make(map[string]interface{}); cur.Next(ctx); count++ {
+		if cur != nil {
+			cur.Decode(&tmp)
+		}
+		if count%1000 == 0 {
+			log.Info("current", log.Int("count", count), log.Any("_id", tmp["_id"]))
+		}
+		idStr := mongodb.BsonIdToSId(tmp["_id"])
+
+		data, _ := Mgo.FindById("zktest_quanliang_0210_fbs", idStr, selected)
+
+		if len(*data) == 0 {
 			continue
 		}
-		//只有 紧急直接零星采购公告 栏目的数据，需要改成 结果-成交
-		channel := util.ObjToString(tmp["channel"])
-		if channel != "紧急直接零星采购公告" {
+		update := make(map[string]interface{})
+		if (*data)["signaturedate"] != nil {
+			update["signaturedate"] = (*data)["signaturedate"]
+		}
+		if (*data)["contractperiod"] != nil {
+			update["contractperiod"] = (*data)["contractperiod"]
+		}
+		if (*data)["expiredate"] != nil {
+			update["expiredate"] = (*data)["expiredate"]
+		}
+
+		if len(update) == 0 {
 			continue
 		}
 
+		fmt.Println(idStr)
+		MgoT.UpdateById("bidding", idStr, map[string]interface{}{"$set": update})
+		//bidding 表
+		//if idStr > "5a862e7040d2d9bbe88e3b1f" {
+		//	bidding, _ := Mgo.FindById("bidding", idStr, map[string]interface{}{"extracttype": 1})
+		//	data := *bidding
+		//	Mgo.UpdateById("bidding", idStr, map[string]interface{}{"$set": update})
+		//
+		//	// 针对存量数据，重复数据不进索引
+		//	if util.IntAll(data["extracttype"]) == -1 {
+		//		tmp = make(map[string]interface{})
+		//		continue
+		//	}
+		//
+		//} else {
+		//	//bidding_back
+		//	bidding, _ := Mgo.FindById("bidding_back", idStr, map[string]interface{}{"extracttype": 1})
+		//	data := *bidding
+		//	Mgo.UpdateById("bidding_back", idStr, map[string]interface{}{"$set": update})
+		//	// 针对存量数据，重复数据不进索引
+		//	if util.IntAll(data["extracttype"]) == -1 {
+		//		tmp = make(map[string]interface{})
+		//		continue
+		//	}
+		//}
+
 		realNum++
-		fmt.Println(mongodb.BsonIdToSId(tmp["_id"]))
-		pool <- true
-		wg.Add(1)
-		go func(tmp map[string]interface{}) {
-			defer func() {
-				<-pool
-				wg.Done()
-			}()
-
-			//1.更新MongoDB
-			update := map[string]interface{}{
-				"toptype": "结果",
-				"subtype": "成交",
-			}
-			if len(update) > 0 {
-				//更新MongoDB
-				updatePool <- []map[string]interface{}{
-					{"_id": tmp["_id"]},
-					{"$set": update},
-				}
-			}
 
-			//2.es 更新字段
-			esUpdate := map[string]interface{}{
-				"toptype": "结果",
-				"subtype": "成交",
-				"id":      mongodb.BsonIdToSId(tmp["_id"]),
-			}
-			if len(esUpdate) > 0 {
-				// 更新es
-				updateEsPool <- []map[string]interface{}{
-					{"_id": mongodb.BsonIdToSId(tmp["_id"])},
-					esUpdate,
-				}
+		//2.es 更新字段
+		esUpdate := update
+		esUpdate["id"] = idStr
+		if len(esUpdate) > 0 {
+			fmt.Println(idStr)
+			// 更新es
+			updateEsPool <- []map[string]interface{}{
+				{"_id": mongodb.BsonIdToSId(tmp["_id"])},
+				esUpdate,
 			}
+		}
+
+		//err := Es.UpdateDocument("bidding", idStr, update)
+		//if err != nil {
+		//	log.Error("es update", err)
+		//}
+		//
+		//err = EsNew.UpdateDocument("bidding", idStr, update)
+		//if err != nil {
+		//	log.Error("esNew update", err)
+		//}
 
-		}(tmp)
 		tmp = make(map[string]interface{})
 	}
-	wg.Wait()
 
-	log.Info("Run Over...Count1:", log.Int("count", count), log.Int("realNum", realNum))
+	log.Info("Run Over...Count:", log.Int("count", count), log.Int("realNum", realNum))
 
-	fmt.Println("结束")
 }
 
 // updateMethod 更新MongoDB
@@ -218,8 +566,8 @@ func updateEsMethod() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk("bidding", arru...)
-					EsNew.UpdateBulk("bidding", arru...)
+					Es.UpdateBulk("projectset", arru...)
+					//EsNew.UpdateBulk("bidding", arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0
@@ -231,8 +579,8 @@ func updateEsMethod() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk("bidding", arru...)
-					EsNew.UpdateBulk("bidding", arru...)
+					Es.UpdateBulk("projectset", arru...)
+					//EsNew.UpdateBulk("bidding", arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0

xlsx/exportBidding.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"fmt"
+	"github.com/xuri/excelize/v2"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// exportBidding 导出bidding 数据 excel
+func exportBidding() {
+	//Mgo := &mongodb.MongodbSim{
+	//	MongodbAddr: "192.168.3.206:27002",
+	//	DbName:      "qfw_data",
+	//	Size:        10,
+	//	UserName:    "root",
+	//	Password:    "root",
+	//	Direct: true,
+	//}
+	//Mgo.InitPool()
+
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	//start, _ := time.Parse("2006-01-02 15:04:05", "2023-09-06 21:00:00")
+	//end, _ := time.Parse("2006-01-02 15:04:05", "2023-09-07 09:00:00")
+	//////
+	q := map[string]interface{}{
+		"publish": map[string]interface{}{
+			"$gte": 1701360000,
+			"$lte": 1706716800,
+		},
+		//"modifyinfo.toptype": map[string]interface{}{
+		//	"$exists": 1,
+		//},
+		"toptype": "拟建",
+		//"title": map[string]interface{}{
+		//	"$regex": "充电",
+		//},
+	}
+
+	query := sess.DB("qfw").C("bidding").Find(q).Select(nil).Iter()
+
+	count := 0
+	file := time.Now().Format("20060102") + "导出数据.xlsx"
+	currentPwd, _ := os.Getwd()
+	exportFile := fmt.Sprintf("%s/%s", currentPwd, file)
+	xlsx := excelize.NewFile(excelize.Options{ShortDatePattern: "yyyy/m/dd"})
+	styleOne, _ := xlsx.NewStyle(
+		&excelize.Style{
+			Alignment: &excelize.Alignment{
+				Horizontal: "left",
+				Vertical:   "left",
+			},
+		},
+	)
+	line := 0
+	sheet := "Sheet1"
+	//xlsx.NewSheet(sheet)
+	//xlsx.DeleteSheet("Sheet1")
+	//_ = xlsx.SetColWidth(sheet, "A", "A", 20)
+	//_ = xlsx.SetColWidth(sheet, "B", "B", 25)
+	//_ = xlsx.SetColWidth(sheet, "C", "C", 30)
+	//_ = xlsx.SetColWidth(sheet, "D", "D", 35)
+	//_ = xlsx.SetColWidth(sheet, "E", "E", 45)
+	//_ = xlsx.SetColWidth(sheet, "F", "F", 20)
+
+	subtitles := []string{"title", "projectname", "href", "jyhref", "toptype", "subtype"}
+	//subtitles := []string{"bidding_id", "title", "detail", "href", "jyhref", "toptype", "new_toptype", "new_subtype"}
+	line++
+	//设置第一行title
+	_ = xlsx.SetSheetRow(sheet, fmt.Sprintf("%s%d", "A", line), &subtitles)
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current --- ", count)
+		}
+		line++
+
+		title := util.ObjToString(tmp["title"])
+		if !strings.Contains(title, "充电") {
+			continue
+		}
+		id := mongodb.BsonIdToSId(tmp["_id"])
+		val := []interface{}{}
+		for _, v := range subtitles {
+			if v == "jyhref" {
+				val = append(val, GetJyURLByID(id))
+			} else if v == "publishtime" {
+				timeObj := time.Unix(util.Int64All(tmp[v]), 0)
+				formattedTime := timeObj.Format("2006-01-02 15:04:05")
+				val = append(val, formattedTime)
+			} else {
+				val = append(val, tmp[v])
+			}
+
+		}
+		err := xlsx.SetSheetRow(sheet, fmt.Sprintf("%s%d", "A", line), &val)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		_ = xlsx.SetCellStyle(sheet, fmt.Sprintf("%s%d", "A", line), "BA"+strconv.Itoa(line), styleOne)
+
+		tmp = make(map[string]interface{})
+
+	}
+	xlsx.Path = exportFile
+	xlsx.Save()
+
+	log.Println("dealTmp over ", count)
+
+}
+
+// GetJyURLByID 获取剑鱼地址
+func GetJyURLByID(id string) string {
+	var Url = "https://www.jianyu360.com/article/content/%s.html"
+	url := fmt.Sprintf(Url, util.CommonEncodeArticle("content", id))
+
+	return url
+}

xlsx/go.mod

@@ -4,18 +4,35 @@ go 1.20
 
 require (
 	github.com/tealeg/xlsx v1.0.5
+	github.com/xuri/excelize/v2 v2.8.0
 	go.mongodb.org/mongo-driver v1.13.1
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16
 )
 
 require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/richardlehane/mscfb v1.0.4 // indirect
+	github.com/richardlehane/msoleps v1.0.3 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca // indirect
+	github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/crypto v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gorm.io/driver/mysql v1.5.4 // indirect
+	gorm.io/gorm v1.25.7 // indirect
 )

xlsx/go.sum

@@ -1,9 +1,63 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -11,53 +65,189 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
+github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
+github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM=
+github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE=
 github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca h1:uvPMDVyP7PXMMioYdyPH+0O+Ta/UO1WFfNYMO3Wz0eg=
+github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
+github.com/xuri/excelize/v2 v2.8.0 h1:Vd4Qy809fupgp1v7X+nCS/MioeQmYVVzi495UCTqB7U=
+github.com/xuri/excelize/v2 v2.8.0/go.mod h1:6iA2edBTKxKbZAa7X5bDhcCg51xdOn1Ar5sfoXRGrQg=
+github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a h1:Mw2VNrNNNjDtw68VsEj2+st+oCSn4Uz7vZw6TbhcV1o=
+github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
 go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo=
+golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.5.4 h1:igQmHfKcbaTVyAIHNhhB888vvxh8EdQ2uSUT0LPcBso=
+gorm.io/driver/mysql v1.5.4/go.mod h1:9rYxJph/u9SWkWc9yY4XJ1F/+xO0S/ChOmbk3+Z5Tvs=
+gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16 h1:K6hILkGcqNRmTO1nsrpZMqG2iupJETmnmZmKt77SsNY=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

xlsx/highBidding.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"fmt"
+	"github.com/xuri/excelize/v2"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+)
+
+func HighBidding() {
+	f, err := excelize.OpenFile("./售后组测试企业_20240201_1683.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("详细数据")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw_high",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+
+	count := 0
+	for i := 1; i < len(rows); i++ {
+		s := rows[i][25]
+		if len(s) == 0 {
+			continue
+		}
+		se := util.SimpleEncrypt{Key: "topJYBX2019"}
+		id := se.DecodeString(s)
+
+		data, _ := Mgo.FindById("ibidding", id, nil)
+		if len(*data) > 0 {
+			count++
+		}
+		//fmt.Println(id)
+	}
+
+	log.Print(count)
+}

xlsx/main.go

@@ -4,12 +4,18 @@ import (
 	"context"
 	"fmt"
 	"github.com/tealeg/xlsx"
+	"github.com/xuri/excelize/v2"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
 	"net/url"
 	"os"
+	"sort"
 	"strings"
 )
 
@@ -20,6 +26,17 @@ type WinnerData struct {
 }
 
 func main() {
+	//导出数据到Excel
+	//exportBidding()
+
+	//fmt.Println(11111)
+	//HighBidding()
+	test()
+	log.Println("over")
+}
+
+// exportWinner 导出
+func exportWinner() {
 	username := "SJZY_RWbid_ES"
 	password := "SJZY@B4i4D5e6S"
 	//addr := "172.17.189.140:27080"
@@ -52,7 +69,7 @@ func main() {
 
 	log.Println(1111)
 	// 选择数据库和集合
-	collection := client.Database("qfw").Collection("wcc_bank_winner_statistic")
+	collection := client.Database("qfw").Collection("wcc_20240103")
 	// 查询数据并排序
 	// 设置查询条件
 	filter := bson.D{
@@ -61,9 +78,14 @@ func main() {
 	}
 	// 设置投影，排除 contenthtml 字段
 	projection := bson.D{
-		{"credit_no", 1}, // 0表示不返回该字段
-		{"winner", 1},    // 0表示不返回该字段
-		{"num", 1},       // 0表示不返回该字段
+		{"title", 1},      // 0表示不返回该字段
+		{"detail", 1},     // 0表示不返回该字段
+		{"href", 1},       // 0表示不返回该字段
+		{"bidding_id", 1}, // 0表示不返回该字段
+		{"subtype", 1},    // 0表示不返回该字段
+		{"jyhref", 1},     // 0表示不返回该字段
+		{"data_type", 1},  // 0表示不返回该字段
+		{"toptype", 1},    // 0表示不返回该字段
 	}
 	//findOptions := options.Find().SetProjection(projection)
 	findOptions := options.Find().SetSort(map[string]int{"num": -1}).SetLimit(100000).SetProjection(projection)
@@ -116,3 +138,182 @@ func main() {
 
 	fmt.Println("数据已成功导出到 exported_data.xlsx")
 }
+
+// exportPhone 根据企业名单，导出联系人电话
+func exportPhone() {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "mixdata",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+
+	//
+
+	username := "datascbi"
+	password := "Da#Bi20221111SC"
+	host := "127.0.0.1:4001"
+	//host := "172.17.4.242:4000"
+	database := "global_common_data"
+	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", username, password, host, database)
+	// 连接到数据库
+	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
+	if err != nil {
+		log.Println("Failed to connect to database:", err)
+		return
+	}
+
+	f, err := excelize.OpenFile("./物业公司清单.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("Sheet1")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	type BaseInfo struct {
+		Name   string `json:"name"`
+		NameId string `json:"name_id"`
+	}
+	type Contact struct {
+		ContactName string `json:"contact_name"`
+		ContactTel  string `json:"contact_tel"`
+	}
+	for i := 1; i < len(rows); i++ {
+		name := rows[i][1]
+		stds, _ := Mgo.FindOne("qyxy_std", map[string]interface{}{"company_name": name})
+		var reportsMap = make([]string, 0)
+		contactsMap := make([]string, 0)
+		if reports, ok := (*stds)["annual_reports"]; ok {
+			if rs, ok := reports.([]interface{}); ok {
+				for _, v := range rs {
+					if da, ok := v.(map[string]interface{}); ok {
+						if util.ObjToString(da["operator_name"]) != "" && util.ObjToString(da["company_phone"]) != "" {
+							tm := util.ObjToString(da["operator_name"]) + "_" + util.ObjToString(da["company_phone"])
+							if !IsInStringArray(tm, reportsMap) {
+								reportsMap = append(reportsMap, tm)
+							}
+						} else if util.ObjToString(da["company_phone"]) != "" {
+							if !IsInStringArray(util.ObjToString(da["company_phone"]), reportsMap) {
+								reportsMap = append(reportsMap, util.ObjToString(da["company_phone"]))
+							}
+						}
+					}
+
+				}
+			}
+		}
+		if len(reportsMap) > 0 {
+			res := strings.Join(reportsMap, ",")
+			f.SetCellValue("Sheet1", fmt.Sprintf("E%v", i+1), res)
+		}
+
+		baseinfo := BaseInfo{}
+		db.Table("dws_f_ent_baseinfo").Select("name", "name_id").Where("name = ? ", name).Scan(&baseinfo)
+		if baseinfo.NameId != "" {
+			contacts := []Contact{}
+			db.Table("dws_f_ent_contact").Select("contact_name", "contact_tel").Where("name_id = ? ", baseinfo.NameId).Scan(&contacts)
+			if len(contacts) > 0 {
+				for _, v := range contacts {
+					if strings.Contains(v.ContactTel, ">") {
+						continue
+					}
+					if v.ContactName != "" && v.ContactTel != "" {
+						s := v.ContactName + "_" + v.ContactTel
+						if !IsInStringArray(s, contactsMap) {
+							contactsMap = append(contactsMap, s)
+						}
+					} else {
+						if !IsInStringArray(v.ContactTel, contactsMap) {
+							contactsMap = append(contactsMap, v.ContactTel)
+						}
+					}
+				}
+			}
+		}
+		if len(contactsMap) > 0 {
+			res := strings.Join(contactsMap, ",")
+			f.SetCellValue("Sheet1", fmt.Sprintf("F%v", i+1), res)
+		}
+	}
+
+	f.Save()
+}
+
+func test() {
+	//Mgo := &mongodb.MongodbSim{
+	//	//MongodbAddr: "172.17.189.140:27080",
+	//	MongodbAddr: "192.168.3.166:27082",
+	//	Size:        10,
+	//	DbName:      "majiajia",
+	//	//UserName:    "SJZY_RWbid_ES",
+	//	//Password:    "SJZY@B4i4D5e6S",
+	//	//Direct:      true,
+	//}
+	//Mgo.InitPool()
+	MgoP := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27080",
+		Size:        10,
+		DbName:      "qfw",
+		Direct:      true,
+		//UserName:    "SJZY_RWbid_ES",
+		//Password:    "SJZY@B4i4D5e6S",
+
+	}
+	MgoP.InitPool()
+
+	f, err := excelize.OpenFile("./河南物业.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("Sheet1")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for i := 1; i < len(rows); i++ {
+		id := rows[i][0]
+		rs, _ := MgoP.FindById("projectset_20230904", id, nil)
+		if rs == nil {
+			continue
+		}
+		if phone, ok := (*rs)["buyer"]; ok {
+			if util.ObjToString(phone) != "" {
+				f.SetCellValue("Sheet1", fmt.Sprintf("K%v", i+1), phone)
+			}
+		}
+	}
+	f.Save()
+
+}
+
+// IsInStringArray 判断数组中是否存在字符串
+func IsInStringArray(str string, arr []string) bool {
+	// 先对字符串数组进行排序
+	sort.Strings(arr)
+	// 使用二分查找算法查找字符串
+	pos := sort.SearchStrings(arr, str)
+	// 如果找到了则返回 true，否则返回 false
+	return pos < len(arr) && arr[pos] == str
+}

xlsx/project.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"fmt"
+	"github.com/xuri/excelize/v2"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"regexp"
+)
+
+// getProject 根据表格biddingID获取项目ID
+func getProject() {
+	MgoP := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27080",
+		Size:        10,
+		DbName:      "qfw",
+		Direct:      true,
+		//UserName:    "SJZY_RWbid_ES",
+		//Password:    "SJZY@B4i4D5e6S",
+
+	}
+	MgoP.InitPool()
+
+	//Mgo := &mongodb.MongodbSim{
+	//	//MongodbAddr: "172.17.189.140:27080",
+	//	MongodbAddr: "127.0.0.1:27083",
+	//	Size:        10,
+	//	DbName:      "qfw",
+	//	UserName:    "SJZY_RWbid_ES",
+	//	Password:    "SJZY@B4i4D5e6S",
+	//	Direct:      true,
+	//}
+	//Mgo.InitPool()
+
+	f, err := excelize.OpenFile("./bidding2.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("Sheet1")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for i := 1; i < len(rows); i++ {
+		biddingID := rows[i][0]
+		where := map[string]interface{}{
+			"ids": biddingID,
+		}
+		p, _ := MgoP.FindOne("projectset_20230904", where)
+		projectId := mongodb.BsonIdToSId((*p)["_id"])
+		f.SetCellValue("Sheet1", fmt.Sprintf("B%v", i+1), projectId)
+
+		//bidding, _ := Mgo.FindById("bidding", biddingID, nil)
+		//href := (*bidding)["href"]
+		//f.SetCellValue("Sheet1", fmt.Sprintf("E%v", i+1), href)
+		//f.SetCellValue("Sheet1", fmt.Sprintf("F%v", i+1), GetJyURLByID(biddingID))
+
+	}
+
+	f.Save()
+
+	log.Println("over")
+}
+
+// dealProject
+func dealProject() {
+	MgoP := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27080",
+		Size:        10,
+		DbName:      "qfw",
+		Direct:      true,
+		//UserName:    "SJZY_RWbid_ES",
+		//Password:    "SJZY@B4i4D5e6S",
+
+	}
+	MgoP.InitPool()
+	f, err := excelize.OpenFile("./bidding2.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("Sheet2")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// 编译正则表达式
+	re := regexp.MustCompile(`包\d+`)
+
+	for i := 1; i < len(rows); i++ {
+		pid := rows[i][0]
+		p, _ := MgoP.FindById("projectset_20230904", pid, nil)
+		f.SetCellValue("Sheet2", fmt.Sprintf("B%v", i+1), (*p)["projectname"])
+		f.SetCellValue("Sheet2", fmt.Sprintf("C%v", i+1), (*p)["multipackage"])
+		f.SetCellValue("Sheet2", fmt.Sprintf("D%v", i+1), (*p)["sourceinfourl"])
+		if util.IntAll((*p)["multipackage"]) == 0 {
+			if list, ok := (*p)["list"].([]interface{}); ok {
+				for _, v := range list {
+					if da, ok := v.(map[string]interface{}); ok {
+						title := da["title"]
+						// 使用正则表达式进行匹配
+						matches := re.FindAllString(util.ObjToString(title), -1)
+						if len(matches) > 0 {
+							for _, match := range matches {
+								fmt.Println(match[0]) // 输出匹配到的关键词
+								f.SetCellValue("Sheet2", fmt.Sprintf("F%v", i+1), match[0])
+								f.SetCellValue("Sheet2", fmt.Sprintf("E%v", i+1), title)
+								break
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	f.Save()
+	log.Println("over")
+
+}

xlsx/xlsx_test.go

@@ -0,0 +1,244 @@
+package main
+
+import (
+	"fmt"
+	"github.com/xuri/excelize/v2"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"testing"
+)
+
+func TestA(T *testing.T) {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27017",
+		Size:        10,
+		DbName:      "wcc",
+		//UserName:    "SJZY_RWbid_ES",
+		//Password:    "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+
+	f, err := excelize.OpenFile("./拼数据.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("Sheet1")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	cells, _ := f.GetCols("Sheet2")
+
+	for i := 1; i < len(rows); i++ {
+		log.Print(rows[i][3])
+		if rows[i][2] == "省级" {
+			for k, v := range cells[0] {
+				if k == 0 || v == "" {
+					continue
+				}
+
+				insert := make(map[string]interface{})
+				insert["top"] = rows[i][0]
+				insert["area"] = rows[i][3]
+				insert["level"] = rows[i][2]
+				insert["sub"] = v
+				insert["name"] = rows[i][3] + v
+				Mgo.InsertOrUpdate("wcc", "wcc_0313", insert)
+			}
+		}
+
+		if rows[i][2] == "市级" {
+			for k, v := range cells[1] {
+				if k == 0 || v == "" {
+					continue
+				}
+				insert := make(map[string]interface{})
+				insert["top"] = rows[i][0]
+				insert["area"] = rows[i][3]
+				insert["city"] = rows[i][4]
+				insert["level"] = rows[i][2]
+				insert["sub"] = v
+				insert["name"] = rows[i][4] + v
+				Mgo.InsertOrUpdate("wcc", "wcc_0313", insert)
+			}
+		}
+
+		if rows[i][2] == "县级" {
+			for k, v := range cells[2] {
+				if k == 0 || v == "" {
+					continue
+				}
+				insert := make(map[string]interface{})
+				insert["top"] = rows[i][0]
+				insert["area"] = rows[i][3]
+				insert["city"] = rows[i][4]
+				insert["district"] = rows[i][5]
+				insert["level"] = rows[i][2]
+				insert["sub"] = v
+				insert["name"] = rows[i][4] + rows[i][5] + v
+				Mgo.InsertOrUpdate("wcc", "wcc_0313", insert)
+			}
+		}
+	}
+
+	fmt.Println("over")
+}
+
+func TestB(T *testing.T) {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+
+	f, err := excelize.OpenFile("./fanglei.11111.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("Sheet1")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	for i := 1; i < len(rows); i++ {
+		id := rows[i][0]
+		re, _ := Mgo.FindById("bidding", id, nil)
+		Mgo.SaveByOriID("wcc_0327_bidding", re)
+	}
+
+	log.Println("over")
+
+}
+
+func TestAAA(t *testing.T) {
+	//87 竞品
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27081",
+		Size:        10,
+		DbName:      "py_theme",
+		UserName:    "",
+		Password:    "",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	f, err := excelize.OpenFile("./专项债券.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		f.Save()
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("项目列表")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	keys := []string{"projectName", "totalInvestment", "regionName", "cityName", "countyName", "capital", "applyDebt", "portfolioFinancing", "specialDebtAsCapital",
+		"expectedReturn", "projectCost", "projectTypeName3", "projectSubject", "startDate", "endDate", "operationStartDate",
+		"operationEndDate", "sourceIncome", "constructionContent", "remarks", "portfolioFinancingSource", "costIncomePercent", "coverageMultiple", "implementingAgency",
+		"accountingFirm", "lawFirm", "createTime", "updateTime"}
+	titles := []string{"项目名称", "总投资（万元）", "省份", "地市", "区县", "不含专项债资本金（万元）", "申请专项债总额（万元）", "其他债务融资（万元）", "专项债作资本金（万元）", "预期收入（万元）", "成本",
+		"项目领域", "项目业主", "建设期开始", "建设期结束", "运营期开始", "运营期结束", "收入来源", "建设内容", "特殊情况备注", "其他债务融资来源", "成本/收入（%）", "覆盖倍数", "主管部门", "会计所", "律所", "入库时间", "更新时间"}
+
+	keys2 := []string{"bondName", "bondShortName", "bondNo", "regionName", "bondType1Name", "bondType2Name", "projectType1Name", "totalAmount", "issueDate", "issuePlaceName", "issueTerm", "issueInterestRate", "issuePhase", "payInterestMethodName",
+		"newBondAmount", "counterBondAmount", "refinancingBondAmount", "redemptionMethod", "valueDate", "expiryDate", "payInterestDate", "latelyPayInterestDate", "lastPayInterest", "cumulativePayInterest"}
+
+	titles2 := []string{"项目名称", "债券名称", "债券简称", "债券编码", "省份", "债券性质", "债券类型", "官方项目类型", "发行金额（万元）", "发行日期", "发行场所", "发行期限(年)", "发行利率(%)", "发行期数", "付息方式", "新增债券（亿元）", "置换债券（亿元）", "再融资债券（亿元）", "赎回方式",
+		"起息日", "到息日", "付息日", "最近付息日", "上期已付息（亿元）", "累计付息（亿元）"}
+	line := 0
+	line++
+	f.NewSheet("项目详情")
+	f.NewSheet("债券详情")
+	bondLine := 0
+	//设置第一行title
+	_ = f.SetSheetRow("项目详情", fmt.Sprintf("%s%d", "A", line), &titles)
+	_ = f.SetSheetRow("债券详情", fmt.Sprintf("%s%d", "A", line), &titles2)
+	//
+	for i := 1; i < len(rows); i++ {
+		projectName := rows[i][0]
+		//projectName := "新建南通至宁波高速铁路（海盐段）"
+		log.Println(projectName)
+		line++
+		where := map[string]interface{}{
+			"project.jcxx.projectName": projectName,
+		}
+		detail, _ := Mgo.FindOne("specialbond_detail", where)
+		if len(*detail) == 0 {
+			continue
+		}
+		insert := make([]interface{}, 0)
+		if project, ok := (*detail)["project"].(map[string]interface{}); ok {
+			if jcxx, ok := project["jcxx"].(map[string]interface{}); ok {
+				for _, v := range keys {
+					insert = append(insert, jcxx[v])
+				}
+			}
+		}
+
+		err := f.SetSheetRow("项目详情", fmt.Sprintf("%s%d", "A", line), &insert)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		// 债券基本信息
+		if bonds, ok := (*detail)["bond"].([]interface{}); ok {
+			for _, bond := range bonds {
+				if bondData, ok := bond.(map[string]interface{}); ok {
+					if jcxx, ok := bondData["jbxx"].(map[string]interface{}); ok {
+						bondInsert := make([]interface{}, 0)
+						bondInsert = append(bondInsert, projectName)
+						for _, v := range keys2 {
+							bondInsert = append(bondInsert, jcxx[v])
+						}
+						bondLine++
+						f.SetSheetRow("债券详情", fmt.Sprintf("%s%d", "A", line), &bondInsert)
+					}
+				}
+			}
+		}
+	}
+
+	//query := sess.DB("py_theme").C("specialbond_detail").Find(nil).Select(nil).Iter()
+	//count := 0
+	//for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+	//	if count%10000 == 0 {
+	//		log.Println("current:", count)
+	//	}
+	//
+	//	id := mongodb.BsonIdToSId(tmp["_id"])
+	//	log.Println(id)
+	//}
+
+}

yuqing/config.go

@@ -22,6 +22,8 @@ type EnvConf struct {
 	Files    []string
 	Start    int
 	End      int
+	Shour    int
+	Ehour    int
 	Coll1    string
 	Coll2    string
 	Collb    string

yuqing/config.toml

@@ -43,6 +43,8 @@
 
     start = -1   ## -1表示导出昨天数据
     end = 0
+    shour = 12   ## 开始时间小时
+    ehour = 12  ## 结束时间小时
     coll1 = "CorporatePublicOpinion" ##
     coll2 = "news_detail" ## 百度的数据
     collb = "bidding_yq"

yuqing/main.go

@@ -184,8 +184,8 @@ func exportFiles() {
 
 	fmt.Println("开始数据导出")
 	now := time.Now()
-	targetTime := time.Date(now.Year(), now.Month(), now.Day()+GF.Env.Start, 12, 0, 0, 0, now.Location())
-	todayTime := time.Date(now.Year(), now.Month(), now.Day()+GF.Env.End, 12, 0, 0, 0, now.Location())
+	targetTime := time.Date(now.Year(), now.Month(), now.Day()+GF.Env.Start, GF.Env.Shour, 0, 0, 0, now.Location())
+	todayTime := time.Date(now.Year(), now.Month(), now.Day()+GF.Env.End, GF.Env.Ehour, 0, 0, 0, now.Location())
 	q := map[string]interface{}{
 		"comeintime": map[string]interface{}{
 			"$gt":  targetTime.Unix(),