update

project_chuan/go.mod

@@ -5,6 +5,7 @@ go 1.23.7
 require (
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/spf13/viper v1.20.1
+	github.com/xuri/excelize/v2 v2.9.1
 	go.uber.org/zap v1.22.0
 	gorm.io/driver/mysql v1.5.7
 	gorm.io/gorm v1.26.1
@@ -27,24 +28,29 @@ require (
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/richardlehane/mscfb v1.0.4 // indirect
+	github.com/richardlehane/msoleps v1.0.4 // indirect
 	github.com/sagikazarmark/locafero v0.7.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.12.0 // indirect
 	github.com/spf13/cast v1.7.1 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tiendc/go-deepcopy v1.6.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/xuri/efp v0.0.1 // indirect
+	github.com/xuri/nfp v0.0.1 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	go.mongodb.org/mongo-driver v1.10.1 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.32.0 // indirect
-	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/sys v0.29.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/crypto v0.38.0 // indirect
+	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/sync v0.14.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

project_chuan/go.sum

@@ -94,6 +94,11 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
+github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
+github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00=
+github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
@@ -121,12 +126,20 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo=
+github.com/tiendc/go-deepcopy v1.6.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
 github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=
+github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
+github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw=
+github.com/xuri/excelize/v2 v2.9.1/go.mod h1:x7L6pKz2dvo9ejrRuD8Lnl98z4JLt0TGAwjhW+EiP8s=
+github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q=
+github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
@@ -149,9 +162,11 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
-golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
+golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
+golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -168,15 +183,15 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
+golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
+golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -187,16 +202,16 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
-golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
+golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

project_chuan/init.go

@@ -15,6 +15,7 @@ import (
 
 var (
 	MgoB            *mongodb.MongodbSim //
+	MgoQY           *mongodb.MongodbSim //
 	MgoP            *mongodb.MongodbSim //
 	GF              GlobalConf
 	JianyuSubjectDB *gorm.DB
@@ -64,6 +65,8 @@ func InitMgo() {
 	//	Direct:      GF.Mongo181.Direct,
 	//}
 	//Mgo181.InitPool()
+
+	//85 项目查询
 	MgoP = &mongodb.MongodbSim{
 		MongodbAddr: GF.MongoP.Host,
 		Size:        10,
@@ -75,6 +78,7 @@ func InitMgo() {
 
 	MgoP.InitPool()
 
+	//bidding 查询
 	MgoB = &mongodb.MongodbSim{
 		MongodbAddr: GF.MongoB.Host,
 		Size:        10,
@@ -85,6 +89,18 @@ func InitMgo() {
 	}
 
 	MgoB.InitPool()
+
+	//qyxy_std 查询
+	MgoQY = &mongodb.MongodbSim{
+		MongodbAddr: GF.MongoB.Host,
+		Size:        10,
+		DbName:      "mixdata",
+		UserName:    GF.MongoB.Username,
+		Password:    GF.MongoB.Password,
+		Direct:      GF.MongoB.Direct,
+	}
+
+	MgoQY.InitPool()
 }
 
 func InitLog() {
@@ -106,6 +122,7 @@ func InitLog() {
 	log.Info("InitLog", zap.Any("duration", time.Since(now).Seconds()))
 }
 
+// InitMysql InitMysql
 func InitMysql() {
 	now := time.Now()
 	username := GF.Mysql.Username

project_chuan/main.go

@@ -6,6 +6,8 @@ import (
 )
 
 func main() {
+
+	//dealXlsxData()
 	dealProposed22Concurrent()
 	//dealProposed()
 	log.Info("main", zap.String("aaaa", "数据处理完毕"))

project_chuan/project.go

@@ -9,6 +9,7 @@ import (
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"sort"
 	"strings"
 	"sync"
 )
@@ -31,6 +32,7 @@ func dealProposed22Concurrent() {
 
 	coll := sess.DB("qfw").C("projectset_proposed")
 	query := map[string]interface{}{
+		//"area": "甘肃",
 		"firsttime": map[string]interface{}{
 			"$gte": 1735660800,
 		},
@@ -38,7 +40,7 @@ func dealProposed22Concurrent() {
 	iter := coll.Find(query).Select(nil).Iter()
 
 	// 3. 并发控制
-	const maxWorkers = 20
+	const maxWorkers = 1
 	taskCh := make(chan map[string]interface{}, 2000)
 	var wg sync.WaitGroup
 
@@ -61,6 +63,11 @@ func dealProposed22Concurrent() {
 		if count%1000 == 0 {
 			log.Info("dealProposed", zap.Int("current", count), zap.Any("projectname", doc["projectname"]))
 		}
+
+		if util.ObjToString(doc["area"]) != "甘肃" {
+			continue
+		}
+
 		taskCh <- cloneMap(doc) // 防止 map 重用
 	}
 	close(taskCh)
@@ -77,47 +84,117 @@ func processOneProposed(tmp map[string]interface{}, client *elastic.Client) {
 	proposedID := mongodb.BsonIdToSId(tmp["_id"])
 	projectName := util.ObjToString(tmp["projectname"])
 	buyer := util.ObjToString(tmp["owner"])
+	proposed_number := util.ObjToString(tmp["proposed_number"])
 
-	results, err := searchES22(client, projectName, buyer, 75, 10)
+	log.Info("processOneProposed", zap.String("开始查询es", projectName))
+	results, err := searchES23(client, projectName, buyer, 18, 50)
 	if err != nil {
 		log.Warn("searchES22 error", zap.Error(err))
 		return
 	}
-
+	log.Info("processOneProposed", zap.String("结束查询es", projectName))
 	projectIds := []string{}
 	biddingIds := []string{}
 	biddings := []map[string]interface{}{}
+	//bidding_id-> bidding map 数据源
+	biddingIdMap := make(map[string]map[string]interface{}, 0)
+	//project_id -> project map
+	projectIdMap := make(map[string]map[string]interface{}, 0)
+	project_bidding_ids := make(map[string][]string, 0) //存储project_id =>[bidding_id]
 
 	for _, re := range results {
 		biddingID := util.ObjToString(re["id"])
 		biddingIds = append(biddingIds, biddingID)
-		biddings = append(biddings, map[string]interface{}{
+		da := map[string]interface{}{
 			"id":          re["id"],
 			"title":       re["title"],
 			"projectname": re["projectname"],
 			"score":       re["score"],
 			"toptype":     re["toptype"],
 			"subtype":     re["subtype"],
-		})
+			"buyer":       re["buyer"],
+			"budget":      re["budget"],
+			"buyerperson": re["buyerperson"],
+			"buyertel":    re["buyertel"],
+			"s_winner":    re["s_winner"],
+			"bidamount":   re["bidamount"],
+			"winnertel":   re["winnertel"],
+			"agency":      re["agency"],
+			"publishtime": re["publishtime"],
+		}
+		biddings = append(biddings, da)
+		biddingIdMap[biddingID] = da
 	}
 
 	for _, bid := range biddingIds {
 		where2 := map[string]interface{}{"ids": bid}
 		projectset, _ := MgoP.FindOne("projectset_20230904", where2)
 		if projectset != nil && len(*projectset) > 0 {
-			projectIds = append(projectIds, mongodb.BsonIdToSId((*projectset)["_id"]))
+			pid := mongodb.BsonIdToSId((*projectset)["_id"])
+			projectIds = append(projectIds, pid)
+			p_bidding_ids := project_bidding_ids[pid]
+			p_bidding_ids = append(p_bidding_ids, bid)
+			projectIdMap[pid] = *projectset
 		}
 	}
 
 	insert := map[string]interface{}{
-		"proposed_id":  proposedID,
-		"bidding_ids":  removeDuplicates(biddingIds),
-		"project_ids":  removeDuplicates(projectIds),
-		"biddings":     biddings,
-		"project_name": tmp["projectname"],
+		"proposed_id":     proposedID,
+		"stype":           1, //代表从拟建数据-> 匹配在建数据
+		"proposed_number": proposed_number,
+		"buyer":           buyer,
+		"project_name":    tmp["projectname"],
+		"area":            tmp["area"],
+		"city":            tmp["city"],
+		"district":        tmp["district"],
+		//"bidding_ids":     removeDuplicates(biddingIds),
+		//"project_ids":     removeDuplicates(projectIds),
+		//"biddings":        biddings,
+	}
+
+	if buyer != "" {
+		where11 := map[string]interface{}{
+			"company_name": buyer,
+		}
+		std, _ := MgoQY.FindOne("qyxy_std", where11)
+		insert["credit_no"] = (*std)["credit_no"]
+	}
+
+	projects := make([]map[string]interface{}, 0)
+	if len(project_bidding_ids) > 0 {
+		for pid, bidding_ids := range project_bidding_ids {
+			p_project := projectIdMap[pid]
+			p_bs := make([]map[string]interface{}, 0)
+			for _, vv := range bidding_ids {
+				bi := biddingIdMap[vv]
+				p_bs = append(p_bs, bi)
+			}
+
+			project := map[string]interface{}{
+				"project_id":  pid,
+				"projectname": p_project["projectname"],
+				"bidamount":   p_project["bidamount"],
+				"area":        p_project["area"],
+				"city":        p_project["city"],
+				"bidstatus":   p_project["bidstatus"],
+				"buyer":       p_project["buyer"],
+				"firsttime":   p_project["firsttime"],
+				"biddings":    p_bs,
+			}
+
+			projects = append(projects, project)
+		}
+	}
+
+	if len(projects) > 0 {
+		insert["projects"] = projects
+	}
+
+	if len(biddings) > 0 {
+		insert["biddings"] = biddings
 	}
 
-	MgoP.InsertOrUpdate("qfw", "wcc_dealProposed22", insert)
+	MgoP.InsertOrUpdate("qfw", "wcc_dealProposed22_0523", insert)
 }
 
 func cloneMap(src map[string]interface{}) map[string]interface{} {
@@ -288,6 +365,176 @@ func removeDuplicates(arr []string) []string {
 	return result
 }
 
+func searchES23(client *elastic.Client, projectName, buyer2 string, scoreThreshold float64, maxResults int) ([]map[string]interface{}, error) {
+	fieldsToTry := []string{"projectname.pname", "title", "detail"}
+
+	filtersToTry := [][]elastic.Query{
+		{elastic.NewTermsQuery("subtype", "中标", "成交", "合同", "单一")},
+		{elastic.NewTermsQuery("toptype", "招标", "预告", "采购意向")},
+		{elastic.NewTermsQuery("toptype", "拟建")},
+	}
+
+	var allResults []*elastic.SearchHit
+	seenIDs := make(map[string]bool)
+
+	for _, field := range fieldsToTry {
+		for _, filter := range filtersToTry {
+			// 构建查询：使用 MultiMatchQuery + phrase
+			query := elastic.NewBoolQuery().
+				Must(elastic.NewMultiMatchQuery(projectName, field).Type("phrase")).
+				Filter(filter...)
+
+			// 执行查询
+			searchResult, err := client.Search().
+				Index("bidding").
+				Query(query).
+				Size(70).
+				Do(context.Background())
+			if err != nil {
+				return nil, err
+			}
+
+			// 去重处理
+			for _, hit := range searchResult.Hits.Hits {
+				if !seenIDs[hit.Id] {
+					seenIDs[hit.Id] = true
+					allResults = append(allResults, hit)
+				}
+			}
+			if len(allResults) >= maxResults {
+				break
+			}
+		}
+		if len(allResults) >= maxResults {
+			break
+		}
+	}
+
+	//
+	//fieldsToTry := []string{"projectname.pname", "title", "detail"}
+	//filtersToTry := [][]elastic.Query{
+	//	{elastic.NewTermsQuery("subtype", "中标", "成交", "合同", "单一")},
+	//	{elastic.NewTermsQuery("toptype", "招标", "预告", "采购意向")},
+	//	{elastic.NewTermsQuery("toptype", "拟建")},
+	//}
+	//
+	//var allResults []*elastic.SearchHit
+	//seenIDs := make(map[string]bool)
+	//
+	//for _, field := range fieldsToTry {
+	//	for _, filter := range filtersToTry {
+	//		// 构建查询
+	//		query := elastic.NewBoolQuery().
+	//			Must(elastic.NewMatchQuery(field, projectName)).
+	//			Filter(filter...)
+	//
+	//		// 执行查询
+	//		searchResult, err := client.Search().
+	//			Index("bidding").
+	//			Query(query).
+	//			Size(70). // 多取一些，后面做筛选和去重
+	//			Do(context.Background())
+	//		if err != nil {
+	//			return nil, err
+	//		}
+	//
+	//		for _, hit := range searchResult.Hits.Hits {
+	//			if !seenIDs[hit.Id] {
+	//				allResults = append(allResults, hit)
+	//				seenIDs[hit.Id] = true
+	//			}
+	//		}
+	//
+	//		if len(allResults) >= maxResults {
+	//			break
+	//		}
+	//	}
+	//	if len(allResults) >= maxResults {
+	//		break
+	//	}
+	//}
+
+	var results []map[string]interface{}
+	seenProjectNames := make(map[string]bool)
+	seenProjectCodes := make(map[string]bool)
+	bidamountMap := make(map[float64]bool)
+
+	for _, hit := range allResults {
+		var doc map[string]interface{}
+		if err := json.Unmarshal(hit.Source, &doc); err != nil {
+			log.Info("解析文档失败", zap.Error(err))
+			continue
+		}
+
+		projectNameValue := util.ObjToString(doc["projectname"])
+		if projectNameValue == "" {
+			continue
+		}
+
+		projectCode := util.ObjToString(doc["projectcode"])
+		if seenProjectCodes[projectCode] {
+			continue
+		}
+		seenProjectCodes[projectCode] = true
+
+		bidamount := util.Float64All(doc["bidamount"])
+		if bidamountMap[bidamount] {
+			continue
+		}
+		bidamountMap[bidamount] = true
+
+		// 相似度筛选
+		score := *hit.Score
+		doc["score"] = score //相似度
+		if score < scoreThreshold {
+			continue
+		}
+
+		//id := util.ObjToString(doc["id"])
+		//doc["jyhref"] = GetJyURLByID(id)
+
+		//if site := util.ObjToString(doc["site"]); site == "中华人民共和国自然资源部" {
+		//	doc["title"] = "土地出让" + "-" + util.ObjToString(doc["title"])
+		//}
+
+		// enrich: total_investment
+		//if bidData, _ := MgoB.FindById("bidding", id, nil); bidData != nil {
+		//	if util.Float64All((*bidData)["total_investment"]) > 0 {
+		//		doc["total_investment"] = (*bidData)["total_investment"]
+		//	}
+		//}
+
+		doc["score"] = score
+		detail := util.ObjToString(doc["detail"])
+
+		// 字段中必须包含 projectName
+		if buyer2 != "" {
+			if !strings.Contains(detail, projectName) && !strings.Contains(detail, buyer2) {
+				continue
+			}
+		}
+
+		if seenProjectNames[projectNameValue] {
+			continue
+		}
+		seenProjectNames[projectNameValue] = true
+
+		results = append(results, doc)
+
+		if len(results) >= maxResults {
+			break
+		}
+	}
+	// 排序：按 score 降序
+	sort.Slice(results, func(i, j int) bool {
+		si := util.Float64All(results[i]["score"])
+		sj := util.Float64All(results[j]["score"])
+		return si > sj
+	})
+
+	return results, nil
+}
+
 func searchES22(client *elastic.Client, projectName, buyer2 string, scoreThreshold float64, maxResults int) ([]map[string]interface{}, error) {
 	fieldsToTry := []string{"projectname.pname", "title", "detail"}
 	filtersToTry := [][]elastic.Query{
@@ -564,3 +811,25 @@ func GetJyURLByID(id string) string {
 
 	return url
 }
+
+// GetIdByURL 解密url，获取bidding ID
+func GetIdByURL(url string) string {
+	if strings.Contains(url, "work-bench") {
+		return ""
+	}
+	if strings.Contains(url, "/article/content") {
+		urls := strings.Split(url, "content/")
+		res := strings.Split(urls[1], ".html")
+		ids := util.CommonDecodeArticle("content", res[0])
+		return ids[0]
+	}
+	if strings.HasSuffix(url, "appid") {
+		urls := strings.Split(url, "entservice/")
+		res := strings.Split(urls[1], ".html")
+		se := util.SimpleEncrypt{Key: "entservice"}
+		id := se.DecodeString(res[0])
+		return id
+	}
+
+	return ""
+}

project_chuan/xlsx.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"fmt"
+	"github.com/xuri/excelize/v2"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+)
+
+func dealXlsxData() {
+	//url := "http://172.17.4.184:19908"
+	////url := "http://127.0.0.1:19908"
+	//username := "jybid"
+	//password := "Top2023_JEB01i@31"
+	////index := "bidding" //索引名称
+	//// 创建 Elasticsearch 客户端
+	//client, err := elastic.NewClient(
+	//	elastic.SetURL(url),
+	//	elastic.SetBasicAuth(username, password),
+	//	elastic.SetSniff(false),
+	//)
+	//if err != nil {
+	//	log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	//}
+
+	// 2. 读取 Excel
+	f, err := excelize.OpenFile("投资入库项目_202504.xlsx")
+	if err != nil {
+		log.Fatal("❌ 无法打开 Excel 文件:", err)
+	}
+	defer func() {
+		f.Save()
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	// 3. 读取 B 列（项目名称）
+	rows, err := f.GetRows("Sheet1") // 替换为正确的 Sheet 名称
+	if err != nil {
+		log.Fatal("❌ 无法读取 Excel:", err)
+	}
+
+	log.Println("开始处理数据--------")
+	for i := 1; i < len(rows); i++ {
+		p1_project_code := rows[i][0]
+		p1_project_name := rows[i][1]
+		p1_project_owner := rows[i][2]
+		p1_project_owner_code := rows[i][3]
+		p1_project_owner_area := rows[i][4]
+		p1_project_total := rows[i][5]
+		//
+		log.Println("项目名称--------", p1_project_name)
+		insert := map[string]interface{}{
+			"p1_project_code":       p1_project_code,
+			"p1_project_name":       p1_project_name,
+			"p1_project_owner":      p1_project_owner,
+			"p1_project_owner_code": p1_project_owner_code,
+			"p1_project_owner_area": p1_project_owner_area,
+			"p1_project_total":      p1_project_total,
+		}
+		start := 8
+		row := rows[i]
+		p1_biddings := make([]map[string]interface{}, 0)
+		p1_bidding_ids := make([]string, 0)
+		for i := start; i+3 < len(row); i += 4 {
+			amount := row[i]
+			title := row[i+1]
+			href := row[i+2]
+			score := row[i+3]
+			bid := GetIdByURL(href)
+			p1_bidding_ids = append(p1_bidding_ids, bid)
+			bidding, _ := MgoB.FindById("bidding", bid, nil)
+			data := map[string]interface{}{
+				"bidding_id": bid,
+				"href":       href,
+				"title":      title,
+				"socre":      score,
+				"amount":     amount,
+				"toptype":    (*bidding)["toptype"],
+				"subtype":    (*bidding)["subtype"],
+			}
+			//log.Println("aa", insert, amount, title, href, score)
+			p1_biddings = append(p1_biddings, data)
+		}
+
+		insert["biddings"] = p1_biddings
+		p1_bidding_ids = removeDuplicates(p1_bidding_ids)
+		p1_projects := make([]map[string]interface{}, 0)
+		for _, vv := range p1_bidding_ids {
+			where2 := map[string]interface{}{
+				"ids": vv,
+			}
+			projectset, _ := MgoP.FindOne("projectset_20230904", where2)
+			if projectset != nil && len(*projectset) > 0 {
+				data := map[string]interface{}{
+					"project_id":  mongodb.BsonIdToSId((*projectset)["_id"]),
+					"projectname": util.ObjToString((*projectset)["projectname"]),
+					"bidamount":   (*projectset)["bidamount"],
+					"sortprice":   (*projectset)["sortprice"],
+					"area":        (*projectset)["area"],
+					"city":        (*projectset)["city"],
+					"district":    (*projectset)["district"],
+				}
+				p1_projects = append(p1_projects, data)
+			}
+		}
+		uniqueProjects := make([]map[string]interface{}, 0)
+		seen := make(map[string]bool)
+
+		for _, project := range p1_projects {
+			pid := project["project_id"].(string)
+			if !seen[pid] {
+				seen[pid] = true
+				uniqueProjects = append(uniqueProjects, project)
+			}
+		}
+
+		if len(uniqueProjects) > 0 {
+			insert["projects"] = uniqueProjects
+		}
+
+		MgoP.InsertOrUpdate("qfw", "wcc_dealXlsxData", insert)
+	}
+
+	log.Println("数据处理完毕--------")
+}
+
+// exportTestData 导出拓扑统计局 样例数据
+func exportTestData() {
+	sess := MgoP.GetMgoConn()
+	defer MgoP.DestoryMongoConn(sess)
+
+	queryMgo := sess.DB("qfw").C("wcc_dealXlsxData").Find(nil).Select(nil).Iter()
+	count := 0
+
+	total_datas := make([][]interface{}, 0)
+	for tmp := make(map[string]interface{}); queryMgo.Next(tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current", count)
+		}
+		//
+		projectname := util.ObjToString(tmp["p1_project_name"])
+		where := map[string]interface{}{
+			"projectname": projectname,
+		}
+		pro_project, _ := MgoP.FindOne("projectset_proposed", where)
+		ppid := ""
+		if pro_project != nil {
+			ppid = mongodb.BsonIdToSId((*pro_project)["_id"])
+		}
+		//1.拟建原始数据
+		v1 := []interface{}{
+			ppid,
+			tmp["p1_project_code"],
+			tmp["p1_project_name"],
+			tmp["p1_project_name"],
+			tmp["p1_project_owner"],
+			tmp["p1_project_owner_code"],
+		}
+
+		//2.在建项目数据
+		if projects, ok := tmp["projects"].([]interface{}); ok {
+			log.Println(projects)
+		}
+
+		total_datas = append(total_datas, v1)
+	}
+
+}