feat:初始化

config.yaml

@@ -0,0 +1,33 @@
+udpPort: ":1889"
+mongodb:
+  main:
+    mongodbAddr: "172.20.45.128:27080,172.31.31.202:27081"
+    dbName: "qfw"
+    userName: "wcj2025"
+    password: "Bidingjy@0522"
+    size: 8
+    #mongodbAddr: "172.20.45.129:27002"
+    #dbName: "qfw"
+  extract:
+    mongodbAddr: "172.17.4.85:27080"
+    dbName: "qfw"
+    size: 8
+    #mongodbAddr: "172.20.45.129:27002"
+    #dbName: "wcj"
+  site:
+    mongodbAddr: "172.17.4.87:27080"
+    size: 8
+    dbName: "editor"
+scoringPoolSize: 5
+preferPoolSize: 5
+logger:
+  filename: "./logs/preferred.log"
+  maxsize: 1024
+  maxbackups: 5
+  maxage: 5
+  compress: true
+  console: true
+nextNode:
+  stype: "bidding"
+  addr: "127.0.0.1"
+  port: 889

db/db.go

@@ -0,0 +1,27 @@
+package db
+
+import (
+	. "app.yhyue.com/moapp/jybase/mongodb"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gctx"
+	"log"
+)
+
+var (
+	Mgo_Extract *MongodbSim
+	Mgo_Main    *MongodbSim
+	Mgo_Site    *MongodbSim
+)
+
+func init() {
+	ctx := gctx.New()
+	g.Config().MustGet(ctx, "mongodb.main").Struct(&Mgo_Main)
+	Mgo_Main.InitPool()
+	log.Println("初始化mongodb main")
+	g.Config().MustGet(ctx, "mongodb.extract").Struct(&Mgo_Extract)
+	Mgo_Extract.InitPool()
+	log.Println("初始化mongodb extract")
+	g.Config().MustGet(ctx, "mongodb.site").Struct(&Mgo_Site)
+	Mgo_Site.InitPool()
+	log.Println("初始化mongodb site")
+}

entity/entity.go

@@ -0,0 +1,17 @@
+package entity
+
+type Rule struct {
+	Type     string      //规则类型
+	Score    int         //分值
+	Min      int         //最小长度
+	Max      int         //最大程度
+	DataType string      //数据类型
+	Value    interface{} //值
+}
+
+type SiteInfo struct {
+	Site      string
+	Subtype   string
+	Toptype   string
+	TypePlate string
+}

go.mod

@@ -0,0 +1,48 @@
+module dataPrefer
+
+go 1.23.4
+
+require (
+	app.yhyue.com/moapp/jybase v0.0.0-20250509080440-038d69d3ad3b
+	github.com/gogf/gf/v2 v2.9.0
+	golang.org/x/net v0.32.0
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40
+)
+
+require (
+	github.com/BurntSushi/toml v1.4.0 // indirect
+	github.com/clbanning/mxj/v2 v2.7.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/fatih/color v1.18.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/grokify/html-strip-tags-go v0.1.0 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
+	go.mongodb.org/mongo-driver v1.17.3 // indirect
+	go.opentelemetry.io/otel v1.32.0 // indirect
+	go.opentelemetry.io/otel/metric v1.32.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.32.0 // indirect
+	go.opentelemetry.io/otel/trace v1.32.0 // indirect
+	golang.org/x/crypto v0.30.0 // indirect
+	golang.org/x/sync v0.11.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.22.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -0,0 +1,267 @@
+app.yhyue.com/moapp/jybase v0.0.0-20250509080440-038d69d3ad3b h1:bQKiTTrlbDIc+Z2y7ynbZrieaaAUB2CisE/rl42FlPY=
+app.yhyue.com/moapp/jybase v0.0.0-20250509080440-038d69d3ad3b/go.mod h1:OEtMbsn7wY/7MLgV7yDUpVDKExUoj3B8h+4w4ZckJQQ=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME=
+github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/gogf/gf/v2 v2.9.0 h1:semN5Q5qGjDQEv4620VzxcJzJlSD07gmyJ9Sy9zfbHk=
+github.com/gogf/gf/v2 v2.9.0/go.mod h1:sWGQw+pLILtuHmbOxoe0D+0DdaXxbleT57axOLH2vKI=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4=
+github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM=
+github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ=
+go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
+go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
+go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M=
+go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8=
+go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4=
+go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
+go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY=
+golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
+golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40 h1:xTeRmpFgwOdu+NbWg/YntX3MnQpttm7jj33C1+JdBTk=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240202055658-e2ef72e18b40/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

main.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	qu "app.yhyue.com/moapp/jybase/common"
+	"dataPrefer/db"
+	"dataPrefer/service"
+	"encoding/json"
+	"flag"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gctx"
+	"gopkg.in/natefinch/lumberjack.v2"
+	"io"
+	mu "jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
+	"log"
+	"net"
+	"os"
+	"sync"
+	"time"
+)
+
+var udpclient mu.UdpClient
+var lock = &sync.Mutex{}
+
+func main() {
+	model := flag.Int("m", 0, "1：非定时任务")
+	flag.Parse()
+	var logger *lumberjack.Logger
+	ctx := gctx.New()
+	g.Config().MustGet(ctx, "logger").Struct(&logger)
+	writers := []io.Writer{logger}
+	if g.Config().MustGet(ctx, "logger.console").Bool() {
+		writers = append(writers, os.Stdout)
+	}
+	log.SetOutput(io.MultiWriter(writers...))
+	udpPort := g.Config().MustGet(ctx, "udpPort").String()
+	udpclient = mu.UdpClient{Local: udpPort, BufSize: 1024}
+	udpclient.Listen(ProcessUdpMsg)
+	log.Println("Udp服务监听", g.Config().MustGet(ctx, "udpPort").String())
+	service.IncDataById("684a9c215f834436f09c2710", "684a9c215f834436f09c2710")
+	//service.Tj()
+	//service.Hz()
+	*model = 1
+	if *model == 0 {
+		<-chan bool(nil)
+	}
+}
+
+// udp接收
+func ProcessUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA:
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			log.Println("接收到udp消息", string(data))
+			stype := qu.ObjToString(mapInfo["stype"])
+			sid, eid := qu.ObjToString(mapInfo["gtid"]), qu.ObjToString(mapInfo["lteid"])
+			key := sid + "-" + eid + "-" + stype
+			go udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
+			if stype == "bidding" {
+				lock.Lock()
+				service.IncDataById(sid, eid)
+				sendNextNode(nil, sid, eid)
+				for {
+					time.Sleep(10 * time.Second)
+					if db.Mgo_Main.Count("bidding_processing_ids", map[string]interface{}{"lteid": eid, "dataprocess": 9}) > 0 {
+						break
+					}
+					log.Println(string(data), "es索引未生完，等待中。。。")
+				}
+				lock.Unlock()
+			} else {
+				sendNextNode(data, "", "")
+			}
+		}
+	case mu.OP_NOOP: //下个节点回应
+		str := string(data)
+		log.Println("其他节点回应：", str)
+	}
+}
+
+// 下节点发送
+func sendNextNode(by []byte, sid string, eid string) {
+	if by == nil {
+		stype := g.Config().MustGet(gctx.New(), "nextNode.stype").String()
+		key := sid + "-" + eid + "-" + stype
+		by, _ = json.Marshal(map[string]interface{}{
+			"gtid":  sid,
+			"lteid": eid,
+			"stype": stype,
+			"key":   key,
+		})
+	}
+	addr := &net.UDPAddr{
+		IP:   net.ParseIP(g.Config().MustGet(gctx.New(), "nextNode.addr").String()),
+		Port: g.Config().MustGet(gctx.New(), "nextNode.port").Int(),
+	}
+	udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr) //发送下节点
+}

main_test.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	. "app.yhyue.com/moapp/jybase/mongodb"
+	"dataPrefer/db"
+	. "dataPrefer/service"
+	"encoding/json"
+	"github.com/gogf/gf/v2/util/gconv"
+	mu "jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
+	"log"
+	"net"
+	"testing"
+)
+
+func TestSendNextNode(t *testing.T) {
+	uc := mu.UdpClient{Local: ":880", BufSize: 1024}
+	uc.Listen(ProcessUdpMsg)
+	by, _ := json.Marshal(map[string]interface{}{
+		"gtid":  "1",
+		"lteid": "2",
+		"stype": "bidding",
+		"key":   "1-2-bidding",
+	})
+	addr := &net.UDPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 889,
+	}
+	log.Println(uc.WriteUdp(by, mu.OP_TYPE_DATA, addr)) //发送下节点
+}
+
+// 370435 201782
+func Tj() {
+	sess := db.Mgo_Main.GetMgoConn()
+	defer db.Mgo_Main.DestoryMongoConn(sess)
+	it := sess.DB("qfw").C(Bidding_master).Find(nil).Select(map[string]interface{}{
+		"old_id":           1,
+		"id":               1,
+		"site_toptype":     1,
+		"old_site_toptype": 1,
+	}).Iter()
+	oldMap := map[string]int{}
+	ooldMap := map[string]int{}
+	newMap := map[string]int{}
+	nnewMap := map[string]int{}
+	update := 0
+	index := 0
+	for m := make(map[string]interface{}); it.Next(&m); {
+		index++
+		if index%5000 == 0 {
+			log.Println("index", index)
+		}
+		if m["old_site_toptype"] == nil {
+			m["old_site_toptype"] = m["site_toptype"]
+		}
+		if m["old_id"] == nil {
+			m["old_id"] = m["id"]
+		}
+		oldMap[gconv.String(m["old_site_toptype"])] = oldMap[gconv.String(m["old_site_toptype"])] + 1
+		newMap[gconv.String(m["site_toptype"])] = newMap[gconv.String(m["site_toptype"])] + 1
+		if gconv.String(m["id"]) != gconv.String(m["old_id"]) {
+			update++
+			ooldMap[gconv.String(m["old_site_toptype"])] = ooldMap[gconv.String(m["old_site_toptype"])] + 1
+			nnewMap[gconv.String(m["site_toptype"])] = nnewMap[gconv.String(m["site_toptype"])] + 1
+		}
+		m = make(map[string]interface{})
+	}
+	log.Println(oldMap)
+	log.Println(newMap)
+	log.Println(update)
+	log.Println(ooldMap)
+	log.Println(nnewMap)
+	log.Println("over...", index)
+}
+
+func Hz() {
+	sess := db.Mgo_Main.GetMgoConn()
+	defer db.Mgo_Main.DestoryMongoConn(sess)
+	it := sess.DB("qfw").C(Bidding_master).Find(map[string]interface{}{"mark": 1}).Select(map[string]interface{}{"id": 1}).Iter()
+	index := 0
+	for m := make(map[string]interface{}); it.Next(&m); {
+		index++
+		id := m["id"].(string)
+		ids := []interface{}{StringTOBsonId(id)}
+		data, _ := db.Mgo_Main.FindById(Bidding, id, `{"_id":1,"repeat_id":1}`)
+		if repeatId, _ := (*data)["repeat_id"].(string); repeatId != "" && repeatId != id {
+			ids = append(ids, StringTOBsonId(repeatId))
+			datas, _ := db.Mgo_Extract.Find(Result_20220219, map[string]interface{}{
+				"repeat_id": repeatId,
+			}, nil, map[string]interface{}{
+				"_id": 1,
+			}, false, -1, -1)
+			if datas != nil {
+				for _, v := range *datas {
+					ids = append(ids, v["_id"])
+				}
+			}
+			datas, _ = db.Mgo_Extract.Find(Result_20220218, map[string]interface{}{
+				"repeat_id": repeatId,
+			}, nil, map[string]interface{}{
+				"_id": 1,
+			}, false, -1, -1)
+			if datas != nil {
+				for _, v := range *datas {
+					ids = append(ids, v["_id"])
+				}
+			}
+		}
+		mp := map[string]bool{}
+		newIds := []interface{}{}
+		for _, v := range ids {
+			if mp[BsonIdToSId(v)] {
+				continue
+			}
+			mp[BsonIdToSId(v)] = true
+			newIds = append(newIds, v)
+		}
+		newDatas, _ := db.Mgo_Main.Find(Bidding, map[string]interface{}{
+			"_id": map[string]interface{}{"$in": newIds},
+		}, nil, nil, false, -1, -1)
+		for _, v := range *newDatas {
+			v["bestid"] = id
+			if BsonIdToSId(v["_id"]) == id {
+				v["isbest"] = 1
+			} else {
+				v["isbest"] = 0
+			}
+		}
+		db.Mgo_Main.SaveBulk("bidding_master_verify", *newDatas...)
+		m = make(map[string]interface{})
+		log.Println("index...", index)
+	}
+}

rule.yaml

@@ -0,0 +1,119 @@
+结构化字段:
+  预告:
+    - type: exists
+      value:
+        - buyer
+        - budget
+        - projectname
+        - area
+        - city
+        - projectcode
+        - procurementlist
+      score: -10
+  招标:
+    - type: exists
+      value:
+        - buyer
+        - budget
+        - projectname
+        - area
+        - city
+        - projectcode
+      score: -5
+    - type: exists
+      value:
+        - bidopentime
+        - bidendtime
+        - buyertel
+        - agency
+      score: -3
+  结果:
+    - type: exists
+      value:
+        - buyer
+        - projectname
+        - projectcode
+        - winner
+        - bidamount
+        - area
+        - city
+      score: -5
+    - type: exists
+      value:
+        - winnertel
+        - projectperiod
+        - purchasinglist
+      score: -3
+  其他:
+    - type: exists
+      value:
+        - buyer
+        - budget
+        - projectname
+        - projectcode
+        - winner
+        - bidamount
+        - area
+        - city
+      score: -5
+采集字段:
+  title:
+    - type: length
+      min: 0
+      max: 5
+      score: -5
+    - type: length
+      min: 50
+      score: -5
+    - type: regexp
+      notReg: "[\\p{Han}]"
+      score: -5
+  detail:
+    - type: length
+      min: 0
+      max: 50
+      score: -10
+    - type: length
+      min: 50
+      max: 200
+      score: -5
+  isValidFile:
+    - type: equal
+      value: false
+      score: -5
+站点画像:
+  政府采购网站:
+    topType: 政府网站
+    yes:
+      政府采购网: 10
+      公共资源交易中心: 10
+      默认: 5
+    not:
+      score: -3
+  需登录网站:
+    score: -5
+  竞品网站:
+    score: -40
+    list:
+      - 干里马招标网
+      - 中国招标与采购网
+      - 元博网
+  延时采集网站:
+    score: -10
+对比大模型:
+  allExistsAndSame:
+    fields:
+      - area:s_area
+      - city:s_city
+      - buyer:s_buyer
+      - budget:s_budget
+      - s_winner:s_winner
+      - bidamount:s_bidamount
+    score: -5
+  partExistsAndSame:
+    fields:
+      - projectname:s_projectname
+      - projectcode:s_projectcode
+      - winnertel:s_winnertel
+      - agency:s_agency
+    score: -3

service/service.go

@@ -0,0 +1,683 @@
+package service
+
+import (
+	. "app.yhyue.com/moapp/jybase/common"
+	. "app.yhyue.com/moapp/jybase/mongodb"
+	"dataPrefer/db"
+	"dataPrefer/entity"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/os/gcfg"
+	"github.com/gogf/gf/v2/os/gctx"
+	"github.com/gogf/gf/v2/util/gconv"
+	"golang.org/x/net/context"
+	"log"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	Result_20220219 = "result_20220219"
+	Result_20220218 = "result_20220218"
+	Bidding         = "bidding"
+	Bidding_master  = "bidding_master"
+)
+
+var (
+	allSite                  = map[string]*entity.SiteInfo{}
+	loginSite                = map[string]bool{}
+	delaySite                = map[string]bool{}
+	ruleConf                 *gcfg.Config
+	competitorSite           = map[string]bool{}
+	gatherRules, structRules map[string][]*entity.Rule
+	allExistsFields          = map[string]string{}
+	partExistsFields         = map[string]string{}
+	govSiteYes               map[string]*g.Var
+	extractSelectFields      = map[string]interface{}{"_id": 1, "agency": 1, "area": 1, "bidamount": 1, "bidendtime": 1, "bidopentime": 1, "budget": 1, "buyer": 1, "buyertel": 1, "city": 1, "detail": 1, "isValidFile": 1, "procurementlist": 1, "projectcode": 1, "projectname": 1, "projectperiod": 1, "purchasinglist": 1, "title": 1, "winner": 1, "s_winner": 1, "winnertel": 1, "prefer_score": 1, "repeat": 1, "site": 1, "toptype": 1, "repeat_id": 1, "ai_zhipu": 1, "ext_ai_record": 1, "dataging": 1}
+	biddingSelectFields      = map[string]interface{}{"_id": 1, "detail": 1, "procurementlist": 1, "purchasinglist": 1, "isValidFile": 1, "pici": 1}
+)
+
+func reLoadConf() {
+	ctx := gctx.New()
+	ruleConf = g.Config("rule.yaml")
+	for _, v := range ruleConf.MustGet(ctx, "站点画像.竞品网站.list").Strings() {
+		competitorSite[v] = true
+	}
+	if err := ruleConf.MustGet(ctx, "采集字段").Scan(&gatherRules); err != nil {
+		log.Fatalln("采集字段 scan error", err)
+	}
+	if err := ruleConf.MustGet(ctx, "结构化字段").Scan(&structRules); err != nil {
+		log.Fatalln("结构化字段 scan error", err)
+	}
+	for _, v := range ruleConf.MustGet(ctx, "对比大模型.allExistsAndSame.fields").Strings() {
+		vs := strings.Split(v, ":")
+		allExistsFields[vs[0]] = vs[1]
+	}
+	for _, v := range ruleConf.MustGet(ctx, "对比大模型.partExistsAndSame.fields").Strings() {
+		vs := strings.Split(v, ":")
+		partExistsFields[vs[0]] = vs[1]
+	}
+	govSiteYes = ruleConf.MustGet(ctx, "站点画像.政府采购网站.yes").MapStrVar()
+}
+func initSite() {
+	log.Println("开始初始化网站数据。。。")
+	wait := &sync.WaitGroup{}
+	wait.Add(1)
+	go func() {
+		sess := db.Mgo_Site.GetMgoConn()
+		defer func() {
+			db.Mgo_Site.DestoryMongoConn(sess)
+			wait.Done()
+		}()
+		siteIt := sess.DB("editor").C("site").Find(nil).Select(map[string]interface{}{
+			"site":         1,
+			"site_subtype": 1,
+			"site_toptype": 1,
+			"type_plate":   1,
+		}).Iter()
+		for m := make(map[string]interface{}); siteIt.Next(&m); {
+			allSite[gconv.String(m["site"])] = &entity.SiteInfo{
+				Subtype:   gconv.String(m["site_subtype"]),
+				Toptype:   gconv.String(m["site_toptype"]),
+				Site:      gconv.String(m["site"]),
+				TypePlate: gconv.String(m["type_plate"]),
+			}
+			m = make(map[string]interface{})
+		}
+		log.Println("site表数据初始化结束。。。", len(allSite))
+	}()
+	//
+	wait.Add(1)
+	go func() {
+		sess := db.Mgo_Site.GetMgoConn()
+		defer func() {
+			db.Mgo_Site.DestoryMongoConn(sess)
+			wait.Done()
+		}()
+		loginSiteIt := sess.DB("editor").C("site_login").Find(nil).Select(map[string]interface{}{
+			"site": 1,
+		}).Iter()
+		for m := make(map[string]interface{}); loginSiteIt.Next(&m); {
+			loginSite[gconv.String(m["site"])] = true
+			m = make(map[string]interface{})
+		}
+		log.Println("site_login表数据初始化结束。。。", len(loginSite))
+	}()
+	//
+	wait.Add(1)
+	go func() {
+		sess := db.Mgo_Site.GetMgoConn()
+		defer func() {
+			db.Mgo_Site.DestoryMongoConn(sess)
+			wait.Done()
+		}()
+		delaySiteIt := sess.DB("spider").C("spider_compete").Find(map[string]interface{}{"compete": true}).Select(map[string]interface{}{
+			"site": 1,
+		}).Iter()
+		for m := make(map[string]interface{}); delaySiteIt.Next(&m); {
+			delaySite[gconv.String(m["site"])] = true
+			m = make(map[string]interface{})
+		}
+		log.Println("spider_compete表数据初始化结束。。。", len(delaySite))
+	}()
+	wait.Wait()
+}
+func IncDataByTime() {
+	//{comeintime:{$gt:1747886400,$lt:1747888200}}
+	query := map[string]interface{}{
+		"comeintime": map[string]interface{}{
+			"$gte": 1747756800,
+			"$lt":  1747843200,
+		},
+	}
+	//query := map[string]interface{}{
+	//	"$or": []map[string]interface{}{
+	//		map[string]interface{}{
+	//			"_id": StringTOBsonId("682bf3325f834436f023b091"),
+	//		},
+	//		map[string]interface{}{
+	//			"repeat_id": "682bf3325f834436f023b091",
+	//		},
+	//	},
+	//}
+	//query := map[string]interface{}{
+	//	"_id": StringTOBsonId("682cd75d5f834436f02824ee"),
+	//}
+	IncData(query)
+}
+func IncDataById(sid, eid string) {
+	query := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gte": StringTOBsonId(sid),
+			"$lte": StringTOBsonId(eid),
+		},
+	}
+	if sid == eid {
+		query = map[string]interface{}{
+			"_id": StringTOBsonId(sid),
+		}
+	}
+	IncData(query)
+}
+func IncData(query map[string]interface{}) {
+	defer Catch()
+	log.Println("start。。。", query)
+	reLoadConf()
+	initSite()
+	//
+	sess := db.Mgo_Main.GetMgoConn()
+	defer db.Mgo_Main.DestoryMongoConn(sess)
+	ctx := gctx.New()
+	it := sess.DB(db.Mgo_Main.DbName).C(Bidding).Find(query).Select(biddingSelectFields).Sort("_id").Iter()
+	index := 0
+	pool := make(chan bool, g.Config().MustGet(ctx, "scoringPoolSize").Int())
+	wait := &sync.WaitGroup{}
+	for m := make(map[string]interface{}); it.Next(&m); {
+		index++
+		if index%5000 == 0 {
+			log.Println("处理数据", index)
+		}
+		pool <- true
+		wait.Add(1)
+		go func(nm map[string]interface{}) {
+			defer func() {
+				<-pool
+				wait.Done()
+			}()
+			bidding, ok := db.Mgo_Extract.FindOneByField(Result_20220219, map[string]interface{}{"_id": nm["_id"]}, extractSelectFields)
+			if !ok || bidding == nil || len(*bidding) == 0 {
+				bidding, ok = db.Mgo_Extract.FindOneByField(Result_20220218, map[string]interface{}{"_id": nm["_id"]}, extractSelectFields)
+			}
+			_id := BsonIdToSId(nm["_id"])
+			if !ok || bidding == nil || len(*bidding) == 0 {
+				log.Println("抽取表中没有找到数据", _id)
+				return
+			} else if gconv.Int((*bidding)["dataging"]) == 1 { //抽取表extract中dataging=1跳过
+				log.Println("抽取表中dataging是1跳过", _id)
+				return
+			} else if gconv.Int((*bidding)["repeat"]) == 1 && gconv.String((*bidding)["repeat_id"]) == "" {
+				log.Println("repeat=1并且repeat_id不存在，过滤掉", _id)
+				return
+			}
+			for k, _ := range *bidding {
+				if nm[k] != nil {
+					continue
+				}
+				nm[k] = (*bidding)[k]
+			}
+			score := MakeScore(ctx, nm)
+			if score == -1 {
+				return
+			}
+			Preferred(ctx, nm, score)
+		}(m)
+		m = make(map[string]interface{})
+	}
+	wait.Wait()
+	log.Println("over。。。", index)
+}
+
+// 去打分
+func MakeScore(ctx context.Context, nm map[string]interface{}) int {
+	totalScore := Scoring(ctx, nm)
+	log.Println("标讯", BsonIdToSId(nm["_id"]), "分数", totalScore)
+	nm["prefer_score"] = totalScore
+	db.Mgo_Main.UpdateById(Bidding, nm["_id"], map[string]interface{}{
+		"$set": map[string]interface{}{
+			"prefer_score": totalScore,
+		},
+	})
+	return totalScore
+}
+
+// 打分
+func Scoring(ctx context.Context, data map[string]interface{}) int {
+	_id := BsonIdToSId(data["_id"])
+	//
+	var countScore = func(field string, rule *entity.Rule) int {
+		var totalScore int
+		switch rule.Type {
+		case "exists":
+			for _, vv := range gconv.Strings(rule.Value) {
+				if data[vv] != nil {
+					continue
+				}
+				totalScore += rule.Score
+				log.Println(_id, vv, "字段不存在", rule.Score)
+			}
+		case "length":
+			valueLen := len([]rune(gconv.String(data[field])))
+			if rule.Min >= 0 && rule.Max > 0 {
+				if valueLen >= rule.Min && valueLen <= rule.Max {
+					totalScore += rule.Score
+					log.Println(_id, field, "字符串长度大于", rule.Min, "小于", rule.Max, rule.Score)
+				}
+			} else if rule.Min >= 0 {
+				if valueLen > rule.Min {
+					totalScore += rule.Score
+					log.Println(_id, field, "字符串长度大于", rule.Min, rule.Score)
+				}
+			}
+		case "equal":
+			var isDeduct bool
+			switch value := rule.Value.(type) {
+			case int, int32, int64:
+				isDeduct = gconv.Int64(value) == gconv.Int64(data[field])
+			case float32, float64:
+				isDeduct = gconv.Float64(value) == gconv.Float64(data[field])
+			case bool:
+				isDeduct = gconv.Bool(value) == gconv.Bool(data[field])
+			default:
+				isDeduct = rule.Value == data[field]
+			}
+			if isDeduct {
+				totalScore += rule.Score
+				log.Println(_id, field, "值等于", rule.Value, rule.Score)
+			}
+		}
+		return totalScore
+	}
+	totalScore := 100
+	sts := structRules[gconv.String(data["toptype"])]
+	if sts == nil {
+		sts = structRules["其他"]
+	}
+	//结构化字段
+	for _, v := range sts {
+		totalScore += countScore("", v)
+	}
+	//采集字段
+	for k, v := range gatherRules {
+		for _, vv := range v {
+			if thisScore := countScore(k, vv); thisScore != 0 {
+				totalScore += thisScore
+				break
+			}
+		}
+	}
+	site := gconv.String(data["site"])
+	if allSite[site] != nil {
+		score := 0
+		if topType := ruleConf.MustGet(ctx, "站点画像.政府采购网站.topType").String(); allSite[site].Toptype == topType {
+			if govSiteYes[allSite[site].Subtype].IsNil() {
+				score = govSiteYes["默认"].Int()
+				log.Println(_id, "地方政府网站", score)
+			} else {
+				score = govSiteYes[allSite[site].Subtype].Int()
+				log.Println(_id, allSite[site].Subtype, "类网站", score)
+			}
+		} else {
+			score = ruleConf.MustGet(ctx, "站点画像.政府采购网站.not.score").Int()
+			log.Println(_id, "非", topType, score)
+		}
+		totalScore += score
+	}
+	//
+	if loginSite[site] {
+		score := ruleConf.MustGet(ctx, "站点画像.需登录网站.score").Int()
+		totalScore += score
+		log.Println(_id, "需登录网站", score)
+	}
+	//
+	if competitorSite[site] {
+		score := ruleConf.MustGet(ctx, "站点画像.竞品网站.score").Int()
+		totalScore += score
+		log.Println(_id, "竞品网站", score)
+	}
+	//
+	if delaySite[site] {
+		score := ruleConf.MustGet(ctx, "站点画像.延时采集网站.score").Int()
+		totalScore += score
+		log.Println(_id, "延时采集网站", score)
+	}
+	//
+	if data["ai_zhipu"] != nil {
+		aiZhipu, _ := data["ai_zhipu"].(map[string]interface{})
+		compare := map[string]interface{}{}
+		extAiRecord, _ := data["ext_ai_record"].(map[string]interface{})
+		for _, v := range []map[string]string{allExistsFields, partExistsFields} {
+			for kk, _ := range v {
+				if extAiRecord[kk] != nil {
+					compare[kk] = extAiRecord[kk]
+				} else if data[kk] != nil {
+					compare[kk] = data[kk]
+				}
+			}
+		}
+		//全部字段存在，且不一致
+		allFieldExists := 0
+		for k, v := range allExistsFields {
+			if compare[k] == nil || aiZhipu[v] == nil {
+				break
+			}
+			allFieldExists++
+		}
+		if allFieldExists > 0 && allFieldExists == len(allExistsFields) {
+			score := ruleConf.MustGet(ctx, "对比大模型.allExistsAndSame.score").Int()
+			for k, v := range allExistsFields {
+				if gconv.String(compare[k]) == gconv.String(aiZhipu[v]) {
+					continue
+				}
+				totalScore += score
+				log.Println(_id, "对比大模型，字段都存在，", v, "的值不一致", score)
+			}
+		}
+		//部分字段存在，且不一致
+		partFieldExists := false
+		for k, v := range partExistsFields {
+			if compare[k] == nil || aiZhipu[v] == nil {
+				continue
+			}
+			partFieldExists = true
+			break
+		}
+		if partFieldExists {
+			score := ruleConf.MustGet(ctx, "对比大模型.partExistsAndSame.score").Int()
+			for k, v := range partExistsFields {
+				if compare[k] == nil || aiZhipu[v] == nil || gconv.String(compare[k]) == gconv.String(aiZhipu[v]) {
+					continue
+				}
+				totalScore += score
+				log.Println(_id, "对比大模型，部分字段存在，", v, "的值不一致", score)
+			}
+		}
+	}
+	return totalScore
+}
+
+// 数据优选
+func Preferred(ctx context.Context, mainData map[string]interface{}, score int) {
+	_id := BsonIdToSId(mainData["_id"])
+	rid := gconv.String(mainData["repeat_id"])
+	isRepeat := gconv.Int(mainData["repeat"])
+	var miniPici, repeatPici int64
+	var bestId string = _id
+	var bestScore int = score
+	//找被我判重掉的数据
+	var allDatas []map[string]interface{}
+	wait := &sync.WaitGroup{}
+	lock := &sync.Mutex{}
+	for _, v := range []string{Result_20220219, Result_20220218} {
+		wait.Add(1)
+		go func(collection string) {
+			defer wait.Done()
+			if isRepeat == 1 && rid == "" {
+				return
+			}
+			findId := _id
+			if isRepeat == 1 {
+				findId = rid
+			}
+			datas, ok := db.Mgo_Extract.Find(collection, map[string]interface{}{"repeat_id": findId}, nil, extractSelectFields, false, -1, -1)
+			if ok && *datas != nil && len(*datas) > 0 {
+				lock.Lock()
+				allDatas = append(allDatas, *datas...)
+				lock.Unlock()
+			}
+		}(v)
+		if isRepeat == 1 && rid != "" {
+			wait.Add(1)
+			go func(collection string) {
+				defer wait.Done()
+				data, ok := db.Mgo_Extract.FindById(collection, rid, extractSelectFields)
+				if ok && *data != nil && len(*data) > 0 {
+					lock.Lock()
+					allDatas = append(allDatas, *data)
+					lock.Unlock()
+				}
+			}(v)
+		}
+	}
+	wait.Wait()
+	pool := make(chan bool, g.Config().MustGet(ctx, "preferPoolSize").Int())
+	repeatMap := map[string]bool{}
+	findIds := []interface{}{StringTOBsonId(_id)}
+	for _, vv := range allDatas {
+		thisId := BsonIdToSId(vv["_id"])
+		if thisId == _id || repeatMap[thisId] {
+			continue
+		}
+		repeatMap[thisId] = true
+		pool <- true
+		wait.Add(1)
+		go func(vm map[string]interface{}) {
+			defer func() {
+				<-pool
+				wait.Done()
+			}()
+			bidObj, _ := db.Mgo_Main.FindOneByField(Bidding, map[string]interface{}{"_id": vm["_id"]}, biddingSelectFields)
+			thisId := BsonIdToSId(vm["_id"])
+			if bidObj == nil || len(*bidObj) == 0 {
+				log.Println("bidding中没有找到", thisId)
+				return
+			}
+			for k, v := range *bidObj {
+				vm[k] = v
+			}
+			if pici := gconv.Int64(vm["pici"]); pici > 0 {
+				if miniPici == 0 || miniPici > pici {
+					miniPici = pici
+				}
+				if thisId == rid {
+					repeatPici = pici
+				}
+			}
+			preferScore := MakeScore(ctx, vm)
+			lock.Lock()
+			findIds = append(findIds, vm["_id"])
+			if preferScore > bestScore || (preferScore == bestScore && thisId > bestId) {
+				bestId = thisId
+				bestScore = preferScore
+			}
+			lock.Unlock()
+		}(vv)
+	}
+	wait.Wait()
+	updateIds := []interface{}{}
+	for _, v := range findIds {
+		if BsonIdToSId(v) == bestId {
+			continue
+		}
+		updateIds = append(updateIds, v)
+	}
+	prevBests, ok := db.Mgo_Main.Find(Bidding, map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$in": findIds,
+		},
+		"isprefer": 1,
+	}, `{"prefer_score":-1,"_id":1}`, `{"_id":1,"pici":1}`, false, 0, 1)
+	if !ok {
+		log.Println("查询上次优选id失败", findIds)
+		return
+	}
+	prevBestId := ""
+	var prevPici int64
+	if prevBests != nil && len(*prevBests) == 1 {
+		prevBestId = BsonIdToSId((*prevBests)[0]["_id"])
+		prevPici = gconv.Int64((*prevBests)[0]["pici"])
+	}
+	if prevBestId == "" {
+		prevBestId = rid
+	}
+	if prevPici == 0 {
+		prevPici = repeatPici
+	}
+	if prevPici == 0 {
+		prevPici = miniPici
+	}
+	//优选
+	if prevBestId == "" || prevBestId != bestId {
+		set := map[string]interface{}{
+			"isprefer":    1,
+			"prefertime":  time.Now().Unix(),
+			"extracttype": 1,
+		}
+		if prevPici > 0 {
+			set["pici"] = prevPici
+		}
+		if prevBestId != "" && prevBestId != bestId {
+			set["old_preferid"] = prevBestId
+		}
+		db.Mgo_Main.UpdateById(Bidding, bestId, map[string]interface{}{
+			"$set": set,
+		})
+		db.Mgo_Main.Save("bidding_prefer_log", map[string]interface{}{
+			"create_time":  time.Now().Unix(),
+			"old_preferid": prevBestId,
+			"preferid":     bestId,
+			"ids":          findIds,
+		})
+	}
+	//取消优选
+	db.Mgo_Main.Update(Bidding, map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$in": updateIds,
+		},
+		"isprefer": 1,
+	}, map[string]interface{}{
+		"$set": map[string]interface{}{
+			"isprefer":    -1,
+			"extracttype": -1,
+			"prefertime":  time.Now().Unix(),
+		},
+		"$unset": map[string]interface{}{"old_preferid": ""},
+	}, false, true)
+	//
+	db.Mgo_Main.Update(Bidding, map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$in": updateIds,
+		},
+		"extracttype": 1,
+	}, map[string]interface{}{
+		"$set": map[string]interface{}{
+			"extracttype": -1,
+		},
+	}, false, true)
+}
+
+// 数据优选
+func PreferredTest(ctx context.Context, mainData map[string]interface{}, score int) {
+	_id := BsonIdToSId(mainData["_id"])
+	rid := gconv.String(mainData["repeat_id"])
+	isRepeat := gconv.Int(mainData["repeat"])
+	var bestId string = _id
+	var bestScore int = score
+	//找被我判重掉的数据
+	var allDatas []map[string]interface{}
+	wait := &sync.WaitGroup{}
+	lock := &sync.Mutex{}
+	for _, v := range []string{Result_20220219, Result_20220218} {
+		wait.Add(1)
+		go func(collection string) {
+			defer wait.Done()
+			if isRepeat == 1 && rid == "" {
+				return
+			}
+			findId := _id
+			if isRepeat == 1 {
+				findId = rid
+			}
+			datas, ok := db.Mgo_Extract.Find(collection, map[string]interface{}{"repeat_id": findId}, nil, extractSelectFields, false, -1, -1)
+			if ok && *datas != nil && len(*datas) > 0 {
+				lock.Lock()
+				allDatas = append(allDatas, *datas...)
+				lock.Unlock()
+			}
+		}(v)
+		if isRepeat == 1 && rid != "" {
+			wait.Add(1)
+			go func(collection string) {
+				defer wait.Done()
+				data, ok := db.Mgo_Extract.FindById(collection, rid, extractSelectFields)
+				if ok && *data != nil && len(*data) > 0 {
+					lock.Lock()
+					allDatas = append(allDatas, *data)
+					lock.Unlock()
+				}
+			}(v)
+		}
+	}
+	wait.Wait()
+	pool := make(chan bool, g.Config().MustGet(ctx, "preferPoolSize").Int())
+	repeatMap := map[string]bool{}
+	findIds := []interface{}{StringTOBsonId(_id)}
+	groupMap := map[string]map[string]interface{}{_id: mainData}
+	for _, vv := range allDatas {
+		thisId := BsonIdToSId(vv["_id"])
+		if thisId == _id || repeatMap[thisId] {
+			continue
+		}
+		repeatMap[thisId] = true
+		pool <- true
+		wait.Add(1)
+		go func(vm map[string]interface{}) {
+			defer func() {
+				<-pool
+				wait.Done()
+			}()
+			preferScore := MakeScore(ctx, vm)
+			lock.Lock()
+			thisId := BsonIdToSId(vm["_id"])
+			groupMap[thisId] = vm
+			findIds = append(findIds, vm["_id"])
+			if preferScore > bestScore || (preferScore == bestScore && thisId > bestId) {
+				bestId = thisId
+				bestScore = preferScore
+			}
+			lock.Unlock()
+		}(vv)
+	}
+	wait.Wait()
+	//找master_id
+	//datas, ok := db.Mgo_Main.Find(Bidding, map[string]interface{}{
+	//	"_id": map[string]interface{}{
+	//		"$in": findIds,
+	//	},
+	//}, map[string]interface{}{"prefer_score": -1, "_id": 1}, map[string]interface{}{"_id": 0, "master_id": 1}, false, -1, -1)
+	//if !ok {
+	//	log.Println("bidding中找master_id出错", findIds)
+	//	return
+	//}
+	updateIds := []interface{}{}
+	existsMasterIds := map[string]bool{}
+	//if datas != nil && len(*datas) > 0 {
+	//	for _, v := range *datas {
+	//		if v["master_id"] == nil {
+	//			continue
+	//		}
+	//		if masterId == "" {
+	//			masterId = gconv.String(v["master_id"])
+	//		}
+	//		existsMasterIds[BsonIdToSId(v["_id"])] = true
+	//	}
+	//}
+	for _, v := range findIds {
+		if existsMasterIds[BsonIdToSId(v)] {
+			continue
+		}
+		updateIds = append(updateIds, v)
+	}
+	//保存最优的数据
+	data, ok := db.Mgo_Main.FindById(Bidding, bestId, nil)
+	if ok && data != nil && len(*data) > 0 {
+		delete(*data, "detail")
+		delete(*data, "contenthtml")
+		delete(*data, "master_id")
+		if oldObj := groupMap[bestId]; oldObj != nil {
+			if repeatId := gconv.String(oldObj["repeat_id"]); repeatId != "" {
+				(*data)["old_id"] = repeatId
+				if oldObj = groupMap[repeatId]; oldObj != nil {
+					if siteObj := allSite[gconv.String(oldObj["site"])]; siteObj != nil {
+						(*data)["old_site_toptype"] = siteObj.Toptype
+					}
+					(*data)["old_prefer_score"] = oldObj["prefer_score"]
+				}
+			}
+		}
+		if allSite[gconv.String((*data)["site"])] != nil {
+			(*data)["site_toptype"] = allSite[gconv.String((*data)["site"])].Toptype
+		}
+		(*data)["id"] = bestId
+		db.Mgo_Main.Update(Bidding_master, map[string]interface{}{"_id": (*data)["_id"]}, map[string]interface{}{"$set": *data}, true, false)
+	}
+}

service/service_test.go

@@ -0,0 +1,12 @@
+package service
+
+import (
+	"github.com/gogf/gf/v2/os/gctx"
+	"main/db"
+	"testing"
+)
+
+func TestScoring(t *testing.T) {
+	data, _ := db.Mgo_Extract.FindById(Result_20220219, "682cd75d5f834436f02824ee", selectFields)
+	Scoring(gctx.New(), *data)
+}