添加es地址3;支持bidding增量 升索引地址1,2,3；项目生索引地址1,3

createEsIndex/bidding_es.go

@@ -321,6 +321,7 @@ func dealData(coll, gtid, lteid, kword string, routines int) {
 			"$lte": mongodb.StringTOBsonId(lteid),
 		},
 	}
+	log.Info("dealData", zap.Any(kword, q))
 	biddingConn := MgoB.GetMgoConn()
 	it := biddingConn.DB(config.Conf.DB.MongoB.Dbname).C(coll).Find(&q).Select(map[string]interface{}{
 		"contenthtml": 0,
@@ -371,7 +372,8 @@ func dealData(coll, gtid, lteid, kword string, routines int) {
 					"set": update,
 				}
 			}
-			saveEsPool <- newTmp
+			//saveEsPool <- newTmp
+			saveBiddingAllPool <- newTmp
 		}(tmp)
 		tmp = map[string]interface{}{}
 	}

createEsIndex/common.toml

@@ -94,10 +94,17 @@ detailfilter = ["(招标网|千里马|采招网|招标采购导航网|招标与
     "[\\(（]?(网址)?(:：)?(http|https|htpps)*(:|：)?\\/\\/www.bidcenter.com.cn\\/",
     "千里马(平台|网站)+", "[“\"]?优质采(平台|电子交易平台|云采购平台|交易平台)?[”\"]?", "《?(中国采购与|中国)?招(投)?标(与采购|采购导航)?网》?",
     "《?元博网(采购与招标网)?》?", "《?(中国)?招标采购导航网》?", "中\\W{0,3}国采\\W{0,3}招\\W{0,3}网\\W*[(（]?(bidcenter.com.cn)?[)）]?", "已方宝", "中国招标与采购"]
+## 华为云集群1
 addr2 = "http://127.0.0.1:19905"
 username2 = "jybid"
 password2 = "Top2023_JEB01i@31"
 indexb2 = "bidding"
+## 华为云新集群2，原来阿里云数据迁移华为云新集群
+addr3 = "http://127.0.0.1:19908"
+username3 = "jybid"
+password3 = "Top2023_JEB01i@31"
+indexb3 = "bidding"
+
 
 ## 预处理数据 索引配置
 #[[pre]]

createEsIndex/config/conf.go

@@ -148,6 +148,10 @@ type es struct {
 	Username2 string
 	Password2 string
 	Indexb2   string
+	Addr3     string // 华为云新集群，存储原来阿里云集群的标讯、项目数据
+	Username3 string
+	Password3 string
+	Indexb3   string
 }
 
 // PreConf 预处理 配置

createEsIndex/go.mod

@@ -3,66 +3,45 @@ module esindex
 go 1.18
 
 require (
+	github.com/RoaringBitmap/roaring v1.9.1
 	github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible
-	github.com/nats-io/nats.go v1.31.0
+	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/robfig/cron v1.2.0
 	github.com/spf13/viper v1.15.0
 	go.mongodb.org/mongo-driver v1.10.2
 	go.uber.org/zap v1.23.0
-	jygit.jydev.jianyu360.cn/BP/jynats v0.0.0-20231206094405-2ff9da3175bc
 	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231223031213-3f719e173cb5
 )
 
 require (
-	github.com/BurntSushi/toml v1.2.0 // indirect
 	github.com/PuerkitoBio/goquery v1.8.0 // indirect
-	github.com/RoaringBitmap/roaring v1.9.0 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
-	github.com/bits-and-blooms/bitset v1.12.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/clbanning/mxj/v2 v2.7.0 // indirect
+	github.com/bits-and-blooms/bitset v1.13.0 // indirect
 	github.com/dchest/captcha v1.0.0 // indirect
-	github.com/fatih/color v1.15.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
-	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
-	github.com/gogf/gf/v2 v2.5.7 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
-	github.com/grokify/html-strip-tags-go v0.0.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/jinzhu/inflection v1.0.0 // indirect
-	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
-	github.com/nats-io/nkeys v0.4.5 // indirect
-	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.2 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.opentelemetry.io/otel v1.14.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.14.0 // indirect
-	go.opentelemetry.io/otel/trace v1.14.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect

createEsIndex/go.sum

@@ -42,8 +42,8 @@ github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
-github.com/RoaringBitmap/roaring v1.9.0 h1:lwKhr90/j0jVXJyh5X+vQN1VVn77rQFfYnh6RDRGCcE=
-github.com/RoaringBitmap/roaring v1.9.0/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
+github.com/RoaringBitmap/roaring v1.9.1 h1:LXcSqGGGMKm+KAzUyWn7ZeREqoOkoMX+KwLOK1thc4I=
+github.com/RoaringBitmap/roaring v1.9.1/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
 github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible h1:QoRMR0TCctLDqBCMyOu1eXdZyMw3F7uGA9qPn2J4+R8=
 github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
@@ -51,16 +51,15 @@ github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEq
 github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
 github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
+github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME=
-github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -76,8 +75,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
@@ -87,16 +84,10 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
-github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
-github.com/gogf/gf/v2 v2.5.7 h1:h+JSoD6z3d2q0uGszvtahrSm4DiM2ECyNjyTwKIo8wE=
-github.com/gogf/gf/v2 v2.5.7/go.mod h1:x2XONYcI4hRQ/4gMNbWHmZrNzSEIg20s2NULbzom5k0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -158,20 +149,12 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
-github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
-github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0=
-github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
-github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
-github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
-github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -192,29 +175,13 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
-github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
-github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
-github.com/nats-io/nkeys v0.4.5 h1:Zdz2BUlFm4fJlierwvGK+yl20IAKUm7eV6AAZXEhkPk=
-github.com/nats-io/nkeys v0.4.5/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
-github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
-github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
-github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
-github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
@@ -227,9 +194,6 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
 github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -260,6 +224,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
@@ -288,13 +253,7 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
-go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
-go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
-go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY=
-go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM=
 go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
-go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
-go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@@ -448,8 +407,6 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -627,11 +584,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/mysql v1.5.2 h1:QC2HRskSE75wBuOxe0+iCkyJZ+RqpudsQtqkp+IMuXs=
-gorm.io/driver/mysql v1.5.2/go.mod h1:pQLhh1Ut/WUAySdTHwBpBv6+JKcj+ua4ZFx1QQTBzb8=
-gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
-gorm.io/gorm v1.25.6 h1:V92+vVda1wEISSOMtodHVRcUIOPYa2tgQtyF+DfFx+A=
-gorm.io/gorm v1.25.6/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -639,10 +591,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-jygit.jydev.jianyu360.cn/BP/jynats v0.0.0-20231206094405-2ff9da3175bc h1:/vhHYTg6vfynVJq03PFLhLMOxZKzUGIBGWpRL1ps6Fg=
-jygit.jydev.jianyu360.cn/BP/jynats v0.0.0-20231206094405-2ff9da3175bc/go.mod h1:bbyXOaS5NyeNIit6mb5fV+Exxqg8g+Su0JnU8u8i2Xs=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231122020338-4956718a7e9e h1:oXoOPTtZIfnvce0ulokHBSjXqhLiN6DPVOYTm3V6z8U=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231122020338-4956718a7e9e/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
 jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231223031213-3f719e173cb5 h1:IJlZ+JTn7UvVeHyALb+yWacmtE94TW2XvBIRgTyRmzU=
 jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20231223031213-3f719e173cb5/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

createEsIndex/init.go

@@ -197,6 +197,17 @@ func InitEs() {
 		Es2.InitElasticSize()
 		log.Info("InitEs", zap.String("华为云Addr2", config.Conf.DB.Es.Addr2))
 	}
+	// 华为云新集群，迁移原来阿里云集群数据，标讯、项目、凭安
+	if config.Conf.DB.Es.Addr3 != "" {
+		Es3 = &elastic.Elastic{
+			S_esurl:  config.Conf.DB.Es.Addr3,
+			I_size:   config.Conf.DB.Es.Size,
+			Username: config.Conf.DB.Es.Username3,
+			Password: config.Conf.DB.Es.Password3,
+		}
+		Es3.InitElasticSize()
+		log.Info("InitEs", zap.String("华为云Addr3", config.Conf.DB.Es.Addr3))
+	}
 
 	log.Info("InitEs", zap.Any("duration", time.Since(now).Seconds()))
 }
@@ -233,20 +244,20 @@ func InitPreProcessField() {
 }
 
 // InitPreEsClient 实例化预处理 索引客户端
-func InitPreEsClient() {
-	if len(config.Conf.Pre) > 0 {
-		for k, v := range config.Conf.Pre {
-			cli := &elastic.Elastic{
-				S_esurl:  v.Addr,
-				I_size:   30,
-				Username: v.Username,
-				Password: v.Password,
-			}
-			cli.InitElasticSize()
-			PreEs[k] = cli
-		}
-	}
-}
+//func InitPreEsClient() {
+//	if len(config.Conf.Pre) > 0 {
+//		for k, v := range config.Conf.Pre {
+//			cli := &elastic.Elastic{
+//				S_esurl:  v.Addr,
+//				I_size:   30,
+//				Username: v.Username,
+//				Password: v.Password,
+//			}
+//			cli.InitElasticSize()
+//			PreEs[k] = cli
+//		}
+//	}
+//}
 
 // InitEsBiddingField 初始化 bidding 索引字段
 func InitEsBiddingField() {
@@ -415,9 +426,13 @@ func InitBitmap() {
 			log.Info("InitBitmap", zap.Error(err))
 		}
 		if len(bs) > 0 {
-			cache.FromBuffer(bs)
+			_, err := cache.FromBuffer(bs)
+			if err != nil {
+				log.Info("InitBitmap", zap.Any("cache.FromBuffer", err))
+			}
 		}
 	}
+	log.Info("InitBitmap", zap.Any("cache.FromBuffer", "success"))
 	//监听，写入文件保存
 	go func() {
 		for {

createEsIndex/main.go

@@ -30,8 +30,8 @@ var (
 	MgoS  *mongodb.MongodbSim
 	Mysql *mysqldb.Mysql
 
-	Es, Es2 *elastic.Elastic
-	PreEs   = make(map[string]*elastic.Elastic, 0) //预处理 索引客户端
+	Es, Es2, Es3 *elastic.Elastic
+	//PreEs        = make(map[string]*elastic.Elastic, 0) //预处理 索引客户端
 
 	UdpClient  udp.UdpClient
 	UdpTaskMap = &sync.Map{}
@@ -48,6 +48,9 @@ var (
 	saveEsAllPool     = make(chan map[string]interface{}, 5000) //存储单机版es,爬虫采集判重使用
 	saveEsAllSp       = make(chan bool, 5)
 
+	saveBiddingAllPool  = make(chan map[string]interface{}, 5000) //保存binding数据到es,stype=bidding_all_data
+	saveBiddingAllBEsSp = make(chan bool, 5)
+
 	saveErrBidPool = make(chan map[string]interface{}, 5000)
 	saveBidSp      = make(chan bool, 5)
 
@@ -108,6 +111,7 @@ func main() {
 	go SaveEsMethod()
 	go SaveAllEsMethod()
 	go SaveProjectEs()
+	go SaveBiddingAllDataEs()
 
 	go SaveBidErr()
 	//添加预处理函数
@@ -398,7 +402,7 @@ func SaveBidErr() {
 	}
 }
 
-// SaveEsMethod 保存到es
+// SaveEsMethod 保存bidding数据到es
 func SaveEsMethod() {
 	//arru := make([]map[string]interface{}, EsBulkSize)
 	//indexu := 0
@@ -408,11 +412,18 @@ func SaveEsMethod() {
 			id := v["id"]
 			ids := v["_id"]
 			Es.Save(config.Conf.DB.Es.IndexB, v)
+			// 华为云集群1
 			if config.Conf.DB.Es.Addr2 != "" {
 				v["id"] = id
 				v["_id"] = ids
 				Es2.Save(config.Conf.DB.Es.Indexb2, v)
 			}
+			// 华为云新集群2，迁移原来阿里云数据
+			if config.Conf.DB.Es.Addr3 != "" {
+				v["id"] = id
+				v["_id"] = ids
+				Es3.Save(config.Conf.DB.Es.Indexb3, v)
+			}
 
 			//arru[indexu] = v
 			//indexu++
@@ -498,6 +509,45 @@ func SaveAllEsMethod() {
 	}
 }
 
+// SaveBiddingAllDataEs 保存bidding数据到Es,stype=bidding_all_data 数据
+func SaveBiddingAllDataEs() {
+	arru := make([]map[string]interface{}, EsBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveBiddingAllPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == EsBulkSize {
+				saveBiddingAllBEsSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveBiddingAllBEsSp
+					}()
+					//1.阿里云集群
+					Es.BulkSave(config.Conf.DB.Es.IndexB, arru)
+				}(arru)
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveBiddingAllBEsSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveBiddingAllBEsSp
+					}()
+					//1.阿里云集群
+					Es.BulkSave(config.Conf.DB.Es.IndexB, arru)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+// SaveProjectEs 保存项目索引数据
 func SaveProjectEs() {
 	arru := make([]map[string]interface{}, EsBulkSize)
 	indexu := 0
@@ -513,6 +563,10 @@ func SaveProjectEs() {
 						<-saveProjectSp
 					}()
 					Es.BulkSave(config.Conf.DB.Es.IndexP, arru)
+					// 华为云新集群，存储标讯、项目、凭安数据
+					if config.Conf.DB.Es.Addr3 != "" {
+						Es3.BulkSave(config.Conf.DB.Es.IndexP, arru)
+					}
 				}(arru)
 				arru = make([]map[string]interface{}, EsBulkSize)
 				indexu = 0
@@ -525,6 +579,10 @@ func SaveProjectEs() {
 						<-saveProjectSp
 					}()
 					Es.BulkSave(config.Conf.DB.Es.IndexP, arru)
+					// 华为云新集群，存储标讯、项目、凭安数据
+					if config.Conf.DB.Es.Addr3 != "" {
+						Es3.BulkSave(config.Conf.DB.Es.IndexP, arru)
+					}
 				}(arru[:indexu])
 				arru = make([]map[string]interface{}, EsBulkSize)
 				indexu = 0

createEsIndex/pre_bidding.go

@@ -1,146 +0,0 @@
-package main
-
-import (
-	"esindex/config"
-	"fmt"
-	"go.uber.org/zap"
-	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
-	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
-	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
-	"strconv"
-	"time"
-)
-
-var (
-	savePreEsPool = make(chan map[string]interface{}, 5000) //保存binding_pre 数据到es
-	savePreEsSp   = make(chan bool, 5)
-)
-
-/*
-	预处理数据，先读取bidding_nomal bidding_file 数据生索引，先不判重、读取附件等
-	读取 bidding_extract 数据，读取完打标签，is_pre = 1,表示已经生了索引
-*/
-
-//dealPreProcess 处理预处理数据
-func dealPreProcess() {
-	sess := MgoB.GetMgoConn()
-	defer MgoB.DestoryMongoConn(sess)
-
-	where := map[string]interface{}{
-		"is_pre": map[string]interface{}{
-			"$exists": 0,
-		},
-	}
-
-	for {
-		it := sess.DB(config.Conf.DB.MongoB.Dbname).C("bidding_extract").Find(&where).Select(nil).Iter()
-		c1, index := 0, 0
-		for tmp := make(map[string]interface{}); it.Next(tmp); c1++ {
-			if len(tmp) == 0 {
-				continue
-			}
-			id := mongodb.BsonIdToSId(tmp["_id"])
-			newTmp, update := GetEsField(tmp, "pre_process")
-
-			insert := make(map[string]interface{}, 0)
-			for k, _ := range PreProcessField {
-				if data, ok := newTmp[k]; ok {
-					insert[k] = data
-				}
-			}
-
-			if len(insert) > 0 {
-				insert["dataweight"] = 0 //索引数据新增 jy置顶字段
-				insert["_id"] = insert["id"]
-				insert["fromtable"] = tmp["fromtable"]
-				//针对中国政府采购网,单独处理
-				if util.ObjToString(tmp["site"]) == "中国政府采购网" {
-					objectType := MatchService(tmp)
-					if objectType != "" {
-						insert["object_type"] = objectType
-					}
-				}
-				update["is_pre"] = 1
-
-				bs := MgoB.UpdateById("bidding_extract", id, map[string]interface{}{
-					"$set": update,
-				})
-
-				if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
-					tmp = make(map[string]interface{})
-					continue
-				}
-				//针对产权数据，暂时不入es 索引库
-				if util.IntAll(tmp["infoformat"]) == 3 {
-					continue
-				}
-
-				if bs {
-					index++
-					savePreEsPool <- insert
-				}
-			}
-
-		}
-
-		if c1 > 0 {
-			log.Info("dealPreProcess - over", zap.Int("count", c1), zap.Int("index", index))
-		}
-
-		time.Sleep(time.Second * 10)
-
-	}
-
-}
-
-//SavePreEsMethod 保存到bidding_pre
-func SavePreEsMethod() {
-	arru := make([]map[string]interface{}, EsBulkSize)
-	indexu := 0
-	for {
-		month := int(time.Now().Month())
-		monthStr := strconv.Itoa(month)
-		year := time.Now().Year()
-		yearStr := strconv.Itoa(year)
-		//预处理索引名称
-		preBiddingIndex := fmt.Sprintf("bidding_%s%s", yearStr, monthStr)
-		if config.Conf.Env.SpecType == "day" {
-			preBiddingIndex = preBiddingIndex + strconv.Itoa(time.Now().Day())
-		} else if config.Conf.Env.SpecType == "hour" {
-			preBiddingIndex = preBiddingIndex + strconv.Itoa(time.Now().Day()) + strconv.Itoa(time.Now().Hour())
-		}
-
-		select {
-		case v := <-savePreEsPool:
-			arru[indexu] = v
-			indexu++
-			if indexu == EsBulkSize {
-				savePreEsSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-savePreEsSp
-					}()
-					for _, client := range PreEs {
-						client.BulkSave(preBiddingIndex, arru)
-					}
-				}(arru)
-				arru = make([]map[string]interface{}, EsBulkSize)
-				indexu = 0
-			}
-		case <-time.After(1000 * time.Millisecond):
-			if indexu > 0 {
-				savePreEsSp <- true
-				go func(arru []map[string]interface{}) {
-					defer func() {
-						<-savePreEsSp
-					}()
-					for _, client := range PreEs {
-						client.BulkSave(preBiddingIndex, arru)
-					}
-				}(arru[:indexu])
-				arru = make([]map[string]interface{}, EsBulkSize)
-				indexu = 0
-			}
-		}
-	}
-}

createEsIndex/utils.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"esindex/config"
 	"fmt"
 	"github.com/cespare/xxhash/v2"
 	"go.mongodb.org/mongo-driver/bson"
@@ -16,7 +15,6 @@ import (
 	"os"
 	"regexp"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 	"unicode"
@@ -434,91 +432,98 @@ func SendUdpMsg(data map[string]interface{}, target *net.UDPAddr) {
 }
 
 // deletePreEsData 删除预处理索引数据
-func deletePreEsData(preId string) {
-	now := time.Now()
-	month := int(time.Now().Month())
-	monthStr := strconv.Itoa(month)
-	year := time.Now().Year()
-	yearStr := strconv.Itoa(year)
-	//当前处理索引名称
-	preBiddingIndex := fmt.Sprintf("bidding_%s%s", yearStr, monthStr)
-	lastIndex := ""
-
-	//按小时创建
-	if config.Conf.Env.SpecType == "hour" {
-		preBiddingIndex = preBiddingIndex + strconv.Itoa(now.Day()) + strconv.Itoa(now.Hour())
-		last := now.Add(-time.Hour)
-		month2 := int(last.Month())
-		monthStr2 := strconv.Itoa(month2)
-		year2 := last.Year()
-		yearStr2 := strconv.Itoa(year2)
-		dayStr2 := strconv.Itoa(last.Day())
-		//上个索引名称
-		lastIndex = "bidding_" + yearStr2 + monthStr2
-		lastIndex = lastIndex + dayStr2 + strconv.Itoa(last.Hour())
-
-	} else if config.Conf.Env.SpecType == "day" {
-		//按天创建
-		preBiddingIndex = preBiddingIndex + strconv.Itoa(time.Now().Day())
-		last := now.AddDate(0, 0, -1)
-		month2 := int(last.Month())
-		monthStr2 := strconv.Itoa(month2)
-		year2 := last.Year()
-		yearStr2 := strconv.Itoa(year2)
-		dayStr2 := strconv.Itoa(last.Day())
-		//上个索引名称
-		lastIndex = "bidding_" + yearStr2 + monthStr2
-		lastIndex = lastIndex + dayStr2
-
-	} else if config.Conf.Env.SpecType == "month" {
-		// 月份；
-		last := now.AddDate(0, -1, 0)
-		month2 := int(last.Month())
-		monthStr2 := strconv.Itoa(month2)
-		year2 := last.Year()
-		yearStr2 := strconv.Itoa(year2)
-		//上个索引名称
-		lastIndex = "bidding_" + yearStr2 + monthStr2
-	}
-
-	//删除预处理 索引数据
-	if len(PreEs) == 0 {
-		time.Sleep(time.Second)
-	}
-	for _, client := range PreEs {
-		if client == nil {
-			continue
-		}
-		// 老索引有数据
-		if client.Count(lastIndex, nil) > 0 {
-			err := client.DeleteByID(lastIndex, preId)
-			if err != nil {
-				fmt.Println("deletePreEsData: ", preId, err)
-			}
-		}
-
-		err := client.DeleteByID(preBiddingIndex, preId)
-		if err != nil {
-			fmt.Println("deletePreEsData: ", preId, err)
-		}
-	}
-
-}
+//func deletePreEsData(preId string) {
+//	now := time.Now()
+//	month := int(time.Now().Month())
+//	monthStr := strconv.Itoa(month)
+//	year := time.Now().Year()
+//	yearStr := strconv.Itoa(year)
+//	//当前处理索引名称
+//	preBiddingIndex := fmt.Sprintf("bidding_%s%s", yearStr, monthStr)
+//	lastIndex := ""
+//
+//	//按小时创建
+//	if config.Conf.Env.SpecType == "hour" {
+//		preBiddingIndex = preBiddingIndex + strconv.Itoa(now.Day()) + strconv.Itoa(now.Hour())
+//		last := now.Add(-time.Hour)
+//		month2 := int(last.Month())
+//		monthStr2 := strconv.Itoa(month2)
+//		year2 := last.Year()
+//		yearStr2 := strconv.Itoa(year2)
+//		dayStr2 := strconv.Itoa(last.Day())
+//		//上个索引名称
+//		lastIndex = "bidding_" + yearStr2 + monthStr2
+//		lastIndex = lastIndex + dayStr2 + strconv.Itoa(last.Hour())
+//
+//	} else if config.Conf.Env.SpecType == "day" {
+//		//按天创建
+//		preBiddingIndex = preBiddingIndex + strconv.Itoa(time.Now().Day())
+//		last := now.AddDate(0, 0, -1)
+//		month2 := int(last.Month())
+//		monthStr2 := strconv.Itoa(month2)
+//		year2 := last.Year()
+//		yearStr2 := strconv.Itoa(year2)
+//		dayStr2 := strconv.Itoa(last.Day())
+//		//上个索引名称
+//		lastIndex = "bidding_" + yearStr2 + monthStr2
+//		lastIndex = lastIndex + dayStr2
+//
+//	} else if config.Conf.Env.SpecType == "month" {
+//		// 月份；
+//		last := now.AddDate(0, -1, 0)
+//		month2 := int(last.Month())
+//		monthStr2 := strconv.Itoa(month2)
+//		year2 := last.Year()
+//		yearStr2 := strconv.Itoa(year2)
+//		//上个索引名称
+//		lastIndex = "bidding_" + yearStr2 + monthStr2
+//	}
+//
+//	//删除预处理 索引数据
+//	if len(PreEs) == 0 {
+//		time.Sleep(time.Second)
+//	}
+//	for _, client := range PreEs {
+//		if client == nil {
+//			continue
+//		}
+//		// 老索引有数据
+//		if client.Count(lastIndex, nil) > 0 {
+//			err := client.DeleteByID(lastIndex, preId)
+//			if err != nil {
+//				fmt.Println("deletePreEsData: ", preId, err)
+//			}
+//		}
+//
+//		err := client.DeleteByID(preBiddingIndex, preId)
+//		if err != nil {
+//			fmt.Println("deletePreEsData: ", preId, err)
+//		}
+//	}
+//
+//}
 
 // saveDb 文件写入
 func saveDb() {
 	mutex.Lock()
 	defer mutex.Unlock()
 	// 如果 cache 为空，则无需执行写入操作
-	if cache.GetCardinality() == 0 {
-		return
+	if cache == nil {
+		log.Error("saveDb", zap.Any("cache", "为空"))
 	}
-	fo, err := os.OpenFile(*dbfile, os.O_CREATE|os.O_RDWR|os.O_SYNC|os.O_TRUNC, 0777)
-	if err != nil {
-		log.Info("saveDb", zap.Error(err))
+
+	if cache.GetCardinality() > 0 {
+		fo, err := os.OpenFile(*dbfile, os.O_CREATE|os.O_RDWR|os.O_SYNC|os.O_TRUNC, 0777)
+		if err != nil {
+			log.Info("saveDb", zap.Error(err))
+		}
+
+		defer fo.Close()
+		_, err = cache.WriteTo(fo)
+		if err != nil {
+			log.Info("saveDb", zap.Any("cache.WriteTo", err))
+		}
 	}
-	defer fo.Close()
-	cache.WriteTo(fo)
 }
 
 // getNewName 获取新的不重复名称