添加 补充 es 缺失数据

JyDatas/go.mod

@@ -4,9 +4,8 @@ go 1.16
 
 require (
 	github.com/BurntSushi/toml v1.2.0
-	github.com/robfig/cron v1.2.0 // indirect
-	github.com/spf13/cobra v1.5.0 // indirect
-	go.uber.org/zap v1.23.0 // indirect
-	gopkg.in/olivere/elastic.v2 v2.0.61 // indirect
-	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1 // indirect
+	github.com/robfig/cron v1.2.0
+	github.com/spf13/cobra v1.5.0
+	go.uber.org/zap v1.23.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
 )

JyDatas/go.sum

@@ -1,7 +1,3 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
-app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
-app.yhyue.com/data_processing/common_utils v0.0.0-20230615083858-55ea206e1f6e h1:jb995Ko1dqe6UnJVT5Gh6rBpAWgWEdtLhVDir4tyysk=
-app.yhyue.com/data_processing/common_utils v0.0.0-20230615083858-55ea206e1f6e/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
@@ -26,6 +22,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
@@ -44,9 +41,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -55,6 +50,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -63,16 +59,15 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
 github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
@@ -97,22 +92,16 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
-github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
-github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
@@ -123,14 +112,12 @@ go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
 go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
 go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -147,7 +134,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
@@ -155,7 +141,6 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -172,7 +157,6 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -205,13 +189,12 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/olivere/elastic.v2 v2.0.61 h1:7cpl3MW8ysa4GYFBXklpo5mspe4NK0rpZTdyZ+QcD4U=
-gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

escheck/es_test.go

@@ -17,8 +17,8 @@ import (
 
 //日报程序
 func TestR(t *testing.T) {
-	start := -2
-	end := -1
+	start := -1
+	end := 0
 
 	esQ := `{"query": {"range": {"id": {"gte": "%s","lt": "%s"}}}}`
 	esQ1 := `{"query": {"bool": {"must": [{"range": {"id": {"gte": "%s","lt": "%s"}}},{"terms": {"bidding.site": ["元博网（采购与招标网）","中国招标与采购网"]}}]}}}`
@@ -252,7 +252,7 @@ func TestR(t *testing.T) {
 	}
 
 	fmt.Println(insert)
-	Mgo.Save("bidding_ribao", insert)
+	//Mgo.Save("bidding_ribao", insert)
 
 	fmt.Println("success")
 }

escheck/id_find.go

@@ -0,0 +1,246 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	"go.mongodb.org/mongo-driver/bson"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
+	"log"
+	"net"
+	"sync"
+	"time"
+)
+
+var (
+	UdpClient udp.UdpClient
+	nextAddr  *net.UDPAddr
+)
+
+func findID() {
+	UdpClient = udp.UdpClient{Local: ":1199", BufSize: 1024}
+	nextAddr = &net.UDPAddr{
+		Port: util.IntAll(17833),
+		IP:   net.ParseIP("127.0.0.1"),
+	}
+	UdpClient.Listen(processUdpMsg)
+
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "127.0.0.1:27083",
+		MongodbAddr: "172.17.189.140:27080",
+		DbName:      "qfw",
+		Size:        10,
+		Direct:      true,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+	}
+	Mgo.InitPool()
+	start := -1
+	end := 0
+	st := util.GetDayStartSecond(start)              //
+	et := util.GetDayStartSecond(end)                //
+	startID := fmt.Sprintf("%x0000000000000000", st) //开始ID
+	endID := fmt.Sprintf("%x0000000000000000", et)   // 结束ID
+
+	//urla := "http://127.0.0.1:19805"
+	urla := "http://172.17.4.184:19805"
+	usernamea := "es_all"
+	passworda := "TopJkO2E_d1x"
+
+	//创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(urla),
+		elastic.SetBasicAuth(usernamea, passworda),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+
+	id1 := mongodb.StringTOBsonId(startID)
+	id2 := mongodb.StringTOBsonId(endID)
+	mq := bson.M{"_id": bson.M{"$gte": id1, "$lt": id2}} //一天时间内的id段
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	fd := bson.M{"extracttype": 1, "sensitive": 1, "dataging": 1, "site": 1, "infoformat": 1, "comeintime": 1, "pici": 1, "publishtime": 1, "competehref": 1, "attach_text": 1}
+	query := sess.DB("qfw").C("bidding").Find(mq).Select(fd).Iter()
+	count := 0
+	ch := make(chan bool, 15)
+	wg := &sync.WaitGroup{}
+
+	var ids = make([]string, 0)
+	var lock sync.Mutex
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current:", count, len(ids))
+		}
+
+		if util.IntAll(tmp["extracttype"]) != -1 && util.ObjToString(tmp["sensitive"]) != "测试" && util.IntAll(tmp["dataging"]) != 1 && util.Float64All(tmp["infoformat"]) != 3 {
+			ch <- true
+			wg.Add(1)
+			go func(tmp map[string]interface{}) {
+				defer func() {
+					<-ch
+					wg.Done()
+				}()
+				id := mongodb.BsonIdToSId(tmp["_id"])
+				exist, _ := documentExists(client, "bidding", id)
+				if !exist {
+					lock.Lock()
+					ids = append(ids, id)
+					lock.Unlock()
+				}
+			}(tmp)
+			tmp = map[string]interface{}{}
+		}
+	}
+
+	wg.Wait()
+
+	for _, v := range ids {
+		data := map[string]interface{}{
+			"stype":  "index_by_id",
+			"infoid": v,
+		}
+		SendUdpMsg(data, nextAddr)
+		time.Sleep(time.Second)
+	}
+
+	log.Println("over", len(ids))
+}
+
+func findIDHWY() {
+	UdpClient = udp.UdpClient{Local: ":1199", BufSize: 1024}
+	nextAddr = &net.UDPAddr{
+		Port: util.IntAll(17833),
+		IP:   net.ParseIP("127.0.0.1"),
+	}
+	UdpClient.Listen(processUdpMsg)
+
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "127.0.0.1:27083",
+		MongodbAddr: "172.17.189.140:27080",
+		DbName:      "qfw",
+		Size:        10,
+		Direct:      true,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+	}
+	Mgo.InitPool()
+	start := -1
+	end := 0
+	st := util.GetDayStartSecond(start)              //
+	et := util.GetDayStartSecond(end)                //
+	startID := fmt.Sprintf("%x0000000000000000", st) //开始ID
+	endID := fmt.Sprintf("%x0000000000000000", et)   // 结束ID
+
+	//urla := "http://127.0.0.1:19805"
+	urla := "http://172.17.4.184:19905"
+	usernamea := "jybid"
+	passworda := "Top2023_JEB01i@31"
+
+	//创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(urla),
+		elastic.SetBasicAuth(usernamea, passworda),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+
+	id1 := mongodb.StringTOBsonId(startID)
+	id2 := mongodb.StringTOBsonId(endID)
+	mq := bson.M{"_id": bson.M{"$gte": id1, "$lt": id2}} //一天时间内的id段
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	fd := bson.M{"extracttype": 1, "sensitive": 1, "dataging": 1, "site": 1, "infoformat": 1, "comeintime": 1, "pici": 1, "publishtime": 1, "competehref": 1, "attach_text": 1}
+	query := sess.DB("qfw").C("bidding").Find(mq).Select(fd).Iter()
+	count := 0
+	ch := make(chan bool, 15)
+	wg := &sync.WaitGroup{}
+
+	var ids = make([]string, 0)
+	var lock sync.Mutex
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current:", count, len(ids))
+		}
+
+		if util.IntAll(tmp["extracttype"]) != -1 && util.ObjToString(tmp["sensitive"]) != "测试" && util.IntAll(tmp["dataging"]) != 1 && util.Float64All(tmp["infoformat"]) != 3 {
+			ch <- true
+			wg.Add(1)
+			go func(tmp map[string]interface{}) {
+				defer func() {
+					<-ch
+					wg.Done()
+				}()
+				id := mongodb.BsonIdToSId(tmp["_id"])
+				exist, _ := documentExists(client, "bidding", id)
+				if !exist {
+					lock.Lock()
+					ids = append(ids, id)
+					lock.Unlock()
+				}
+			}(tmp)
+			tmp = map[string]interface{}{}
+		}
+	}
+
+	wg.Wait()
+
+	for _, v := range ids {
+		data := map[string]interface{}{
+			"stype":  "index_by_id",
+			"infoid": v,
+		}
+		SendUdpMsg(data, nextAddr)
+		time.Sleep(time.Second)
+	}
+
+	log.Println("over", len(ids))
+}
+
+//processUdpMsg 处理udp
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case udp.OP_TYPE_DATA:
+
+	case udp.OP_NOOP:
+		da := string(data)
+		log.Println("收到回复数据", da)
+
+	default:
+		fmt.Println("current_listen : processUdpMsg =====", act)
+	}
+}
+
+//SendUdpMsg 通知处理企业新增数据
+func SendUdpMsg(data map[string]interface{}, target *net.UDPAddr) {
+	bytes, _ := json.Marshal(data)
+	err := UdpClient.WriteUdp(bytes, udp.OP_TYPE_DATA, target)
+	if err != nil {
+		log.Println(err)
+	}
+	log.Println(data)
+	log.Println(target)
+
+}
+
+// documentExists 检查指定 ID 是否存在于 Elasticsearch 中
+func documentExists(client *elastic.Client, indexName, documentID string) (bool, error) {
+	exists, err := client.Exists().
+		Index(indexName).
+		Id(documentID).
+		Do(context.Background())
+	if err != nil {
+		return false, err
+	}
+
+	return exists, nil
+}

escheck/id_test.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	"go.mongodb.org/mongo-driver/bson"
+	"io"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func TestFindId(T *testing.T) {
+
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "127.0.0.1:27083",
+		DbName:      "qfw",
+		Size:        10,
+		Direct:      true,
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+	}
+	Mgo.InitPool()
+	start := -1
+	end := 0
+	st := util.GetDayStartSecond(start)              //
+	et := util.GetDayStartSecond(end)                //
+	startID := fmt.Sprintf("%x0000000000000000", st) //开始ID
+	endID := fmt.Sprintf("%x0000000000000000", et)   // 结束ID
+
+	urla := "http://127.0.0.1:19805"
+	usernamea := "es_all"
+	passworda := "TopJkO2E_d1x"
+
+	//创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(urla),
+		elastic.SetBasicAuth(usernamea, passworda),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+
+	id1 := mongodb.StringTOBsonId(startID)
+	id2 := mongodb.StringTOBsonId(endID)
+	mq := bson.M{"_id": bson.M{"$gte": id1, "$lt": id2}} //一天时间内的id段
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	var existsMap sync.Map
+
+	fd := bson.M{"extracttype": 1, "sensitive": 1, "dataging": 1, "site": 1, "infoformat": 1, "comeintime": 1, "pici": 1, "publishtime": 1, "competehref": 1, "attach_text": 1}
+	query := sess.DB("qfw").C("bidding").Find(mq).Select(fd).Iter()
+	count := 0
+	ch := make(chan bool, 15)
+	wg := &sync.WaitGroup{}
+
+	//var ids = make([]string, 0)
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current:", count)
+		}
+
+		if util.IntAll(tmp["extracttype"]) != -1 && util.ObjToString(tmp["sensitive"]) != "测试" && util.IntAll(tmp["dataging"]) != 1 && util.Float64All(tmp["infoformat"]) != 3 {
+			ch <- true
+			wg.Add(1)
+			go func(tmp map[string]interface{}) {
+				defer func() {
+					<-ch
+					wg.Done()
+				}()
+				id := mongodb.BsonIdToSId(tmp["_id"])
+				exist, _ := documentExists(client, "bidding", id)
+				if !exist {
+					existsMap.Store(id, id)
+				}
+			}(tmp)
+			tmp = map[string]interface{}{}
+
+		}
+
+	}
+
+	wg.Wait()
+
+	existsMap.Range(func(key, _ interface{}) bool {
+		fmt.Println(key)
+		return true
+	})
+
+	log.Println("over")
+}
+
+// getElasticsearchIDs 获取 Elasticsearch 中的 ID 列表
+func getElasticsearchIDs(client *elastic.Client, indexName string, startID, endID string) ([]string, error) {
+	query := elastic.NewRangeQuery("id").Gte(startID).Lte(endID)
+	ctx := context.Background()
+	//开始滚动搜索
+	scrollID := ""
+	scroll := "1m"
+	searchSource := elastic.NewSearchSource().
+		Query(query).
+		Size(10000).
+		Sort("_doc", false) //降序排序
+	searchService := client.Scroll(indexName).
+		Size(10000).
+		Scroll(scroll).
+		SearchSource(searchSource)
+
+	res, err := searchService.Do(ctx)
+
+	if err != nil {
+		if err == io.EOF {
+			fmt.Println("没有数据")
+		} else {
+			panic(err)
+		}
+
+	}
+	defer client.ClearScroll().ScrollId(scrollID).Do(ctx) // 在退出时清理资源
+	fmt.Println("总数是：", res.TotalHits())
+
+	var ids []string
+	total := 0
+	for len(res.Hits.Hits) > 0 {
+		for _, hit := range res.Hits.Hits {
+			ids = append(ids, hit.Id)
+		}
+		total = total + len(res.Hits.Hits)
+		scrollID = res.ScrollId
+		res, err = client.Scroll().ScrollId(scrollID).Scroll(scroll).Do(ctx)
+
+		fmt.Println("current count:", total)
+		if err != nil {
+			if err == io.EOF {
+				// 滚动到最后一批数据，退出循环
+				break
+			}
+			log.Printf("滚动搜索失败：%s", err)
+			break // 处理错误时退出循环
+		}
+	}
+	fmt.Println("结束~~~~~~~~~~~~~~~")
+	return ids, nil
+}
+
+// convertToObjectId 将 Elasticsearch 的 ID 转换为 MongoDB 的 ObjectId
+func convertToObjectId(ids []string) []interface{} {
+	var objectIDs []interface{}
+	for _, id := range ids {
+		objectIDs = append(objectIDs, strings.TrimSpace(id))
+	}
+	return objectIDs
+}
+
+// documentExists 检查指定 ID 是否存在于 Elasticsearch 中
+//func documentExists(client *elastic.Client, indexName, documentID string) (bool, error) {
+//	exists, err := client.Exists().
+//		Index(indexName).
+//		Id(documentID).
+//		Do(context.Background())
+//	if err != nil {
+//		return false, err
+//	}
+//
+//	return exists, nil
+//}

escheck/main.go

@@ -43,7 +43,7 @@ var (
 	esQ1                        = `{"query": {"bool": {"must": [{"range": {"id": {"gte": "%s","lt": "%s"}}},{"terms": {"bidding.site": ["元博网（采购与招标网）","中国招标与采购网"]}}]}}}`
 )
 
-func init() {
+func Init() {
 	util.ReadConfig(&config)
 	jkmail := config["jkmail"].(map[string]interface{})
 	to, _ = jkmail["to"].(string)
@@ -72,6 +72,7 @@ func init() {
 }
 
 func main() {
+	Init()
 	log.Println("start..")
 	if len(Ts) > 0 {
 		local, _ := time.LoadLocation("Asia/Shanghai")