cm

.gitignore

@@ -1,2 +1 @@
-field_sync
-field_dispose
+*_linux

README.md

@@ -41,6 +41,12 @@
 + bidding_file数据累计异常监控（累计数据量>10000条）  
 
 ## data_fusion 数据融合-基于判重数据进行的信息融合
++ 基于判重数据进行数据的融合
+  + 模版数据选取规则，基于给定字段打分(1-3)标准进行打分
+  + 竞品网站数据打分结果为0
+  + 打分结果一致的情况下，优先选择非重复数据为模板数据
+  + redis存储规则，key: repeat_id，value：模版id-id1-id2-idx
+  + 融合记录表，_id是repeat_id
 
 ## data_clear_sync 数据清洗同步线上
 + 数据清洗平台清洗过的数据同步至bidding表、es表、项目表中

data_clear_sync/common.toml

@@ -11,22 +11,23 @@ user = "root"
 password = "=PDT49#80Z!RVv52_z"
 
 [db.mongo]
-addr = "192.168.3.207:29099"
+addr = "192.168.3.71:29099"
 dbname = "wjh"
 size = 5
 user = ""
 password = ""
 [db.mongo1]
-addr = "192.168.3.207:29099"
+addr = "192.168.3.71:29099"
 dbname = "wjh"
 coll = "marked"
 size = 5
 user = ""
 password = ""
 [db.mongo2]
-addr = "192.168.3.207:29099"
+addr = "192.168.3.71:29099"
 dbname = "wjh"
 size = 5
+coll = "marked"
 user = ""
 password = ""
 

data_clear_sync/go.mod

@@ -3,8 +3,8 @@ module data_clear_sync
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20230510082025-6d175201278b
 	github.com/BurntSushi/toml v1.2.1
 	go.mongodb.org/mongo-driver v1.10.1
 	go.uber.org/zap v1.24.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67
 )

data_clear_sync/go.sum

@@ -1,5 +1,3 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20230510082025-6d175201278b h1:LuxfO97Gi4BzZlhNcZC9S3tk+5Z/0IMXpiyHMY4RcGk=
-app.yhyue.com/data_processing/common_utils v0.0.0-20230510082025-6d175201278b/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
@@ -209,3 +207,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67 h1:5rabE3IQRjI/NXfgJyquRTMdzSy7SfufFhmREixLZXg=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

data_clear_sync/init.go

@@ -1,11 +1,11 @@
 package main
 
 import (
-	"app.yhyue.com/data_processing/common_utils/elastic"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"data_clear_sync/config"
 	"fmt"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"os"
 )
 

data_clear_sync/main.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	"app.yhyue.com/data_processing/common_utils/redis"
 	"data_clear_sync/config"
 	"flag"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
 	"log"
 	"time"
 )
@@ -137,7 +137,7 @@ func updateEsFuc() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk("bidding_v1", "bidding", arru...)
+					Es.UpdateBulk("bidding", arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0
@@ -149,7 +149,7 @@ func updateEsFuc() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk("bidding_v1", "bidding", arru...)
+					Es.UpdateBulk("bidding", arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0

data_clear_sync/task.go

@@ -1,14 +1,14 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/redis"
 	"data_clear_sync/config"
 	"fmt"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
 	"strings"
 	"sync"
 	"time"
@@ -127,7 +127,114 @@ func taskinfoV1() {
 
 func taskinfoV2() {
 	defer util.Catch()
+	sess := MongoV2.GetMgoConn()
+	defer MongoV2.DestoryMongoConn(sess)
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	//q := bson.M{"_id": mongodb.StringTOBsonId("6229797d32770a446e976450")}
+	f := bson.M{"v_baseinfo.contenthtml": 0, "v_baseinfo.description": 0, "v_baseinfo.detail": 0, "v_baseinfo.kvtext": 0, "v_baseinfo.projectinfo": 0}
+	it := sess.DB(config.Conf.DB.Mongo2.Dbname).C(config.Conf.DB.Mongo2.Coll).Find(nil).Select(&f).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(tmp); count++ {
+		if count%2000 == 0 {
+			log.Info("taskinfoV2", zap.Int("current:", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			binfo := util.ObjToMap(tmp["v_baseinfo"])
+			tinfo := util.ObjToMap(tmp["v_taginfo"])
+			info, _ := Mongo.FindById("bidding", id, f)
+			coll := "bidding"
+			if len(*info) == 0 {
+				info, _ = Mongo.FindById("bidding_back", id, f)
+				if len(*info) == 0 {
+					util.Debug("not find id", id)
+					return
+				}
+				coll = "bidding_back"
+			}
+			update := make(map[string]interface{})
+			updateEs := make(map[string]interface{})
+			mdinfo := make(map[string]interface{})
+			record := make(map[string]interface{})
+			for _, f := range Fields {
+				if f == "budget" || f == "bidamount" {
+					if (*tinfo)[f] != nil && util.Float64All((*binfo)[f]) != util.Float64All((*info)[f]) {
+						update[f] = util.Float64All((*binfo)[f])
+						record[f] = util.Float64All((*binfo)[f])
+						updateEs[f] = util.Float64All((*binfo)[f])
+						mdinfo[f] = "数据清洗"
+					}
+				} else if f == "s_winner" {
+					if (*tinfo)[f] != nil && util.Float64All((*binfo)[f]) != util.Float64All((*info)[f]) {
+						update[f] = util.ObjToString((*binfo)[f])
+						record[f] = util.ObjToString((*binfo)[f])
+						updateEs[f] = util.ObjToString((*binfo)[f])
+						cid := companyFun(update)
+						update["entidlist"] = cid
+						record["entidlist"] = cid
+						updateEs["entidlist"] = cid
+					}
+
+				} else {
+					if (*tinfo)[f] != nil && util.ObjToString((*binfo)[f]) != util.ObjToString((*info)[f]) {
+						update[f] = util.ObjToString((*binfo)[f])
+						record[f] = util.ObjToString((*binfo)[f])
+						updateEs[f] = util.ObjToString((*binfo)[f])
+						mdinfo[f] = "数据清洗"
+					}
+				}
+			}
+			for _, f := range config.Conf.Serve.ExField {
+				if f == "package" {
+
+				}
+			}
+			if md, ok := (*info)["modifyinfo"].(map[string]interface{}); ok {
+				for k, v := range mdinfo {
+					md[k] = v
+				}
+				mdinfo = md
+			}
 
+			now := time.Now().Unix()
+			record["updatetime"] = now
+			record["modifypath"] = "数据清洗"
+
+			if len(update) > 0 {
+
+				updateEsPool <- []map[string]interface{}{
+					{"_id": id},
+					updateEs,
+				}
+				update["modifyinfo"] = mdinfo
+				if coll == "bidding" {
+					updatePool <- []map[string]interface{}{
+						{"_id": mongodb.StringTOBsonId(id)},
+						{"$set": update},
+					}
+				} else {
+					Mongo.UpdateById("bidding_back", id, map[string]interface{}{"$set": update})
+				}
+				updateRcPool <- []map[string]interface{}{
+					{"_id": mongodb.StringTOBsonId(id)},
+					{"$set": bson.M{"updatetime": now}, "$push": bson.M{"modify": record}},
+				}
+			}
+
+		}(tmp)
+		tmp = map[string]interface{}{}
+	}
+	wg.Wait()
+	log.Info("taskinfoV2 over...", zap.Int("count:", count))
 }
 
 // @Description entidlist

data_fusion/common.toml

@@ -34,6 +34,7 @@ winnerorder = 1
 addr = "192.168.3.207:27092"
 dbname = "wjh"
 coll = "bidding"
+record = ""
 size = 15
 user = ""
 password = ""

data_fusion/config/conf.go

@@ -62,6 +62,7 @@ type mgo struct {
 	Addr     string
 	Dbname   string
 	Coll     string
+	Record   string
 	Size     int
 	User     string
 	Password string

data_fusion/go.mod

@@ -3,7 +3,8 @@ module data_fusion
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005
 	github.com/BurntSushi/toml v1.2.1
+	go.mongodb.org/mongo-driver v1.12.0 // indirect
 	go.uber.org/zap v1.24.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
 )

data_fusion/go.sum

@@ -1,5 +1,5 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005 h1:AEEi+8ao9pTVqPIh6uVvjxBby/i43fFj7DwVo+feDAE=
-app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
@@ -7,21 +7,55 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
 github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -29,14 +63,21 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -51,13 +92,23 @@ github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
+go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
@@ -69,42 +120,94 @@ go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
 go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -112,9 +215,13 @@ gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1 h1:MbtiS3BwDnB7uRjHFMCBe7/oW+aLaGMeQNiQa3qvd4c=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

data_fusion/init.go

@@ -1,12 +1,12 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"data_fusion/config"
 	"fmt"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"os"
 )
 

data_fusion/main.go

@@ -1,13 +1,14 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/redis"
 	"data_fusion/config"
 	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
 	"reflect"
 	"strings"
 	"sync"
@@ -48,166 +49,194 @@ func main() {
 
 	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("635ff15d631ff1ac3d095c41")}
 	//f := map[string]interface{}{"contenthtml": 0}
-	it := sess.DB(config.Conf.DB.Mongo.Dbname).C("bidding_fusion").Find(nil).Select(nil).Iter()
+	it := sess.DB(config.Conf.DB.Mongo.Dbname).C(config.Conf.DB.Mongo.Coll).Find(nil).Select(nil).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); it.Next(tmp); count++ {
 		if count%20000 == 0 {
 			log.Info("main", zap.Int("current:", count))
 		}
-		if repeat := util.IntAll(tmp["extracttype"]); repeat != -1 {
-			continue
-		}
 		ch <- true
 		wg.Add(1)
 
-		rid := util.ObjToString(tmp["repeat_id"])
-		ArrLock[util.HashCode(rid)%config.Conf.Serve.Thread].Lock()
-
 		go func(tmp map[string]interface{}) {
-			defer func(rid string) {
+			defer func() {
 				<-ch
 				wg.Done()
+			}()
 
-				ArrLock[util.HashCode(rid)%config.Conf.Serve.Thread].Unlock()
-			}(rid)
-			repeatId := util.ObjToString(tmp["repeat_id"])
-			if str := redis.GetStr("fusion_id", repeatId); str != "" {
-				mid := strings.Split(str, "-")[0]
-				tmp1 := findData(mid)
-				w, s := getWeight(tmp)
-				w1, _ := getWeight(tmp1)
-				var update map[string]interface{}
-				var fs []string
-				if w > w1 {
-					update, fs = mergeTmp(tmp, tmp1)
-					if len(update) > 0 {
+			if repeat := util.IntAll(tmp["extracttype"]); repeat != -1 {
+				updatePool <- []map[string]interface{}{
+					{"_id": tmp["_id"]},
+					{"$set": bson.M{"fusion_tag": 1}},
+				}
+			} else {
+				repeatId := util.ObjToString(tmp["repeat_id"])
+				ArrLock[util.HashCode(repeatId)%config.Conf.Serve.Thread].Lock()
+				if str := redis.GetStr("fusion_id", repeatId); str != "" {
+					mid := strings.Split(str, "-")[0]
+					tmp1 := findData(mid)
+					w, s := getWeight(tmp)
+					w1, _ := getWeight(tmp1)
+					var update map[string]interface{}
+					var fs []string
+					if w > w1 {
+						update, fs = mergeTmp(tmp, tmp1)
+						update1 := util.DeepCopy(update).(map[string]interface{})
+						update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
+						update1["weight"] = w
+						update["fusion_tag"] = 1
+						if fs != nil && len(fs) > 0 {
+							updatePool <- []map[string]interface{}{
+								{"_id": tmp["_id"]},
+								{"$set": update, "$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
+							}
+						} else {
+							updatePool <- []map[string]interface{}{
+								{"_id": tmp["_id"]},
+								{"$set": update},
+							}
+						}
 						updatePool <- []map[string]interface{}{
-							{"_id": tmp["_id"]},
-							{"$set": update, "$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
-							//{"$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
+							{"_id": tmp1["_id"]},
+							{"$set": bson.M{"fusion_tag": 0}},
+						}
+						record := make(map[string]interface{})
+						record["$set"] = map[string]interface{}{
+							"template_id":     mongodb.BsonIdToSId(tmp["_id"]),
+							"template_weight": w,
+						}
+						if w == 0 {
+							update1["remark"] = s
+						}
+						record["$push"] = map[string]interface{}{
+							"ids":    mongodb.BsonIdToSId(tmp["_id"]),
+							"record": update1,
+						}
+						recordPool <- []map[string]interface{}{
+							{"_id": mongodb.StringTOBsonId(repeatId)},
+							record,
+						}
+						redis.PutCKV("fusion_id", repeatId, fmt.Sprintf("%s-%s", mongodb.BsonIdToSId(tmp["_id"]), str))
+					} else {
+						update, fs = mergeTmp(tmp1, tmp)
+						update1 := util.DeepCopy(update).(map[string]interface{})
+						update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
+						update1["weight"] = w
+						update["fusion_tag"] = 1
+						if fs != nil && len(fs) > 0 {
+							updatePool <- []map[string]interface{}{
+								{"_id": (tmp1)["_id"]},
+								{"$set": update, "$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
+							}
+						} else {
+							updatePool <- []map[string]interface{}{
+								{"_id": (tmp1)["_id"]},
+								{"$set": update},
+							}
 						}
-					}
-					record := make(map[string]interface{})
-					record["$set"] = map[string]interface{}{
-						"template_id":     mongodb.BsonIdToSId(tmp["_id"]),
-						"template_weight": w,
-					}
-					update1 := util.DeepCopy(update).(map[string]interface{})
-					update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
-					update1["weight"] = w
-					if w == 0 {
-						update1["remark"] = s
-					}
-					record["$push"] = map[string]interface{}{
-						"ids":    mongodb.BsonIdToSId(tmp["_id"]),
-						"record": update1,
-					}
-					recordPool <- []map[string]interface{}{
-						{"_id": mongodb.StringTOBsonId(repeatId)},
-						record,
-					}
-					redis.PutCKV("fusion_id", repeatId, fmt.Sprintf("%s-%s", mongodb.BsonIdToSId(tmp["_id"]), str))
-				} else {
-					update, fs = mergeTmp(tmp1, tmp)
-					if len(update) > 0 {
 						updatePool <- []map[string]interface{}{
-							{"_id": (tmp1)["_id"]},
-							{"$set": update, "$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
-							//{"$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
+							{"_id": (tmp)["_id"]},
+							{"$set": bson.M{"fusion_tag": 0}},
 						}
+						record := make(map[string]interface{})
+						record["$set"] = map[string]interface{}{
+							"template_weight": w1,
+						}
+
+						if w == 0 {
+							update1["remark"] = s
+						}
+						record["$push"] = map[string]interface{}{
+							"ids":    mongodb.BsonIdToSId(tmp["_id"]),
+							"record": update1,
+						}
+						recordPool <- []map[string]interface{}{
+							{"_id": mongodb.StringTOBsonId(repeatId)},
+							record,
+						}
+						redis.PutCKV("fusion_id", mid, fmt.Sprintf("%s-%s", str, mongodb.BsonIdToSId(tmp["_id"])))
 					}
-					record := make(map[string]interface{})
-					record["$set"] = map[string]interface{}{
-						"template_weight": w1,
-					}
-					update1 := util.DeepCopy(update).(map[string]interface{})
-					update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
-					update1["weight"] = w
-					if w == 0 {
-						update1["remark"] = s
-					}
-					record["$push"] = map[string]interface{}{
-						"ids":    mongodb.BsonIdToSId(tmp["_id"]),
-						"record": update1,
-					}
-					recordPool <- []map[string]interface{}{
-						{"_id": mongodb.StringTOBsonId(repeatId)},
-						record,
-					}
-					redis.PutCKV("fusion_id", mid, fmt.Sprintf("%s-%s", str, mongodb.BsonIdToSId(tmp["_id"])))
-				}
-			} else {
-				tmp1 := findData(repeatId)
-				w, s := getWeight(tmp)
-				w1, s1 := getWeight(tmp1)
-				var update map[string]interface{}
-				var fs []string
-				if w > w1 {
-					update, fs = mergeTmp(tmp, tmp1)
-					if len(update) > 0 {
+				} else {
+					tmp1 := findData(repeatId)
+					w, s := getWeight(tmp)
+					w1, s1 := getWeight(tmp1)
+					var update map[string]interface{}
+					var fs []string
+					if w > w1 {
+						update, fs = mergeTmp(tmp, tmp1)
 						set := util.DeepCopy(update).(map[string]interface{})
-						set["fusion_fields"] = fs
+						if fs != nil && len(fs) > 0 {
+							set["fusion_fields"] = fs
+						}
+						set["fusion_tag"] = 1
 						updatePool <- []map[string]interface{}{
 							{"_id": tmp["_id"]},
 							{"$set": set},
-							//{"$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
-							//{"$set": map[string]interface{}{"fusion_fields": fs}},
 						}
-					}
-					record := make(map[string]interface{})
-					record["_id"] = tmp1["_id"]
-					record["template_id"] = mongodb.BsonIdToSId(tmp["_id"])
-					record["template_weight"] = w
-					record["ids"] = []string{mongodb.BsonIdToSId(tmp["_id"]), mongodb.BsonIdToSId(repeatId)}
-					var recordlist []map[string]interface{}
-					recordlist = append(recordlist, map[string]interface{}{"infoid": mongodb.BsonIdToSId(tmp["_id"]), "weight": w})
-					update1 := util.DeepCopy(update).(map[string]interface{})
-					update1["infoid"] = mongodb.BsonIdToSId(tmp1["_id"])
-					update1["weight"] = w1
-					if w1 == 0 {
-						update1["remark"] = s1
-					}
-					recordlist = append(recordlist, update1)
-					record["record"] = recordlist
-					recordPool <- []map[string]interface{}{
-						{"_id": tmp1["_id"]},
-						{"$set": record},
-					}
-					redis.PutCKV("fusion_id", repeatId, fmt.Sprintf("%s-%s", mongodb.BsonIdToSId(tmp["_id"]), mongodb.BsonIdToSId(tmp1["_id"])))
-				} else {
-					update, fs = mergeTmp(tmp1, tmp)
-					if len(update) > 0 {
+						updatePool <- []map[string]interface{}{
+							{"_id": tmp1["_id"]},
+							{"$set": bson.M{"fusion_tag": 0}},
+						}
+						record := make(map[string]interface{})
+						record["_id"] = tmp1["_id"]
+						record["template_id"] = mongodb.BsonIdToSId(tmp["_id"])
+						record["template_weight"] = w
+						record["ids"] = []string{mongodb.BsonIdToSId(tmp["_id"]), mongodb.BsonIdToSId(repeatId)}
+						var recordlist []map[string]interface{}
+						recordlist = append(recordlist, map[string]interface{}{"infoid": mongodb.BsonIdToSId(tmp["_id"]), "weight": w})
+						update1 := util.DeepCopy(update).(map[string]interface{})
+						update1["infoid"] = mongodb.BsonIdToSId(tmp1["_id"])
+						update1["weight"] = w1
+						if w1 == 0 {
+							update1["remark"] = s1
+						}
+						recordlist = append(recordlist, update1)
+						record["record"] = recordlist
+						recordPool <- []map[string]interface{}{
+							{"_id": tmp1["_id"]},
+							{"$set": record},
+						}
+						redis.PutCKV("fusion_id", repeatId, fmt.Sprintf("%s-%s", mongodb.BsonIdToSId(tmp["_id"]), mongodb.BsonIdToSId(tmp1["_id"])))
+					} else {
+						update, fs = mergeTmp(tmp1, tmp)
 						set := util.DeepCopy(update).(map[string]interface{})
-						set["fusion_fields"] = fs
+						if fs != nil && len(fs) > 0 {
+							set["fusion_fields"] = fs
+						}
+						set["fusion_tag"] = 1
 						updatePool <- []map[string]interface{}{
 							{"_id": tmp1["_id"]},
 							{"$set": set},
-							//{"$addToSet": map[string]interface{}{"fusion_fields": map[string]interface{}{"$each": fs}}},
-							//{"$set": map[string]interface{}{"fusion_fields": fs}},
 						}
+						updatePool <- []map[string]interface{}{
+							{"_id": tmp["_id"]},
+							{"$set": bson.M{"fusion_tag": 0}},
+						}
+						record := make(map[string]interface{})
+						record["_id"] = tmp1["_id"]
+						record["template_id"] = mongodb.BsonIdToSId(tmp1["_id"])
+						record["template_weight"] = w1
+						record["ids"] = []string{mongodb.BsonIdToSId(tmp["_id"]), mongodb.BsonIdToSId(tmp1["_id"])}
+						var recordlist []map[string]interface{}
+						recordlist = append(recordlist, map[string]interface{}{"infoid": mongodb.BsonIdToSId(tmp1["_id"]), "weight": w1})
+						update1 := util.DeepCopy(update).(map[string]interface{})
+						update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
+						update1["weight"] = w
+						if w == 0 {
+							update1["remark"] = s
+						}
+						recordlist = append(recordlist, update1)
+						record["record"] = recordlist
+						recordPool <- []map[string]interface{}{
+							{"_id": tmp1["_id"]},
+							{"$set": record},
+						}
+						redis.PutCKV("fusion_id", repeatId, fmt.Sprintf("%s-%s", mongodb.BsonIdToSId(tmp1["_id"]), mongodb.BsonIdToSId(tmp["_id"])))
 					}
-					record := make(map[string]interface{})
-					record["_id"] = tmp1["_id"]
-					record["template_id"] = mongodb.BsonIdToSId(tmp1["_id"])
-					record["template_weight"] = w1
-					record["ids"] = []string{mongodb.BsonIdToSId(tmp["_id"]), mongodb.BsonIdToSId(tmp1["_id"])}
-					var recordlist []map[string]interface{}
-					recordlist = append(recordlist, map[string]interface{}{"infoid": mongodb.BsonIdToSId(tmp1["_id"]), "weight": w1})
-					update1 := util.DeepCopy(update).(map[string]interface{})
-					update1["infoid"] = mongodb.BsonIdToSId(tmp["_id"])
-					update1["weight"] = w
-					if w == 0 {
-						update1["remark"] = s
-					}
-					recordlist = append(recordlist, update1)
-					record["record"] = recordlist
-					recordPool <- []map[string]interface{}{
-						{"_id": tmp1["_id"]},
-						{"$set": record},
-					}
-					redis.PutCKV("fusion_id", repeatId, fmt.Sprintf("%s-%s", mongodb.BsonIdToSId(tmp1["_id"]), mongodb.BsonIdToSId(tmp["_id"])))
 				}
+				ArrLock[util.HashCode(repeatId)%config.Conf.Serve.Thread].Unlock()
 			}
+
 		}(tmp)
 		tmp = map[string]interface{}{}
 	}
@@ -219,7 +248,7 @@ func main() {
 }
 
 func findData(id string) map[string]interface{} {
-	tmp, _ := MgoB.FindById("bidding_fusion", id, nil)
+	tmp, _ := MgoB.FindById(config.Conf.DB.Mongo.Coll, id, nil)
 	if tmp != nil && len(*tmp) > 0 {
 		return *tmp
 	} else {
@@ -334,7 +363,7 @@ func updateMethod() {
 					defer func() {
 						<-recordSp
 					}()
-					MgoB.UpSertBulk("bidding_fusion_record", arru...)
+					MgoB.UpSertBulk(config.Conf.DB.Mongo.Record, arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 500)
 				indexu = 0
@@ -346,7 +375,7 @@ func updateMethod() {
 					defer func() {
 						<-recordSp
 					}()
-					MgoB.UpSertBulk("bidding_fusion_record", arru...)
+					MgoB.UpSertBulk(config.Conf.DB.Mongo.Record, arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 500)
 				indexu = 0
@@ -369,7 +398,7 @@ func updateMethod1() {
 					defer func() {
 						<-updateSp
 					}()
-					MgoB.UpdateBulk("bidding_fusion", arru...)
+					MgoB.UpdateBulk(config.Conf.DB.Mongo.Coll, arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 500)
 				indexu = 0
@@ -381,7 +410,7 @@ func updateMethod1() {
 					defer func() {
 						<-updateSp
 					}()
-					MgoB.UpdateBulk("bidding_fusion", arru...)
+					MgoB.UpdateBulk(config.Conf.DB.Mongo.Coll, arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 500)
 				indexu = 0

data_project/common.toml

@@ -11,14 +11,14 @@ siteColl = "site"
 [db.mongoB]
 addr = "192.168.3.71:29099"
 dbname = "wjh"
-coll = "bidding"
+coll = "wjh_bidding_0614_1"
 size = 15
 user = ""
 password = ""
 [db.mongoP]
 addr = "192.168.3.71:29099"
 dbname = "wjh"
-coll = "bidding"
+coll = "projectset"
 size = 15
 user = ""
 password = ""

data_project/config.json

@@ -46,5 +46,21 @@
         "back_rep_regexp": ["(有限公$)#有限公司", "(有限公同$)#有限公司", "(有限责任公$)#有限责任公司", "(公司公司&)#公司", "(公司等&)#公司", "([a-z]限公司$)#有限公司", "(有公司$)#有限公司", "(招标代理有$)#招标代理有限公司"],
         "blacklist": ["承包项目", "（二标段", "举行了", "提交一份公司", "驱动更新", "邱海丽", "集中代理采购", "公司组织的", "利用项目", "地址:", "交易网", "年龄:", "性别:", "竞争性谈判", "null", "停车场", "招标人", "制造项目", "(中|招)标公告",
             "“组组通”工程", "备案单位（章）", "小时", "梁工"]
-    }
+    },
+    "site_list": ["a_ahwgjsxmglyxgs_kbgs", "ah_aqsggzyjyfww_jrkb", "am_amtbxzqzfwhj_gkzb_2021/1/1hzhkbjg", "am_amtbxzqzfwhj_qtcgxm_2021/1/1hzhkb/kqbjjg",
+        "a_cdswyglxh_kbxx","ah_czsggzyjyw_jsgc_bqgs", "ah_czsggzyjyw_jsgc_kbjl","ah_czsggzyjyw_zfcg_kbxx", "nm_cfggzyjyw_jsgc_kbjggs",
+        "fj_fzsfqsjsgcdzztbjypt_zbgc_kbjl", "fj_fzsjsgcdzztbjypt_zbgc_kbjl", "fj_fzsljxjsgcdzztbjyw_zbgc_kbjl", "fj_fzslyxjsgcdzztbjyw_zbgc_kbjl",
+        "fj_fzsmwqjsgcdzztbjypt_zbgc_kbjl","fj_fzsmwqjsgcdzztbjyw_zbgc_kbjl","fj_fzsmhxjsgcdzztbjypt_zbgc_kbjl","fj_fzsmhxjsgcdzztbjyw_zbgc_kbjl",
+        "fj_fzsmqxjsgcdzztbjyw_zbgc_kbjl","fj_fzsclqjsgcdzztbjypt_zbgc_kbjl","a_gttgzbcgpt_kbgg","a_gjggzyjypt_gcjs_kbjl","a_gjggzyjypt_gcjs_kbjl_new",
+        "a_gjggzyjypt_gcjs_kbjl_bu","zj_hzsggzyjy_gcjs_kbjggs","a_hbszbtbggfwpt_kbjl","a_hbszbtbggfwpt_kbjl_bu","hb_hbsggzyjydzfwxt_gcjs_kbjl",
+        "zj_hzsggzyjyzx_gcjs_xe","ah_hbsggzyjyw_zfcg_zbgs_sbj","ah_hbsggzyjyw_jsgc_bggg_sbj","hb_hgshaxggzyjyzx_xxzx_kbrcap","hb_hssggzyjyw_gcjs_kbqk",
+        "a_jsjsgczbw_kbqkgs","zj_jhsywsrmzf_gcztbxx_kbjggg","hb_jmsdbqzfcgdzjypt_kbxx","hb_jmsggzyjyjdglj_bzkb","zj_lssggzyjyw_jsgc_kbsk",
+        "a_lygttzjttgyxgs_kbgg","ha_lhsggzyjyxxw_jsgc_kbqk","sc_mssggzydzjypt_jsgc_kbjl","sc_njsggzyjyxxw_jsgc_dy","nm_nmgzzqggzyjyw_gcjs_kbjl",
+        "sc_pzhsggzyjyfwzx_jsgc_kbylb","sc_msspsqggzyjyw_jsgc_kbjl","sc_mssqsxggzyjyw_jsgc_kbjl","hn_sysjsgczbtbw_zbdlbxwj","gd_szsggzyjyzx_jsgc_kbqkgs",
+        "gd_szsggzyjyzx_ygwy_kbqkgs","a_szsjsgcjyfwzxbafzx_kbqkgs","a_szsnsqfybjy_zbtzgg_kbrq","a_szsqhjstzkgjtyxgscgxxpt_kbgs","a_szsyfzbyxgs_kbxx",
+        "gd_szszfhjsj_kbqkgs","js_szgyyqggzyjyzx_jsgc_zjfbgs","sc_snsggzyjyw_jsgc_jggg","zj_tzsyhggzyjyzx_jsgc_kbqk","zj_tzsyhggzyjyzx_kbjggg",
+        "hb_tmsggzyjyxxw_jsgc_kbqk","ah_whsggzyjyfww_kbxx_cgxm","ah_whsggzyjyfww_kbxx_gcxm","ah_whsggzyjyfww_zfcg_kbxx","a_whsdxhqzfcgdzjyxt_kbxx",
+        "hb_whsggzyjypt_gckbap","hb_whshsqzfcgdzjyxt_kbxx","hb_whsjsgczbtbjypt_kbjl","sd_ytszfcgjdglpt_kbzx","a_ycw_kbjl","zj_zjsggzyjyfwpt_gcxm_kbjggs",
+        "zj_zjsggzyjyzx_jyxx_kbjg","zj_zjszsqdxqxcglwyh_kbjggs","a_zgxajtdzzbcgjypt_zb_kbjl","a_zgxajtdzzbcgjypt_fzb_kbjl","a_zgzbtbggfwpt_wasjgf_kbjl",
+        "zj_zssggzyjyw_gcjs_yccljggg"]
 }

data_project/go.mod

@@ -3,10 +3,10 @@ module project
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20230519053026-75421af90e41
 	github.com/BurntSushi/toml v1.2.0
 	github.com/goinggo/mapstructure v0.0.0-20140717182941-194205d9b4a9
 	github.com/robfig/cron v1.2.0
 	go.mongodb.org/mongo-driver v1.11.6
-	go.uber.org/zap v1.24.0 // indirect
+	go.uber.org/zap v1.24.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
 )

data_project/go.sum

@@ -1,5 +1,3 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20230519053026-75421af90e41 h1:MRE2bNcC4rzlzkMozqeAqyMazU/t8EHwH0vp/6W49JE=
-app.yhyue.com/data_processing/common_utils v0.0.0-20230519053026-75421af90e41/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
@@ -92,7 +90,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
@@ -120,7 +117,6 @@ go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
 go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
 go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
@@ -216,3 +212,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1 h1:MbtiS3BwDnB7uRjHFMCBe7/oW+aLaGMeQNiQa3qvd4c=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

data_project/init.go

@@ -1,13 +1,13 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/redis"
-	"app.yhyue.com/data_processing/common_utils/udp"
 	"fmt"
 	"go.mongodb.org/mongo-driver/bson/primitive"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"math"
 	"os"
 	"project/config"
@@ -23,6 +23,7 @@ var (
 	BiddingColl, ProjectColl, BackupColl, SiteColl string              //抽取表、项目表、项目快照表、站点表
 	Thread                                         int                 //配置项线程数
 	BlackList                                      []interface{}
+	SkipSiteList                                   []string
 	BlaskListMap                                   map[string]bool
 
 	RedisCode string
@@ -64,6 +65,8 @@ func init() {
 		MongodbAddr: config.Conf.DB.MongoP.Addr,
 		Size:        config.Conf.DB.MongoP.Size,
 		DbName:      config.Conf.DB.MongoP.Dbname,
+		UserName:    config.Conf.DB.MongoP.User,
+		Password:    config.Conf.DB.MongoP.Password,
 	}
 	MgoP.InitPool()
 	MgoB = &mongodb.MongodbSim{
@@ -78,6 +81,8 @@ func init() {
 		MongodbAddr: config.Conf.DB.MongoS.Addr,
 		Size:        config.Conf.DB.MongoS.Size,
 		DbName:      config.Conf.DB.MongoS.Dbname,
+		UserName:    config.Conf.DB.MongoS.User,
+		Password:    config.Conf.DB.MongoS.Password,
 	}
 	MgoS.InitPool()
 
@@ -102,6 +107,7 @@ func init() {
 	for _, v := range BlackList {
 		BlaskListMap[util.ObjToString(v)] = true
 	}
+	SkipSiteList = util.ObjArrToStringArr(cof["site_list"].([]interface{}))
 	initWinnerRegexp(cof)
 	initBuyerRegexp(cof)
 	initAgencyRegexp(cof)

data_project/load_data.go

@@ -1,11 +1,11 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/redis"
 	"encoding/json"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
 	"strings"
 	"sync"
 	"time"

data_project/main.go

@@ -1,15 +1,14 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/elastic"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/nsq"
-	"app.yhyue.com/data_processing/common_utils/udp"
 	"encoding/json"
-	"fmt"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	gonsq "jygit.jydev.jianyu360.cn/data_processing/common_utils/nsq"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"net"
 	"os"
 	"os/signal"
@@ -95,7 +94,6 @@ func main() {
 		if !ok {
 			continue
 		}
-		SingleThread <- true
 		tasktype := util.ObjToString(mapinfo["stype"])
 		switch tasktype {
 		case "ql": //全量合并
@@ -103,6 +101,7 @@ func main() {
 				defer func() {
 					<-SingleThread
 				}()
+				SingleThread <- true
 				P_QL.currentType = tasktype
 				P_QL.pici = time.Now().Unix()
 				P_QL.taskQl(mapinfo)
@@ -112,6 +111,7 @@ func main() {
 				defer func() {
 					<-SingleThread
 				}()
+				SingleThread <- true
 				P_QL.currentType = tasktype
 				P_QL.pici = time.Now().Unix()
 				P_QL.taskZl(mapinfo)
@@ -121,6 +121,7 @@ func main() {
 				defer func() {
 					<-SingleThread
 				}()
+				SingleThread <- true
 				P_QL.currentType = tasktype
 				P_QL.pici = time.Now().Unix()
 				P_QL.taskZl(mapinfo)
@@ -128,7 +129,6 @@ func main() {
 		case "updateInfo": //招标字段变更
 			go func() {
 				defer func() {
-					<-SingleThread
 				}()
 				P_QL.currentType = tasktype
 				P_QL.pici = time.Now().Unix()
@@ -137,7 +137,6 @@ func main() {
 		case "updatePro": //修改项目外围字段(只修改外围字段值)
 			go func() {
 				defer func() {
-					<-SingleThread
 				}()
 				P_QL.currentType = tasktype
 				P_QL.pici = time.Now().Unix()
@@ -146,7 +145,6 @@ func main() {
 		case "deleteInfo": // 删除招标公告
 			go func() {
 				defer func() {
-					<-SingleThread
 				}()
 				P_QL.currentType = tasktype
 				P_QL.pici = time.Now().Unix()
@@ -155,14 +153,12 @@ func main() {
 		case "spider": // 爬虫代码code、isflow
 			go func() {
 				defer func() {
-					<-SingleThread
 				}()
 				go P_QL.loadSpiderCode()
 			}()
 		case "history": //历史数据合并,暂时不写
 			go func() {
 				defer func() {
-					<-SingleThread
 				}()
 			}()
 		default:
@@ -172,8 +168,8 @@ func main() {
 }
 
 func mainT() {
-	sid = "623d1ed4923488e17244bea7"
-	eid = "6327653467a6b0a286122df9"
+	sid = "56614b76af5374118a002100"
+	eid = "64884e4d8aea8786d11c99f9"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()
@@ -185,7 +181,7 @@ func mainT() {
 	}
 	mapinfo["gtid"] = sid
 	mapinfo["lteid"] = eid
-	mapinfo["stype"] = "ql"
+	mapinfo["stype"] = "project"
 	mapinfo["ip"] = "127.0.0.1"
 	mapinfo["port"] = "1782"
 
@@ -201,7 +197,7 @@ func mainT() {
 	time.Sleep(99999 * time.Hour)
 }
 
-//udp调用信号
+// udp调用信号
 func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 	switch act {
 	case udp.OP_TYPE_DATA: //上个节点的数据
@@ -211,12 +207,15 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 		if err != nil {
 			_ = udpclient.WriteUdp([]byte("err:"+err.Error()), udp.OP_NOOP, ra)
 		} else if mapInfo != nil {
+
 			key, _ := mapInfo["key"].(string)
 			if key == "" {
 				key = "udpok"
 			}
 			go udpclient.WriteUdp([]byte(key), udp.OP_NOOP, ra)
-			UdpChan <- mapInfo
+			if util.ObjToString(mapInfo["stype"]) != "monitor" {
+				UdpChan <- mapInfo
+			}
 		}
 	case udp.OP_NOOP: //下个节点回应
 		ok := string(data)
@@ -245,7 +244,6 @@ func (p *ProjectTask) nsqMethod() {
 	for {
 		select {
 		case obj := <-Mcmer.Ch: //从通道读取即可
-			log.Info("project nsq: " + fmt.Sprint(obj))
 			id := strings.Split(util.ObjToString(obj), "=")
 			if mongodb.IsObjectIdHex(id[1]) {
 				p.taskinfo(id[1])

data_project/project.go

@@ -1,11 +1,11 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/redis"
 	"encoding/json"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
 	"math"
 	"sort"
 	"strings"
@@ -703,7 +703,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	var project *Project
 	err := json.Unmarshal(bys, &project)
 	if err != nil {
-		log.Info("project Unmarshal err，" + err.Error())
+		log.Info("project Unmarshal err，" + err.Error() + "，pid: " + pInfo.Id.Hex())
 		return
 	}
 	set := map[string]interface{}{}

data_project/task.go

@@ -1,17 +1,17 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/redis"
-	"app.yhyue.com/data_processing/common_utils/udp"
 	"encoding/json"
 	"fmt"
 	"github.com/goinggo/mapstructure"
 	"github.com/robfig/cron"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"project/config"
 	"regexp"
 	"strings"
@@ -27,9 +27,9 @@ import (
 转换成info对象
 **/
 
-//var PreRegexp = map[string][]*regexp.Regexp{}
-//var BackRegexp = map[string][]*regexp.Regexp{}
-//var BackRepRegexp = map[string][]RegexpInfo{}
+// var PreRegexp = map[string][]*regexp.Regexp{}
+// var BackRegexp = map[string][]*regexp.Regexp{}
+// var BackRepRegexp = map[string][]RegexpInfo{}
 var BlackRegexp = map[string][]*regexp.Regexp{}
 
 var (
@@ -52,7 +52,7 @@ type RegexpInfo struct {
 	repstr string
 }
 
-//项目合并对象
+// 项目合并对象
 type ProjectTask struct {
 	InitMinTime int64 //最小时间，小于0的处理一次
 	name        string
@@ -164,7 +164,7 @@ func (p *ProjectTask) updateAllQueue() {
 	}
 }
 
-//项目合并内存更新
+// 项目合并内存更新
 func (p *ProjectTask) clearMem() {
 	c := cron.New()
 	// 创建项目的时间大于7天
@@ -255,7 +255,7 @@ func (p *ProjectTask) clearMem() {
 	c.Start()
 }
 
-//全量合并
+// 全量合并
 func (p *ProjectTask) taskQl(udpInfo map[string]interface{}) {
 	defer util.Catch()
 	p.thread = util.IntAllDef(Thread, 4)
@@ -283,14 +283,14 @@ func (p *ProjectTask) taskQl(udpInfo map[string]interface{}) {
 	if util.ObjToString(udpInfo["coll"]) != "" {
 		BiddingColl = "bidding_back"
 	} else {
-		BiddingColl = "bidding"
+		BiddingColl = "wjh_bidding_0614_1"
 	}
 	//生成查询语句执行
 	p.enter(MgoB.DbName, BiddingColl, q)
 
 }
 
-//增量合并
+// 增量合并
 func (p *ProjectTask) taskZl(udpInfo map[string]interface{}) {
 	defer util.Catch()
 	//1、检查pubilshtime索引
@@ -330,7 +330,7 @@ func (p *ProjectTask) taskZl(udpInfo map[string]interface{}) {
 	nextNode(udpInfo, p.pici)
 }
 
-//招标字段更新
+// 招标字段更新
 func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
 	defer util.Catch()
 	infoid := udpInfo["infoid"].(string)
@@ -422,7 +422,7 @@ func (p *ProjectTask) delInfoPro(udpInfo map[string]interface{}) {
 	}
 }
 
-//通知下个节点nextNode
+// 通知下个节点nextNode
 func nextNode(mapInfo map[string]interface{}, pici int64) {
 	mapInfo["stype"] = "project"
 	mapInfo["query"] = map[string]interface{}{
@@ -493,7 +493,7 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 		c, _ := sess.DB(db).C(coll).Find(q).Count()
 		log.Info(fmt.Sprintf("共查询: %d条", c))
 	}
-	ms := sess.DB(db).C(coll).Find(q).Select(fields).Sort("publishtime")
+	ms := sess.DB(db).C(coll).Find(q).Select(fields)
 	query := ms.Iter()
 	var lastid interface{}
 L:
@@ -517,26 +517,31 @@ L:
 						log.Info("current---", zap.Int("count", count), zap.String("lastid", mongodb.BsonIdToSId(lastid)))
 					}
 				}
-				//extracttype -1: 重复，1: 不重复
-				if util.IntAll(tmp["extracttype"]) == 1 {
-					if util.ObjToString(tmp["toptype"]) != "采购意向" && util.ObjToString(tmp["toptype"]) != "产权" {
-						if P_QL.currentType == "ql" {
-							infoPool <- tmp
-						} else if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
-							// id段增量数据
-							infoPool <- tmp
-						} else if P_QL.currentType == "project_history" && tmp["history_updatetime"] != nil {
-							// id段 历史数据
-							infoPool <- tmp
+				// 判重过滤 中标记录
+				if !siteJudge(util.ObjToString(tmp["spidercode"])) {
+					//extracttype -1: 重复，1: 不重复
+					if util.IntAll(tmp["extracttype"]) == 1 {
+						if util.ObjToString(tmp["toptype"]) != "采购意向" && util.ObjToString(tmp["toptype"]) != "产权" {
+
+							if P_QL.currentType == "ql" {
+								infoPool <- tmp
+							} else if P_QL.currentType == "project" && util.IntAll(tmp["dataging"]) == 0 {
+								// id段增量数据
+								infoPool <- tmp
+							} else if P_QL.currentType == "project_history" && tmp["history_updatetime"] != nil {
+								// id段 历史数据
+								infoPool <- tmp
+							}
 						}
+					} else {
+						countRepeat++
+						//if P_QL.currentType == "project" {
+						//	log.Info("repeat err---", tmp["_id"])
+						//}
 					}
-				} else {
-					countRepeat++
-					//if P_QL.currentType == "project" {
-					//	log.Info("repeat err---", tmp["_id"])
-					//}
+					count++
 				}
-				count++
+
 			} else {
 				break L
 			}
@@ -819,7 +824,8 @@ var Elements = []string{
 	"subtype",
 }
 
-/**
+/*
+*
 修改的字段
 修改的字段是否是影响合并流程的要素字段
 */
@@ -834,7 +840,7 @@ func modifyEle(tmp map[string]interface{}) bool {
 	return merge
 }
 
-//补全位置信息
+// 补全位置信息
 func (p *ProjectTask) fillInPlace(tmp map[string]interface{}) {
 	area := util.ObjToString(tmp["area"])
 	city := util.ObjToString(tmp["city"])
@@ -875,7 +881,7 @@ func (p *ProjectTask) fillInPlace(tmp map[string]interface{}) {
 	}
 }
 
-//从数组中删除元素
+// 从数组中删除元素
 func deleteSlice(arr []string, v, stype string) []string {
 	for k, v1 := range arr {
 		if v1 == v {
@@ -886,7 +892,7 @@ func deleteSlice(arr []string, v, stype string) []string {
 	return arr
 }
 
-//校验评审专家
+// 校验评审专家
 func ClearRp(tmp []string) []string {
 	arrTmp := []string{}
 	for _, v := range tmp {
@@ -935,3 +941,14 @@ func QyFilter(name, stype string) string {
 	}
 	return name
 }
+
+// @Description 过滤中标记录数据
+// @Author J 2023/6/12 15:11
+func siteJudge(code string) bool {
+	for _, s := range SkipSiteList {
+		if code == s {
+			return true
+		}
+	}
+	return false
+}

data_project/udptaskmap.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	"app.yhyue.com/data_processing/common_utils/udp"
 	"fmt"
 	"io/ioutil"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"log"
 	"net"
 	"net/http"

data_project/update.go

@@ -1,17 +1,17 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/redis"
-	"app.yhyue.com/data_processing/common_utils/udp"
 	"encoding/json"
 	"fmt"
 	"github.com/goinggo/mapstructure"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"math"
 	"project/config"
 	"reflect"
@@ -20,7 +20,8 @@ import (
 	"time"
 )
 
-/**
+/*
+*
 直接修改保存项目字段信息
 */
 func (p *ProjectTask) modifyUpdate(pid string, index, position int, tmpPro, modifyProMap map[string]interface{}) {
@@ -201,6 +202,9 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index, position int, tmp ma
 // 删除
 func (p *ProjectTask) delJudge(infoid, pid string) {
 	tmpPro, _ := MgoP.FindById(ProjectColl, pid, nil)
+	if tmpPro != nil || len(*tmpPro) == 0 {
+		return
+	}
 	ids := (*tmpPro)["ids"].([]interface{})
 	proList := (*tmpPro)["list"].([]interface{})
 	if len(ids) == 1 {
@@ -454,7 +458,7 @@ func (p *ProjectTask) ReMerge(info *Info, tmp map[string]interface{}, tmpPro map
 	return true
 }
 
-//内部合并
+// 内部合并
 func (p *ProjectTask) innerMerge(infoList []interface{}, tmpPro map[string]interface{}) (map[string]interface{}, *ProjectCache) {
 	newP := make(map[string]interface{})
 	newP["_id"] = tmpPro["_id"]
@@ -1171,7 +1175,7 @@ func deleteSlice1(arr []interface{}, v interface{}) []interface{} {
 	return arr
 }
 
-//	修改字段值
+// 修改字段值
 func UpdateValue(proMap map[string]interface{}, index, position int, modifyMap map[string]interface{}) map[string]interface{} {
 	updataSet := make(map[string]interface{})
 	infoList := proMap["list"].([]interface{})
@@ -1212,7 +1216,7 @@ func UpdateValue(proMap map[string]interface{}, index, position int, modifyMap m
 	return updataSet
 }
 
-//备份(快照)
+// 备份(快照)
 func backupPro(tmp map[string]interface{}) {
 	tmp1 := make(map[string]interface{})
 	for k, v := range tmp {

data_tidb/bidding.go

@@ -1,12 +1,12 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"data_tidb/config"
 	"fmt"
 	"github.com/shopspring/decimal"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"reflect"
 	"regexp"
 	"sort"

data_tidb/go.mod

@@ -3,10 +3,10 @@ module data_tidb
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d
 	github.com/BurntSushi/toml v1.2.0
 	github.com/shopspring/decimal v1.3.1
 	github.com/spf13/cobra v1.5.0
 	go.mongodb.org/mongo-driver v1.10.3
 	go.uber.org/zap v1.23.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061 // indirect
 )

data_tidb/go.sum

@@ -1,30 +1,66 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
-app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
 github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -32,17 +68,24 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
 github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -69,6 +112,9 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.mongodb.org/mongo-driver v1.10.3 h1:XDQEvmh6z1EUsXuIkXE9TaVeqHw6SwS1uf93jFs0HBA=
 go.mongodb.org/mongo-driver v1.10.3/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
@@ -80,35 +126,56 @@ go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -116,6 +183,25 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -124,8 +210,13 @@ gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3M
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061 h1:xfmoVsDEqVv1XzAVxgGEDi+W9ojHBJc6OmTbg3b1tP0=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

data_tidb/init.go

@@ -1,13 +1,13 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/mysqldb"
 	"data_tidb/config"
 	"fmt"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
 	"os"
 )
 

data_tidb/main.go

@@ -1,17 +1,17 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/redis"
-	"app.yhyue.com/data_processing/common_utils/udp"
 	"data_tidb/config"
 	"encoding/json"
 	"fmt"
 	"github.com/spf13/cobra"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"net"
 	"sync"
 	"time"
@@ -28,7 +28,7 @@ func init() {
 	InitMysql()
 	InitField()
 
-	//redis.InitRedis1("qyxy_id=127.0.0.1:4379", 1)
+	redis.InitRedis1("qyxy_id=127.0.0.1:8379", 1)
 	//redis.InitRedis1("qyxy_id=192.168.3.166:4379", 1)
 	log.Info("init success")
 }
@@ -39,12 +39,14 @@ func main() {
 	//go SaveTagFunc()
 	//go saveErrMethod()
 
-	rootCmd := &cobra.Command{Use: "my cmd"}
-	rootCmd.AddCommand(bidding())
-	rootCmd.AddCommand(project())
-	if err := rootCmd.Execute(); err != nil {
-		fmt.Println("rootCmd.Execute failed", err.Error())
-	}
+	//rootCmd := &cobra.Command{Use: "my cmd"}
+	//rootCmd.AddCommand(bidding())
+	//rootCmd.AddCommand(project())
+	//if err := rootCmd.Execute(); err != nil {
+	//	fmt.Println("rootCmd.Execute failed", err.Error())
+	//}
+
+	taskMysql()
 
 	//UdpClient = udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
 	//UdpClient.Listen(processUdpMsg)
@@ -84,17 +86,17 @@ func taskMysql() {
 	pool := make(chan bool, 5) //控制线程数
 	wg := &sync.WaitGroup{}
 
-	finalId := 0
+	finalId := 6500183
 	//lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT * FROM %s ORDER BY id DESC LIMIT 1", "dws_f_ent_baseinfo"))
-	lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT id, projectid, infoid, name_id, identity_type+0 FROM %s ORDER BY id DESC LIMIT 1", "dws_f_bpmc_relation"))
-	if len(*lastInfo) > 0 {
-		finalId = util.IntAll((*lastInfo)[0]["id"])
-	}
+	//lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT id, projectid, infoid, name_id, identity_type+0 FROM %s ORDER BY id DESC LIMIT 1", "dws_f_bpmc_relation_new"))
+	//if len(*lastInfo) > 0 {
+	//	finalId = util.IntAll((*lastInfo)[0]["id"])
+	//}
 	log.Info("查询最后id---", zap.Int("finally id: ", finalId))
 	lastid, count := 0, 0
 	for {
 		log.Info("重新查询，lastid---", zap.Int("lastid: ", lastid))
-		q := fmt.Sprintf("SELECT id, projectid, infoid, name_id, identity_type+0 FROM %s WHERE id > %d ORDER BY id ASC limit 1000000", "dws_f_bpmc_relation", lastid)
+		q := fmt.Sprintf("SELECT id, projectid, infoid, name_id, identity_type+0 FROM %s WHERE id > %d ORDER BY id ASC limit 1000000", "dws_f_bpmc_relation_new", lastid)
 		//q := fmt.Sprintf("SELECT id, projectid, infoid, name_id, identity_type+0 FROM %s WHERE id=61771536 ORDER BY id ASC limit 1000000", "dws_f_bpmc_relation")
 		//q := fmt.Sprintf("SELECT id, name, name_id FROM %s WHERE id>%d ORDER BY id ASC limit 1000000", "dws_f_ent_baseinfo", lastid)
 		rows, err := MysqlTool.DB.Query(q)
@@ -163,52 +165,58 @@ func taskMysql() {
 					wg.Done()
 				}()
 				cid := util.Int64All(tmp["id"])
-				pid := util.ObjToString(tmp["projectid"])
+				iid := util.ObjToString(tmp["infoid"])
 				name_id := util.ObjToString(tmp["name_id"])
 				identity_type := util.Int64All(tmp["identity_type+0"])
 				if name_id != "" {
-					pinfo, _ := MongoP.FindById("projectset", pid, bson.M{"ids": 1})
-					if len(*pinfo) > 0 {
-						for _, id := range util.ObjArrToStringArr((*pinfo)["ids"].([]interface{})) {
-							coll := "bidding"
-							//if id > "5a862e7040d2d9bbe88e3b1f" {
-							//	coll = "bidding"
-							//} else {
-							//	coll = "bidding_back"
-							//}
-							info, _ := MongoB.FindById(coll, id, bson.M{"agencytel": 1, "agencyperson": 1, "buyertel": 1, "buyerperson": 1, "winnertel": 1, "winnerperson": 1})
-							if len(*info) > 0 {
-								if identity_type == 1 {
-									if util.ObjToString((*info)["buyertel"]) != "" && util.ObjToString((*info)["buyerperson"]) != "" {
-										q := map[string]interface{}{"name_id": name_id, "identity_type": identity_type, "contact_name": util.ObjToString((*info)["buyerperson"]), "contact_tel": util.ObjToString((*info)["buyertel"])}
-										cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
-										if cinfo != nil && len(*cinfo) > 0 {
-											MysqlTool.Update("dws_f_bpmc_relation", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
-											break
-										}
-									}
-								} else if identity_type == 2 {
-									if util.ObjToString((*info)["winnertel"]) != "" && util.ObjToString((*info)["winnerperson"]) != "" {
-										if util.ObjToString((*info)["winnertel"]) != "" && util.ObjToString((*info)["winnerperson"]) != "" {
-											q := map[string]interface{}{"name_id": name_id, "identity_type": identity_type, "contact_name": util.ObjToString((*info)["winnerperson"]), "contact_tel": util.ObjToString((*info)["winnertel"])}
-											cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
-											if cinfo != nil && len(*cinfo) > 0 {
-												MysqlTool.Update("dws_f_bpmc_relation", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
-												break
-											}
-										}
-									}
-								} else if identity_type == 4 {
-									if util.ObjToString((*info)["agencytel"]) != "" && util.ObjToString((*info)["agencyperson"]) != "" {
-										if util.ObjToString((*info)["agencytel"]) != "" && util.ObjToString((*info)["agencyperson"]) != "" {
-											q := map[string]interface{}{"name_id": name_id, "identity_type": identity_type, "contact_name": util.ObjToString((*info)["agencyperson"]), "contact_tel": util.ObjToString((*info)["agencytel"])}
-											cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
-											if cinfo != nil && len(*cinfo) > 0 {
-												MysqlTool.Update("dws_f_bpmc_relation", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
-												break
-											}
-										}
-									}
+					coll := "bidding"
+					if iid > "5a862e7040d2d9bbe88e3b1f" {
+						coll = "bidding"
+					} else {
+						coll = "bidding_back"
+					}
+					info, _ := MongoB.FindById(coll, iid, bson.M{"agencytel": 1, "agencyperson": 1, "buyertel": 1, "buyerperson": 1, "winnertel": 1, "winnerperson": 1})
+					if len(*info) > 0 {
+						if identity_type == 1 {
+							if util.ObjToString((*info)["buyertel"]) != "" {
+								q := make(map[string]interface{})
+								q["name_id"] = name_id
+								q["identity_type"] = identity_type
+								q["contact_tel"] = util.ObjToString((*info)["buyertel"])
+								if util.ObjToString((*info)["buyerperson"]) != "" {
+									q["contact_name"] = util.ObjToString((*info)["buyerperson"])
+								}
+								cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+								if cinfo != nil && len(*cinfo) > 0 {
+									MysqlTool.Update("dws_f_bpmc_relation_new", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
+								}
+							}
+						} else if identity_type == 2 {
+							if util.ObjToString((*info)["winnertel"]) != "" {
+								q := make(map[string]interface{})
+								q["name_id"] = name_id
+								q["identity_type"] = identity_type
+								q["contact_tel"] = util.ObjToString((*info)["winnertel"])
+								if util.ObjToString((*info)["winnerperson"]) != "" {
+									q["contact_name"] = util.ObjToString((*info)["winnerperson"])
+								}
+								cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+								if cinfo != nil && len(*cinfo) > 0 {
+									MysqlTool.Update("dws_f_bpmc_relation_new", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
+								}
+							}
+						} else if identity_type == 4 {
+							if util.ObjToString((*info)["agencytel"]) != "" {
+								q := make(map[string]interface{})
+								q["name_id"] = name_id
+								q["identity_type"] = identity_type
+								q["contact_tel"] = util.ObjToString((*info)["agencytel"])
+								if util.ObjToString((*info)["agencyperson"]) != "" {
+									q["contact_name"] = util.ObjToString((*info)["agencyperson"])
+								}
+								cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+								if cinfo != nil && len(*cinfo) > 0 {
+									MysqlTool.Update("dws_f_bpmc_relation_new", bson.M{"id": cid}, bson.M{"contact_id": (*cinfo)["id"]})
 								}
 							}
 						}
@@ -290,9 +298,9 @@ func project() *cobra.Command {
 		Use:   "project",
 		Short: "Start processing project data",
 		Run: func(cmd *cobra.Command, args []string) {
-			go SaveProFunc()
-			go SaveProTagFunc()
-			go SaveProbFunc()
+			//go SaveProFunc()
+			//go SaveProTagFunc()
+			//go SaveProbFunc()
 			go SaveRelationFunc()
 			taskP()
 		},
@@ -735,7 +743,7 @@ func SaveRelationFunc() {
 					defer func() {
 						<-saveRelationSp
 					}()
-					MysqlTool.InsertBulk("dws_f_bpmc_relation", RelationField, arru...)
+					MysqlTool.InsertBulk("dws_f_bpmc_relation_new", RelationField, arru...)
 				}(arru)
 				arru = make([]map[string]interface{}, saveSize)
 				indexu = 0
@@ -747,7 +755,7 @@ func SaveRelationFunc() {
 					defer func() {
 						<-saveRelationSp
 					}()
-					MysqlTool.InsertBulk("dws_f_bpmc_relation", RelationField, arru...)
+					MysqlTool.InsertBulk("dws_f_bpmc_relation_new", RelationField, arru...)
 				}(arru[:indexu])
 				arru = make([]map[string]interface{}, saveSize)
 				indexu = 0

data_tidb/project.go

@@ -1,13 +1,13 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/redis"
 	"data_tidb/config"
 	"fmt"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
 	"strings"
 	"sync"
 	"time"
@@ -21,7 +21,7 @@ func taskP() {
 	wg := &sync.WaitGroup{}
 
 	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("5c4044d4a5cb26b9b7b963cc")}
-	query := sess.DB(config.Conf.DB.MongoP.Dbname).C("projectset").Find(nil).Sort("_id").Iter()
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C("projectset_20230407").Find(nil).Sort("_id").Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
 		if count%20000 == 0 {
@@ -34,9 +34,9 @@ func taskP() {
 				<-ch
 				wg.Done()
 			}()
-			taskPro(tmp)
-			taskBusiness(tmp)
-			taskProTag(tmp)
+			//taskPro(tmp)
+			//taskBusiness(tmp)
+			//taskProTag(tmp)
 
 			taskRelation(tmp)
 
@@ -261,7 +261,7 @@ func taskRelation(tmp map[string]interface{}) {
 				if code := redis.GetStr("qyxy_id", b); code != "" {
 					saveM[f] = code
 				}
-			} else if f == "contract_id" {
+			} else if f == "contact_id" {
 				// todo
 			} else if f == "identity_type" {
 				saveM[f] = 1 // 001
@@ -283,7 +283,7 @@ func taskRelation(tmp map[string]interface{}) {
 				if code := redis.GetStr("qyxy_id", a); code != "" {
 					saveM[f] = code
 				}
-			} else if f == "contract_id" {
+			} else if f == "contact_id" {
 
 			} else if f == "identity_type" {
 				saveM[f] = 4 // 100
@@ -309,7 +309,7 @@ func taskRelation(tmp map[string]interface{}) {
 				if code := redis.GetStr("qyxy_id", ws); code != "" {
 					saveM[f] = code
 				}
-			} else if f == "contract_id" {
+			} else if f == "contact_id" {
 
 			} else if f == "identity_type" {
 				saveM[f] = 2 // 010

data_tidb/util.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"reflect"
 	"strings"
 	"unicode/utf8"

field_py/go.mod

@@ -3,11 +3,11 @@ module field-dispose
 go 1.16
 
 require (
-	app.yhyue.com/BP/servicerd v0.0.0-20201203055056-87643512f867
-	app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f
 	github.com/BurntSushi/toml v1.2.0
-	go.mongodb.org/mongo-driver v1.11.0
+	go.mongodb.org/mongo-driver v1.11.7
 	go.uber.org/zap v1.22.0
-	google.golang.org/grpc v1.49.0
-	google.golang.org/protobuf v1.27.1
+	google.golang.org/grpc v1.56.0
+	google.golang.org/protobuf v1.30.0
+	jygit.jydev.jianyu360.cn/BP/servicerd v0.0.0-20230621085048-b7e09df0c982
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
 )

field_py/init.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	"app.yhyue.com/data_processing/common_utils/log"
 	"field-dispose/config"
 	"fmt"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"os"
 )
 

field_py/main.go

@@ -1,16 +1,16 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/elastic"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/udp"
 	"encoding/json"
 	"field-dispose/config"
 	"fmt"
 	"go.uber.org/zap"
 	"io/ioutil"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"net"
 	"net/http"
 	"sync"
@@ -121,10 +121,14 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 				UdpChan <- mapInfo
 			case "tout-true":
 				Skipping = true
+				go UdpClient.WriteUdp([]byte(fmt.Sprintf("Skipping:%s", "true")), udp.OP_NOOP, ra)
 			case "tout-false":
 				Skipping = false
+				go UdpClient.WriteUdp([]byte(fmt.Sprintf("Skipping:%s", "false")), udp.OP_NOOP, ra)
+			case "monitor":
+				log.Info("monitor", zap.Any("mapInfo:", mapInfo))
+				go UdpClient.WriteUdp([]byte(util.ObjToString(mapInfo["key"])), udp.OP_NOOP, ra)
 			}
-
 		}
 	case udp.OP_NOOP: //下个节点回应
 		ok := string(data)
@@ -231,7 +235,7 @@ func updateEsMethod() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk(config.Conf.DB.Es.IndexS, config.Conf.DB.Es.TypeS, arru...)
+					Es.UpdateBulk(config.Conf.DB.Es.IndexS, arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0
@@ -243,7 +247,7 @@ func updateEsMethod() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk(config.Conf.DB.Es.IndexS, config.Conf.DB.Es.TypeS, arru...)
+					Es.UpdateBulk(config.Conf.DB.Es.IndexS, arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0

field_py/task.go

@@ -1,9 +1,6 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -15,13 +12,16 @@ import (
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"net/http"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	service "app.yhyue.com/BP/servicerd/proto"
+	service "jygit.jydev.jianyu360.cn/BP/servicerd/proto"
 )
 
 var IpDialErrNum, IpGetErrNum, ExtractDialErrNum = int64(0), int64(0), int64(0)
@@ -186,7 +186,7 @@ func rpcGetFieldP(reqStr, id string) (map[string]interface{}, error) {
 	//重试获取ip、port
 	if Skipping {
 		for i := 1; i <= 3; i++ {
-			repl, err := ipClient.Apply(context.Background(), &service.ApplyReqData{Name: "goods_service", Balance: 3})
+			repl, err := ipClient.Apply(context.Background(), &service.ApplyReqData{Name: "goods_service", Balance: 2})
 			if err != nil {
 				continue
 			} else {
@@ -197,7 +197,7 @@ func rpcGetFieldP(reqStr, id string) (map[string]interface{}, error) {
 		}
 	} else {
 		for {
-			repl, err := ipClient.Apply(context.Background(), &service.ApplyReqData{Name: "goods_service", Balance: 3})
+			repl, err := ipClient.Apply(context.Background(), &service.ApplyReqData{Name: "goods_service", Balance: 2})
 			if err != nil {
 				continue
 			} else {
@@ -226,39 +226,39 @@ func rpcGetFieldP(reqStr, id string) (map[string]interface{}, error) {
 	req := &proto.GoodsRequest{
 		Contents: reqStr,
 	}
-	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*2)
-	defer cancel()
-	//go func(ctx context.Context) {
-	//	select {
-	//	case <-ctx.Done():
-	//		return
-	//	case <-time.After(2 * time.Minute):
-	//		// 超时处理
-	//		//log.Info("rpcGetFieldP 字段识别超过2min", zap.Any("serve", "goods_service"), zap.String("id", id), zap.Any("ip+port", addr))
-	//		ErrorInfoCache <- map[string]interface{}{
-	//			"err":        "接口处理超过2min",
-	//			"type":       "采购意向/标的物",
-	//			"id":         id,
-	//			"comeintime": time.Now().Unix(),
-	//			"ok":         false,
-	//			"gtid":       gtid,
-	//			"letid":      lteid,
-	//		}
-	//		return
-	//	}
-	//}(ctx)
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Second*30)
+	go func(ctx context.Context) {
+		select {
+		case <-ctx.Done():
+			//_, _ = ipClient.Release(context.Background(), &service.ApplyRepData{Ip: ip, Port: int32(port)})
+		case <-time.After(time.Second * 30):
+			// 超时处理
+			//log.Info("rpcGetFieldP 字段识别超过2min", zap.Any("serve", "goods_service"), zap.String("id", id), zap.Any("ip+port", addr))
+			//ErrorInfoCache <- map[string]interface{}{
+			//	"err":        "接口处理超过2min",
+			//	"type":       "采购意向/标的物",
+			//	"id":         id,
+			//	"comeintime": time.Now().Unix(),
+			//	"ok":         false,
+			//	"gtid":       id,
+			//	"letid":      id,
+			//}
+
+		}
+	}(ctx)
 	resp, err := client.GoodsExtract(ctx, req)
+	cancel()
 	if err != nil {
-		_, _ = ipClient.Release(context.Background(), &service.ApplyRepData{Ip: ip, Port: int32(port)})
+		//_, _ = ipClient.Release(context.Background(), &service.ApplyRepData{Ip: ip, Port: int32(port)})
 		return nil, err
 	}
 	result := map[string]interface{}{}
 	if json.Unmarshal([]byte(resp.Goods), &result) != nil {
-		_, _ = ipClient.Release(context.Background(), &service.ApplyRepData{Ip: ip, Port: int32(port)})
+		//_, _ = ipClient.Release(context.Background(), &service.ApplyRepData{Ip: ip, Port: int32(port)})
 		return nil, errors.New("Json Unmarshal Error")
 	}
 	// 服务中心释放服务
-	_, _ = ipClient.Release(context.Background(), &service.ApplyRepData{Ip: ip, Port: int32(port)})
+	//_, _ = ipClient.Release(context.Background(), &service.ApplyRepData{Ip: ip, Port: int32(port)})
 	return result, nil
 }
 
@@ -317,9 +317,18 @@ func rpcGetFieldR(reqStr, id string) (map[string]interface{}, error) {
 	req := &proto.ContentRequest{
 		Contents: reqStr,
 	}
-	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*2)
-	defer cancel()
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Second*30)
+	go func(ctx context.Context) {
+		select {
+		case <-ctx.Done():
+
+		case <-time.After(time.Second * 30):
+			// 超时处理
+			//log.Info("rpcGetFieldP 字段识别超过2min", zap.Any("serve", "extract_expert_service"), zap.String("id", id), zap.Any("ip+port", addr))
+		}
+	}(ctx)
 	resp, err := client.Extract(ctx, req)
+	cancel()
 	if err != nil {
 		return nil, err
 	}

field_sync/file.go

@@ -1,12 +1,12 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
 	"bufio"
 	"fmt"
 	"go.uber.org/zap"
 	"io"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"os"
 	"regexp"
 	"strconv"

field_sync/go.mod

@@ -3,11 +3,11 @@ module field_sync
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005
 	github.com/BurntSushi/toml v1.2.0
 	github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible
 	go.mongodb.org/mongo-driver v1.11.0
 	go.uber.org/zap v1.22.0
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
 )

field_sync/go.sum

@@ -1,7 +1,5 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f h1:5fUbVRwPM3oBsZgvG76Bia3I4SdwdBB6PvJ6B28Qkyc=
-app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
-app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005 h1:AEEi+8ao9pTVqPIh6uVvjxBby/i43fFj7DwVo+feDAE=
-app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
@@ -10,21 +8,55 @@ github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible h1:QoRMR0TCctLDqBCMyOu1e
 github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
 github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -32,15 +64,22 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -60,6 +99,9 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.mongodb.org/mongo-driver v1.11.0 h1:FZKhBSTydeuffHj9CBjXlR8vQLee1cQyTWYPA6/tqiE=
 go.mongodb.org/mongo-driver v1.11.0/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
@@ -70,28 +112,45 @@ go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
 go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -100,7 +159,10 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ=
 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -108,6 +170,25 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -115,9 +196,13 @@ gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1 h1:MbtiS3BwDnB7uRjHFMCBe7/oW+aLaGMeQNiQa3qvd4c=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

field_sync/init.go

@@ -1,10 +1,10 @@
 package main
 
 import (
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"field_sync/config"
 	"fmt"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"os"
 )
 

field_sync/jykeyword.go

@@ -1,11 +1,11 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mfw"
 	"encoding/json"
 	"field_sync/config"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mfw"
 	"regexp"
 	"strings"
 	"time"

field_sync/main.go

@@ -1,12 +1,6 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	gonsq "app.yhyue.com/data_processing/common_utils/nsq"
-	"app.yhyue.com/data_processing/common_utils/redis"
-	"app.yhyue.com/data_processing/common_utils/udp"
 	"encoding/json"
 	"field_sync/config"
 	"field_sync/oss"
@@ -14,6 +8,12 @@ import (
 	"go.uber.org/zap"
 	"gopkg.in/mgo.v2/bson"
 	"io/ioutil"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	gonsq "jygit.jydev.jianyu360.cn/data_processing/common_utils/nsq"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"net"
 	"net/http"
 	"strings"
@@ -109,6 +109,8 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					}()
 					biddingAllTask(data, mapInfo)
 				}()
+			case "monitor":
+				log.Info("monitor", zap.Any("mapInfo:", mapInfo))
 			default:
 				pool <- true
 				go func() {

field_sync/oss/ossclient.go

@@ -1,10 +1,10 @@
 package oss
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
 	"fmt"
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"io/ioutil"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"os"
 )
 

field_sync/task.go

@@ -1,17 +1,17 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/redis"
-	"app.yhyue.com/data_processing/common_utils/udp"
 	"encoding/json"
 	"field_sync/config"
 	"field_sync/oss"
 	"fmt"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"net"
 	"reflect"
 	"regexp"
@@ -167,15 +167,15 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 							del[k] = 1
 							//util.Debug("抽取结果没有值,bidding有值：field--", k, "val--", v2)
 						}
-
 					}
-					//if util.IntAll(compare["repeat"]) == 1 {
-					//	update["extracttype"] = -1
-					//	update["dataprocess"] = 7
-					//} else {
-					//	update["extracttype"] = 1
-					//	update["dataprocess"] = 8
-					//}
+					if util.IntAll(compare["repeat"]) == 1 {
+						update["extracttype"] = -1
+						update["dataprocess"] = 7
+						update["repeat_id"] = compare["repeat_id"]
+					} else {
+						update["extracttype"] = 1
+						update["dataprocess"] = 8
+					}
 					break
 				} else {
 					if cid < tid {
@@ -294,6 +294,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 			if util.IntAll(compare["repeat"]) == 1 {
 				update["extracttype"] = -1
 				update["dataprocess"] = 7
+				update["repeat_id"] = compare["repeat_id"]
 			} else {
 				update["extracttype"] = 1
 				update["dataprocess"] = 8
@@ -655,6 +656,7 @@ func taskinfo(id string) {
 	if util.IntAll((*extractM)["repeat"]) == 1 {
 		update["extracttype"] = -1
 		update["dataprocess"] = 7
+		update["repeat_id"] = (*extractM)["repeat_id"]
 	} else {
 		update["extracttype"] = 1
 		update["dataprocess"] = 8
@@ -708,7 +710,7 @@ func methodPb(tmp map[string]interface{}) int64 {
 		}
 	}
 	for _, d := range DateTimeSelect {
-		if tmp[d] != nil {
+		if tmp[d] != nil && util.Int64All(tmp[d]) < time.Now().Unix() {
 			return util.Int64All(tmp[d])
 		}
 	}
@@ -716,7 +718,7 @@ func methodPb(tmp map[string]interface{}) int64 {
 }
 
 // Duplicate
-//@Description 去重
+// @Description 去重
 // @Author J 2023/5/24 09:53
 func Duplicate(a interface{}) (ret []interface{}) {
 	va := reflect.ValueOf(a)

monitor/go.mod

@@ -3,10 +3,10 @@ module monitor
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005
 	github.com/BurntSushi/toml v1.2.1
 	github.com/robfig/cron v1.2.0
 	github.com/spf13/cobra v1.6.1
 	go.mongodb.org/mongo-driver v1.11.0
 	go.uber.org/zap v1.23.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67 // indirect
 )

monitor/go.sum

@@ -1,9 +1,5 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
-app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
-app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005 h1:AEEi+8ao9pTVqPIh6uVvjxBby/i43fFj7DwVo+feDAE=
-app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
-app.yhyue.com/data_processing/field_info_tag v1.1.1 h1:ouJx+s77O8N69RALYyBkOPNyVeJ8WZL+2Ot/eZSEHTw=
-app.yhyue.com/data_processing/field_info_tag v1.1.1/go.mod h1:i9NKOqhWmcDl0Cl5qHBBBx46Da0Qzys7L0J+3my/Hbw=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
@@ -13,8 +9,12 @@ github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x0
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/antlabs/strsim v0.0.3 h1:J9AHxnybJZHKBoxeup1VZNWt3ST8QD+ieDJsm/nEpRo=
 github.com/antlabs/strsim v0.0.3/go.mod h1:bIcymn+2jtt01korFun0bs8PsYZeQa82aHoYMi7cm30=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -23,19 +23,49 @@ github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
 github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
 github.com/donnie4w/go-logger v0.0.0-20170827050443-4740c51383f4 h1:T9PR91sjTtrA1HmZB4G+M7OLCelch0f6rIEY7Mm1T4U=
 github.com/donnie4w/go-logger v0.0.0-20170827050443-4740c51383f4/go.mod h1:L7S4x0R7vv3xoOhGuyAJyCO2MYzWOpccM4Isn8jIUgY=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/go-ego/gse v0.70.2 h1:y2UMOHJMtI+0b2GjxTtQfKON5DMmlyX1hOQHTo8UVVs=
 github.com/go-ego/gse v0.70.2/go.mod h1:kesekpZfcFQ/kwd9b27VZHUOH5dQUjaaQUZ4OGt4Hj4=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -43,18 +73,25 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
 github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -84,6 +121,9 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.mongodb.org/mongo-driver v1.11.0 h1:FZKhBSTydeuffHj9CBjXlR8vQLee1cQyTWYPA6/tqiE=
 go.mongodb.org/mongo-driver v1.11.0/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
@@ -95,35 +135,56 @@ go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -131,6 +192,25 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
@@ -139,8 +219,19 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXL
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/olivere/elastic.v2 v2.0.61 h1:7cpl3MW8ysa4GYFBXklpo5mspe4NK0rpZTdyZ+QcD4U=
 gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20221205033056-885644941005 h1:AEEi+8ao9pTVqPIh6uVvjxBby/i43fFj7DwVo+feDAE=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20221205033056-885644941005/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67 h1:5rabE3IQRjI/NXfgJyquRTMdzSy7SfufFhmREixLZXg=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+jygit.jydev.jianyu360.cn/data_processing/field_info_tag v1.1.1 h1:ouJx+s77O8N69RALYyBkOPNyVeJ8WZL+2Ot/eZSEHTw=
+jygit.jydev.jianyu360.cn/data_processing/field_info_tag v1.1.1/go.mod h1:i9NKOqhWmcDl0Cl5qHBBBx46Da0Qzys7L0J+3my/Hbw=

monitor/init.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"fmt"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"monitor/config"
 	"os"
 )

monitor/main.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
 	"bytes"
 	"encoding/json"
 	"fmt"
@@ -11,6 +9,8 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
 	"io/ioutil"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"monitor/config"
 	"net/http"
 )

processing_ids/go.mod

@@ -3,7 +3,7 @@ module processing_ids
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061
 	github.com/robfig/cron v1.2.0
 	go.mongodb.org/mongo-driver v1.11.0 // indirect
 	go.uber.org/zap v1.23.0

processing_ids/go.sum

@@ -1,5 +1,5 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
-app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=

processing_ids/main.go

@@ -1,9 +1,6 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"flag"
 	"fmt"
 	"github.com/robfig/cron"
@@ -11,6 +8,9 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.uber.org/zap"
 	"io/ioutil"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"net/http"
 	"time"
 )