es7 更新存储，oss 配置到文件

README.md

@@ -1,7 +1,18 @@
 ## 数据处理项目-索引数据处理
 
-
 ### udpcreateindex 索引处理程序-v1.0
 
 ### createEsIndex 索引处理程序-v2.0
 
+
+
+
+
+### udpcreateindex
+
+
+
+1. 配置文件和 196 正式环境区别
+   1. [log] 配置下`logpath`  189 环境为空，196 环境下配置不为空；为空将输出控制台
+2. bidding_es 文件中 `GetEsField` 方法 为公用的，全量跑数据不需要`saveErr`，``saveErr`  用来存储 针对 `projectscope` 、`detail` 、 `filetext` 附件内容 错误的；全量跑数据时屏蔽掉，跑增量数据打开。
+

common.toml

@@ -0,0 +1,201 @@
+
+[udp]
+locport = ":17834"
+jyaddr = "172.17.145.178"
+jyport = 11118
+
+[db]
+[db.mongoB]
+addr = "172.17.4.187:27082,172.17.145.163:27083"
+dbname = "qfw"
+coll = "bidding"
+size = 15
+user = "SJZY_RWbid_ES"
+password = "SJZY@B4i4D5e6S"
+
+[db.mongoP]
+addr = "172.17.4.85:27080"
+dbname = "qfw"
+coll = "projectset_20230407"
+size = 15
+user = ""
+password = ""
+
+[db.mongoQ]
+addr = "172.17.4.187:27082,172.17.145.163:27083"
+dbname = "mixdata"
+coll = ""
+size = 15
+user = "SJZY_RWbid_ES"
+password = "SJZY@B4i4D5e6S"
+
+[db.oss]
+    endpoint = "oss-cn-beijing-internal.aliyuncs.com"## 正式环境
+  ##  endpoint = "oss-cn-beijing.aliyuncs.com"## 测试环境
+    accesskey = "LTAI4G5x9aoZx8dDamQ7vfZi"
+    accesssecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
+    bucketname = "topjy"
+
+
+[db.es]
+addr = "http://172.17.4.184:19805,http://172.17.148.50:19805,http://172.17.145.164:19805"
+addrp = "http://172.17.145.178:9800"
+size = 5
+username = "es_all"
+password = "TopJkO2E_d1x"
+indexb = "bidding_v1"
+typeb = "bidding"
+indexp = "projectset_v1"
+typep = "projectset"
+indexwinner = "winner"
+typewinner = "winner"
+indexbuyer = "buyer"
+typebuyer = "buyer"
+detailfilter = ["(招标网|千里马|采招网|招标采购导航网|招标与采购网|中国招投标网|中国采购与招标网|中国采购与招标|优质采)[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}(qianlima|zhaobiao|okcis|zbytb|infobidding|bidcenter|youzhicai|chinabidding|Chinabidding｜CHINABIDDING)[a-z0-9.\\/\\/]{0,40}",
+    "招标网[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}zhaobiao[a-z0-9.\\/\\/]{0,40}",
+    "千里马[\\w\\W]{0,15}[a-z0-9:\\/\\/.]{0,20}qianlima[a-z0-9.\\/\\/]{0,10}",
+    "[\\(（]?(网址)?[:：；;]?(http|https|htpps)*[:：]?(\\/\\/)?(www|jinan|WWW)?.(zhaobiao|chinabidding|Chinabidding|CHINABIDDING|infobidding|zbytb|okcis|qianlima|youzhicai).(com|cn|COM|CN)?(.cn|.CN)?\\/?[\\)）]?",
+    "[\\(（]?(网址)?(:：)?(http|https|htpps)*(:|：)?\\/\\/www.bidcenter.com.cn\\/",
+    "千里马(平台|网站)+", "[“\"]?优质采(平台|电子交易平台|云采购平台|交易平台)?[”\"]?", "《?(中国采购与|中国)?招(投)?标(与采购|采购导航)?网》?",
+    "《?元博网(采购与招标网)?》?", "《?(中国)?招标采购导航网》?", "中\\W{0,3}国采\\W{0,3}招\\W{0,3}网\\W*[(（]?(bidcenter.com.cn)?[)）]?", "已方宝", "中国招标与采购"]
+
+[mail]
+send = true
+to = "wangjianghan@topnet.net.cn"
+api = "http://172.17.145.179:19281/_send/_mail"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = "./log/log.out"
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"
+
+[db.es.fieldes]
+"_id" = ""
+"buyerzipcode" = "string"
+"winnertel" = "string"
+"winnerperson" = "string"
+"contractcode" = "string"
+"winneraddr" = "string"
+"agencyaddr" = "string"
+"buyeraddr" = "string"
+"signaturedate" = "int64"
+"projectperiod" = "string"
+"projectaddr" = "string"
+"agencytel" = "string"
+"agencyperson" = "string"
+"buyerperson" = "string"
+"agency" = "string"
+"projectscope" = "string"
+"projectcode" = "string"
+"bidopentime" = "int64"
+"supervisorrate" = "float64"
+"buyertel" = "string"
+"bidamount" = "float64"
+"winner" = "string"
+"buyer" = "string"
+"budget" = "float64"
+"projectname" = "string"
+"bidstatus" = "string"
+"buyerclass" = "string"
+"topscopeclass" = ""
+"s_topscopeclass" = "string"
+"s_subscopeclass" = "string"
+"area" = "string"
+"city" = "string"
+"district" = "string"
+"s_winner" = "string"
+"title" = "string"
+"detail" = "string"
+"site" = "string"
+"comeintime" = "int64"
+"href" = "string"
+"infoformat" = "int32"
+"publishtime" = "int64"
+"s_sha" = "string"
+"spidercode" = "string"
+"subtype" = "string"
+"toptype" = "string"
+"projectinfo" = ""
+"purchasing" = "string"
+"purchasinglist" = ""
+"channel" = "string"
+"winnerorder" = ""
+"project_scale" = "string"
+"project_duration" = "int32"
+"project_timeunit" = "string"
+"project_startdate" = "int64"
+"project_completedate" = "int64"
+"payway" = "string"
+"contract_guarantee" = "bool"
+"bid_guarantee" = "bool"
+"qualifies" = ""
+"entidlist" = ""
+"funds" = "string"
+"review_experts" = "string"
+"bidmethod" = "string"
+"bidendtime" = "int64"
+"bidopenaddress" = "string"
+"docamount" = "float64"
+"agencyrate" = "float64"
+"agencyfee" = "float64"
+"bidway" = "string"
+"getdocmethod" = "string"
+"china_bidding" = "string"
+"purchasing_tag" = "string"
+"multipackage" = "int32"
+"isValidFile" = "bool"
+"bid_field" = "string"
+"bidstarttime" = "int64"
+"docendtime" = "int64"
+"docstarttime" = "int64"
+"signendtime" = "int64"
+"signstarttime" = "int64"
+"issue_quota" = "float64"
+"bidopen_shape" = "string"
+"quote_mode" = "string"
+"is_acquire_tender" = "bool"
+"is_payment_deposit" = "bool"
+"is_joint_bidding" = "bool"
+"procurementlist" = ""
+"object_type"="string"  ##针对中国政府采购网，添加字段，区分货物、服务和工程
+[db.es.fieldprojectinfo]
+"approvecode" = "string"
+"approvecontent" = "string"
+"approvestatus" = "string"
+"approvetime" = "string"
+"approvedept" = "string"
+"approvenumber" = "string"
+"projecttype" = "string"
+"approvecity" = "string"
+[db.es.fieldpurchasinglist]
+"itemname" = "string"
+"item" = "string"
+"brandname" = "string"
+"model" = "string"
+"unitname" = "string"
+"number" = "float64"
+"unitprice" = "float64"
+"totalprice" = "float64"
+[db.es.fieldprocurementlist]
+"projectname" = "string"
+"buyer" = "string"
+"item" = "string"
+"projectscope" = "string"
+"expurasingtime" = "string"
+"totalprice" = "float64"
+[db.es.fieldwinnerorder]
+"sort" = "int"
+"sortstr" = "string"
+"entname" = "string"

createEsIndex/bidding_es.go

@@ -16,6 +16,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"unicode/utf8"
 )
 
 var (
@@ -160,7 +161,7 @@ func biddingAllTask(mapInfo map[string]interface{}) {
 		q = convertToMongoID(q)
 	}
 
-	ch := make(chan bool, 20)
+	ch := make(chan bool, 50)
 	wg := &sync.WaitGroup{}
 
 	//bidding库
@@ -376,8 +377,11 @@ func GetEsField(tmp map[string]interface{}, stype string) (map[string]interface{
 					newTmp[field] = util.ObjToString(tmp["title"]) + " " + text
 					update["cleartag"] = b
 				}
-			} else if field == "_id" || field == "topscopeclass" || field == "entidlist" {
+			} else if field == "topscopeclass" || field == "entidlist" {
 				newTmp[field] = tmp[field]
+			} else if field == "_id" {
+				newTmp["_id"] = mongodb.BsonIdToSId(tmp["_id"])
+				newTmp["id"] = mongodb.BsonIdToSId(tmp["_id"])
 			} else if field == "publishtime" || field == "comeintime" {
 				//字段类型不正确，特别处理
 				if tmp[field] != nil && util.Int64All(tmp[field]) > 0 {
@@ -465,24 +469,26 @@ func getFileText(tmp map[string]interface{}) (filetext string) {
 					if resultMap, ok := result.(map[string]interface{}); resultMap != nil && ok {
 						if attach_url := util.ObjToString(resultMap["attach_url"]); attach_url != "" {
 							bs := oss.OssGetObject(attach_url, mongodb.BsonIdToSId(tmp["_id"])) //oss读数据
-							//if utf8.RuneCountInString(filetext+bs) < fileLength {
-							//	filetext += bs + "\n"
-							//} else {
-							//	if utf8.RuneCountInString(bs) > fileLength {
-							//		filetext = bs[0:fileLength]
-							//	} else {
-							//		filetext = bs
-							//	}
-							//	break
-							//}
-							if len(filetext) > 500000 {
-								filetext = filetext[0:500000]
-								break
+							//正式环境
+							if utf8.RuneCountInString(filetext+bs) < fileLength {
+								filetext += bs + "\n"
 							} else {
-								if len(bs) <= 500000 {
-									filetext += bs + "\n"
+								if utf8.RuneCountInString(bs) > fileLength {
+									filetext = bs[0:fileLength]
+								} else {
+									filetext = bs
 								}
+								break
 							}
+							//测试环境
+							//if len(filetext) > 500000 {
+							//	filetext = filetext[0:500000]
+							//	break
+							//} else {
+							//	if len(bs) <= 500000 {
+							//		filetext += bs + "\n"
+							//	}
+							//}
 						}
 					}
 				}

createEsIndex/buyertask.go

@@ -77,7 +77,7 @@ func buyerEsTaskOnce() {
 			arrEs = append(arrEs, savetmp)
 			if len(arrEs) >= EsBulkSize {
 				tmps := arrEs
-				Es.BulkSave(config.Conf.DB.Es.IndexBuyer, config.Conf.DB.Es.TypeBuyer, &tmps, true)
+				Es.BulkSave(config.Conf.DB.Es.IndexBuyer, tmps)
 				arrEs = []map[string]interface{}{}
 			}
 			buyerEsLock.Unlock()
@@ -89,7 +89,7 @@ func buyerEsTaskOnce() {
 	buyerEsLock.Lock()
 	if len(arrEs) > 0 {
 		tmps := arrEs
-		Es.BulkSave(config.Conf.DB.Es.IndexBuyer, config.Conf.DB.Es.TypeBuyer, &tmps, true)
+		Es.BulkSave(config.Conf.DB.Es.IndexBuyer, tmps)
 		arrEs = []map[string]interface{}{}
 	}
 	buyerEsLock.Unlock()

createEsIndex/common.toml

@@ -1,45 +1,53 @@
 [udp]
-locport = ":1783"
-jyaddr = "127.0.0.1"
-jyport = 11118
+    locport = ":17834"
+    jyaddr = "127.0.0.1"
+    jyport = 11118
 
 [db]
 [db.mongoB]
-addr = "192.168.3.207:29099"
-dbname = "wcc"
-coll = "bidding_test"
-size = 15
-user = ""
-password = ""
+    addr = "127.0.0.1:27017"
+    dbname = "wcc"
+    coll = "bidding_wcc_random2"
+    size = 15
+#    user = "root"
+#    password = "root"
 
 [db.mongoP]
-addr = "192.168.3.207:29099"
-dbname = "wcc"
-coll = "projectset"
-size = 15
-user = ""
-password = ""
+    addr = "192.168.3.206:27002"
+    dbname = "qfw_data"
+    coll = "projectset"
+    size = 15
+    user = "root"
+    password = "root"
 
 [db.mongoQ]
-addr = "192.168.3.207:29099"
-dbname = "mixdata"
-coll = "qyxy_std"
-size = 15
-user = ""
-password = ""
+    addr = "192.168.3.206:27002"
+    dbname = "mixdata"
+    coll = "qyxy_std"
+    size = 15
+    user = "root"
+    password = "root"
 
+[db.oss]
+    endpoint = "oss-cn-beijing-internal.aliyuncs.com"## 正式环境
+  ##  endpoint = "oss-cn-beijing.aliyuncs.com"## 测试环境
+    accesskey = "LTAI4G5x9aoZx8dDamQ7vfZi"
+    accesssecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
+    bucketname = "topjy"
 [db.es]
-addr = "http://192.168.3.206:9800"
-addrp = "http://192.168.3.206:9800"
-size = 5
-indexb = "bidding_v1"
-typeb = "bidding"
-indexp = "projectset_v1"
-typep = "projectset"
-indexwinner = "winner"
-typewinner = "winner"
-indexbuyer = "buyer"
-typebuyer = "buyer"
+    addr = "http://192.168.3.149:9200" ## 正常bidding 链接
+    addrp = "http://127.0.0.1:9200"  ## biddingall 链接
+#    username = "elastic"
+#    password = "123456"
+    size = 5
+    indexb = "wcc_test"
+    typeb = "bidding"
+    indexp = "projectset_v1"
+    typep = "projectset"
+    indexwinner = "winner"
+    typewinner = "winner"
+    indexbuyer = "buyer_v2"
+    typebuyer = "buyer"
 detailfilter = ["(招标网|千里马|采招网|招标采购导航网|招标与采购网|中国招投标网|中国采购与招标网|中国采购与招标|优质采)[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}(qianlima|zhaobiao|okcis|zbytb|infobidding|bidcenter|youzhicai|chinabidding|Chinabidding｜CHINABIDDING)[a-z0-9.\\/\\/]{0,40}",
     "招标网[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}zhaobiao[a-z0-9.\\/\\/]{0,40}",
     "千里马[\\w\\W]{0,15}[a-z0-9:\\/\\/.]{0,20}qianlima[a-z0-9.\\/\\/]{0,10}",

createEsIndex/config/conf.go

@@ -20,6 +20,8 @@ func Init(conf string) {
 		fmt.Printf("Err %v", err)
 		os.Exit(1)
 	}
+
+	fmt.Println("aa", Conf.DB.Oss)
 }
 
 type conf struct {
@@ -63,8 +65,16 @@ type db struct {
 	MongoP mgo
 	MongoQ mgo
 	Es     es
+	Oss    oss
 }
 
+//oss oss 阿里云配置
+type oss struct {
+	Endpoint     string
+	AccessKey    string
+	AccessSecret string
+	BucketName   string
+}
 type mgo struct {
 	Addr     string
 	Dbname   string
@@ -78,6 +88,8 @@ type es struct {
 	Addr                 string
 	AddrP                string
 	Size                 int
+	Username             string
+	Password             string
 	IndexB               string
 	TypeB                string
 	IndexP               string

createEsIndex/go.mod

@@ -3,11 +3,11 @@ module esindex
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f
+	app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061
 	github.com/BurntSushi/toml v1.2.0
 	github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible
-	github.com/robfig/cron v1.2.0 // indirect
-	go.mongodb.org/mongo-driver v1.10.2 // indirect
+	github.com/robfig/cron v1.2.0
+	go.mongodb.org/mongo-driver v1.10.2
 	go.uber.org/zap v1.23.0
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
 )

createEsIndex/go.sum

@@ -1,5 +1,9 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f h1:5fUbVRwPM3oBsZgvG76Bia3I4SdwdBB6PvJ6B28Qkyc=
-app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+app.yhyue.com/data_processing/common_utils v0.0.0-20230308013947-0ef15d576c68 h1:560oa2np5fPKQOGvsOYMV9vNQ8Y6y9O+WlUoNbnZ0UU=
+app.yhyue.com/data_processing/common_utils v0.0.0-20230308013947-0ef15d576c68/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
+app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061 h1:UaE1gor8fIhSBAXcZsNOQDMI5sD9LxGECXFh7Ik4rk4=
+app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
@@ -8,20 +12,56 @@ github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible h1:QoRMR0TCctLDqBCMyOu1e
 github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
 github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -29,16 +69,25 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
 github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -58,10 +107,12 @@ github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgk
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.mongodb.org/mongo-driver v1.10.2 h1:4Wk3cnqOrQCn0P92L3/mmurMxzdvWWs5J9jinAVKD+k=
 go.mongodb.org/mongo-driver v1.10.2/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
@@ -73,28 +124,45 @@ go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -103,7 +171,10 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ=
 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -111,6 +182,25 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -118,10 +208,11 @@ gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/olivere/elastic.v2 v2.0.61 h1:7cpl3MW8ysa4GYFBXklpo5mspe4NK0rpZTdyZ+QcD4U=
-gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

createEsIndex/init.go

@@ -65,14 +65,18 @@ func InitMgo() {
 
 func InitEs() {
 	Es = &elastic.Elastic{
-		S_esurl: config.Conf.DB.Es.Addr,
-		I_size:  config.Conf.DB.Es.Size,
+		S_esurl:  config.Conf.DB.Es.Addr,
+		I_size:   config.Conf.DB.Es.Size,
+		Username: config.Conf.DB.Es.Username,
+		Password: config.Conf.DB.Es.Password,
 	}
 	Es.InitElasticSize()
 
 	Es1 = &elastic.Elastic{
-		S_esurl: config.Conf.DB.Es.AddrP,
-		I_size:  config.Conf.DB.Es.Size,
+		S_esurl:  config.Conf.DB.Es.AddrP,
+		I_size:   config.Conf.DB.Es.Size,
+		Username: config.Conf.DB.Es.Username,
+		Password: config.Conf.DB.Es.Password,
 	}
 	Es1.InitElasticSize()
 }

createEsIndex/main.go

@@ -289,7 +289,7 @@ func SaveEsMethod() {
 					defer func() {
 						<-saveEsSp
 					}()
-					Es.BulkSave(config.Conf.DB.Es.IndexB, config.Conf.DB.Es.TypeB, &arru, true)
+					Es.BulkSave(config.Conf.DB.Es.IndexB, arru)
 				}(arru)
 				arru = make([]map[string]interface{}, EsBulkSize)
 				indexu = 0
@@ -301,7 +301,7 @@ func SaveEsMethod() {
 					defer func() {
 						<-saveEsSp
 					}()
-					Es.BulkSave(config.Conf.DB.Es.IndexB, config.Conf.DB.Es.TypeB, &arru, true)
+					Es.BulkSave(config.Conf.DB.Es.IndexB, arru)
 				}(arru[:indexu])
 				arru = make([]map[string]interface{}, EsBulkSize)
 				indexu = 0
@@ -324,7 +324,7 @@ func SaveAllEsMethod() {
 					defer func() {
 						<-saveEsAllSp
 					}()
-					Es1.BulkSave("biddingall", "bidding", &arru, true)
+					Es1.BulkSave("biddingall", arru)
 				}(arru)
 				arru = make([]map[string]interface{}, EsBulkSize)
 				indexu = 0
@@ -336,7 +336,7 @@ func SaveAllEsMethod() {
 					defer func() {
 						<-saveEsAllSp
 					}()
-					Es1.BulkSave("biddingall", "bidding", &arru, true)
+					Es1.BulkSave("biddingall", arru)
 				}(arru[:indexu])
 				arru = make([]map[string]interface{}, EsBulkSize)
 				indexu = 0
@@ -359,7 +359,7 @@ func SaveProjectEs() {
 					defer func() {
 						<-saveProjectSp
 					}()
-					Es.BulkSave(config.Conf.DB.Es.IndexP, config.Conf.DB.Es.TypeP, &arru, true)
+					Es.BulkSave(config.Conf.DB.Es.IndexP, arru)
 				}(arru)
 				arru = make([]map[string]interface{}, EsBulkSize)
 				indexu = 0
@@ -371,7 +371,7 @@ func SaveProjectEs() {
 					defer func() {
 						<-saveProjectSp
 					}()
-					Es.BulkSave(config.Conf.DB.Es.IndexP, config.Conf.DB.Es.TypeP, &arru, true)
+					Es.BulkSave(config.Conf.DB.Es.IndexP, arru)
 				}(arru[:indexu])
 				arru = make([]map[string]interface{}, EsBulkSize)
 				indexu = 0

createEsIndex/oss/ossclient.go

@@ -3,6 +3,7 @@ package oss
 import (
 	util "app.yhyue.com/data_processing/common_utils"
 	"app.yhyue.com/data_processing/common_utils/log"
+	"esindex/config"
 	"fmt"
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"go.uber.org/zap"
@@ -11,15 +12,15 @@ import (
 )
 
 var (
-	ossEndpoint        = "oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
-	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
-	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
-	ossBucketName      = "topjy"
-	ossclient          *oss.Client
+	//ossEndpoint        = "oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
+	//ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
+	//ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
+	//ossBucketName      = "topjy"
+	ossclient *oss.Client
 )
 
 func InitOss() {
-	client, err := oss.New(ossEndpoint, ossAccessKeyId, ossAccessKeySecret)
+	client, err := oss.New(config.Conf.DB.Oss.Endpoint, config.Conf.DB.Oss.AccessKey, config.Conf.DB.Oss.AccessSecret)
 	if err != nil {
 		fmt.Println("Error:", err)
 		os.Exit(-1)
@@ -30,7 +31,7 @@ func InitOss() {
 func OssGetObject(objectName, id string) string {
 	util.Catch()
 	// 获取存储空间。
-	bucket, err := ossclient.Bucket(ossBucketName)
+	bucket, err := ossclient.Bucket(config.Conf.DB.Oss.BucketName)
 	if err != nil {
 		return ""
 	}

createEsIndex/project_es.go

@@ -50,6 +50,7 @@ func projectTask(data []byte, mapInfo map[string]interface{}) {
 	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
 		if n%2000 == 0 {
 			log.Info("current", zap.Int("count", n))
+			log.Info("current", zap.Any("_id", tmp["_id"]))
 		}
 		newTmp := make(map[string]interface{})
 		newTmp["s_projectname"] = tmp["projectname"]
@@ -143,9 +144,12 @@ func projectTask(data []byte, mapInfo map[string]interface{}) {
 					} else {
 						newTmp[f] = tmp[f]
 					}
-				} else if f == "ids" || f == "_id" || f == "mpc" || f == "mpn" || f == "review_experts" || f == "winnerorder" ||
+				} else if f == "ids" || f == "mpc" || f == "mpn" || f == "review_experts" || f == "winnerorder" ||
 					f == "entidlist" || f == "first_cooperation" || f == "subscopeclass" {
 					newTmp[f] = tmp[f]
+				} else if f == "_id" {
+					newTmp["_id"] = mongodb.BsonIdToSId(tmp["_id"])
+					newTmp["id"] = mongodb.BsonIdToSId(tmp["_id"])
 				} else {
 					if fieldval := tmp[f]; reflect.TypeOf(fieldval).String() != ftype {
 						continue

createEsIndex/winnertask.go

@@ -39,7 +39,7 @@ func winnerEsTaskOnce() {
 	num_1 := 0
 	for tmp := make(map[string]interface{}); it_1.Next(&tmp); num_1++ {
 		if num_1%2000 == 0 && num_1 > 0 {
-			log.Info("current", zap.Int("count", num_1))
+			log.Info("winnerEsTaskOnce current", zap.Int("count", num_1))
 		}
 		pool <- true
 		wg.Add(1)
@@ -68,7 +68,7 @@ func winnerEsTaskOnce() {
 			arrEs = append(arrEs, savetmp)
 			if len(arrEs) >= EsBulkSize {
 				tmps := arrEs
-				Es.BulkSave(config.Conf.DB.Es.IndexWinner, config.Conf.DB.Es.TypeWinner, &tmps, true)
+				Es.BulkSave(config.Conf.DB.Es.IndexWinner, tmps)
 				arrEs = []map[string]interface{}{}
 			}
 			winerEsLock.Unlock()
@@ -80,7 +80,7 @@ func winnerEsTaskOnce() {
 	winerEsLock.Lock()
 	if len(arrEs) > 0 {
 		tmps := arrEs
-		Es.BulkSave(config.Conf.DB.Es.IndexWinner, config.Conf.DB.Es.TypeWinner, &tmps, true)
+		Es.BulkSave(config.Conf.DB.Es.IndexWinner, tmps)
 		arrEs = []map[string]interface{}{}
 	}
 	winerEsLock.Unlock()