Merge branch 'master' of https://jygit.jydev.jianyu360.cn/data_processing/data_field_dispose

README.md

@@ -36,6 +36,7 @@
   + 采购单位、代理机构唯一标识  
 + projectset数据
   + 基础信息
+  + 企业通讯录与标讯关联标数据处理
 
 ## monitor 数据异常监控
 + bidding_file数据累计异常监控（累计数据量>10000条）  

data_mgo_to_tidb_project/common.toml

@@ -0,0 +1,52 @@
+[db]
+
+[db.mysql]
+addr = "192.168.3.217:4000"
+dbname = "global_common_data"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+
+[db.mongop]
+addr = "192.168.3.206:27002"
+dbname = "qfw_data"
+size = 5
+user = "jyDevGroup"
+password = "jy@DevGroup"
+
+[db.redis]
+address = "other=192.168.3.149:1712"
+
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"
+
+[info]
+#projectset id
+projectsetIdGt = "5a7986d540d2d9bbe8396c7f"
+projectsetIdLt = "59a4e2cf5d11e1c745ea7022"
+ch = 5
+tableName = "projectset"
+
+[alarm]
+  isOpen = true                        # 异常通知开关。默认关闭
+  address = "172.17.148.49:4260"        # nsq消息通知地址。默认关闭
+  toppic = "jyalert"                     # 消息管道
+  isJsonEncode = false                 # 是否加密
+  id = "project_alert"                       # id
+  title = "项目同步出错"             # 消息标题
+  text = "项目同步出错，请查看"               # 消息正文

data_mgo_to_tidb_project/config/conf.go

@@ -0,0 +1,94 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	Conf *conf
+)
+
+// Init Config
+func Init(conf string) {
+	_, err := toml.DecodeFile(conf, &Conf)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+}
+
+type conf struct {
+	DB    db
+	Log   log
+	Info  info
+	Alarm alarm
+}
+
+// Log Config
+type log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type db struct {
+	MongoP mgo
+	Mysql  mysql
+	Redis  redis
+}
+
+type redis struct {
+	Address string
+}
+
+type mgo struct {
+	Addr     string
+	Dbname   string
+	Size     int
+	User     string
+	Password string
+}
+
+type mysql struct {
+	Addr     string
+	Dbname   string
+	Size     int
+	User     string
+	Password string
+}
+
+type duration struct {
+	time.Duration
+}
+
+type info struct {
+	ProjectsetIdGt string
+	ProjectsetIdLt string
+	Ch             int    //并发
+	TableName      string //表名 projectset
+}
+
+type alarm struct {
+	IsOpen       bool
+	Address      string
+	Toppic       string
+	IsJsonEncode bool
+	Id           string
+	Title        string
+	Text         string
+}
+
+// UnmarshalText parse 10s to time.Time
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}

data_mgo_to_tidb_project/config/conf_test.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var confs = `# log
+[log]
+logpath = ""
+maxsize = 10
+compress = true
+maxage =  7
+maxbackups = 10
+loglevel  = "info"
+format = "text"
+
+[serve]
+grpcAddr = "192.168.3.12:10021"
+udpPort = "1782"
+
+[db]
+[db.mongo]
+addr = "192.168.3.207:27092"
+dbname = "qfw"
+size = 10
+user = ""
+password = ""
+[db.mongo1]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+size = 5
+user = ""
+password = ""
+
+[db.es]
+addr = "http://192.168.3.206:9800"
+size = 5
+indexm = "medical_institution_v1"
+typem = "medical_institution"
+indexs = "supplier_product_v1"
+types = "supplier_product"
+
+`
+
+func TestInit(t *testing.T) {
+	testfile := "/tmp/crocodile.toml"
+	ioutil.WriteFile(testfile, []byte(confs), 0644)
+	Init(testfile)
+	t.Logf("%+v", Conf.Serve.GrpcAddr)
+	os.Remove(testfile)
+}

data_mgo_to_tidb_project/entity.go

@@ -0,0 +1,10 @@
+package main
+
+const (
+	T_dwd_f_bid_project_baseinfo      = "dwd_f_bid_project_baseinfo"      //项目基础信息
+	T_dwd_f_bid_project_record        = "dwd_f_bid_project_record"        //项目-标讯关系表
+	T_dwd_f_bid_project_final_package = "dwd_f_bid_project_final_package" //项目子包信息(最终状态包信息)
+	T_dwd_f_bid_project_package       = "dwd_f_bid_project_package"       //TODO 项目包记录表 没有
+	T_dwd_f_bid_project_contacts      = "dwd_f_bid_project_contacts"      //项目通讯录
+	T_dwd_f_bid_bidunit_record        = "dwd_f_bid_bidunit_record"        //标讯采购人、投标人关系记录
+)

data_mgo_to_tidb_project/field-criteria.json

@@ -0,0 +1,43 @@
+{
+  "dws_f_project_baseinfo": {
+    "field_array": ["projectid", "projectcode", "projectname", "area_code", "city_code", "district_code", "budget", "bidamount", "bidstatus", "bidtype",
+      "bidopentime", "createtime", "firsttime", "zbtime", "jgtime", "lasttime", "buyer_id", "agency_id", "updatetime"],
+    "field_criteria": {
+      "budget": {
+        "stype": "float",
+        "min": 0,
+        "max": 100000000000,
+        "decimal": 4
+      },
+      "bidamount": {
+        "stype": "float",
+        "min": 0,
+        "max": 100000000000,
+        "decimal": 4
+      },
+      "projectname": {
+        "stype": "string",
+        "length": 500,
+        "intercept": true
+      }
+    }
+  },
+  "dws_f_project_business": {
+    "field_array": ["projectid", "buyer_id", "agency_id", "winner_id", "area_code", "city_code", "district_code", "budget", "bidamount", "bidstatus", "bidtype",
+      "buyerclass_code", "firsttime", "zbtime", "jgtime", "lasttime", "bidopentime"],
+    "field_criteria": {
+      "budget": {
+        "stype": "float",
+        "min": 0,
+        "max": 100000000000,
+        "decimal": 4
+      },
+      "bidamount": {
+        "stype": "float",
+        "min": 0,
+        "max": 100000000000,
+        "decimal": 4
+      }
+    }
+  }
+}

data_mgo_to_tidb_project/go.mod

@@ -0,0 +1,14 @@
+module data_mgo_to_tidb_project
+
+go 1.16
+
+require (
+	app.yhyue.com/moapp/jybase v0.0.0-20231120053339-0b7406056861
+	bp.jydev.jianyu360.cn/BP/jynsq v0.0.0-20220222052708-ebc43af90698
+	github.com/BurntSushi/toml v1.2.0
+	github.com/gogf/gf/v2 v2.5.6
+	go.mongodb.org/mongo-driver v1.10.3
+	go.uber.org/zap v1.23.0
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061
+)

data_mgo_to_tidb_project/go.sum

@@ -0,0 +1,480 @@
+app.yhyue.com/moapp/esv1 v0.0.0-20220414031211-3da4123e648d/go.mod h1:91/lSD/hS+ckMVP3WdidRzDhC60lLMdyce9QHy0cSMA=
+app.yhyue.com/moapp/jybase v0.0.0-20231120053339-0b7406056861 h1:5EwPJ7sFwG8e1sTlyvF4PVqjCguxn9kzUFWY9cXPfvw=
+app.yhyue.com/moapp/jybase v0.0.0-20231120053339-0b7406056861/go.mod h1:wcWbj8aQHrFOKO/Bydf9/3UcNlfZK15lrvv40xJSl8w=
+bp.jydev.jianyu360.cn/BP/jynsq v0.0.0-20220222052708-ebc43af90698 h1:2xGDnrmbGt4VzAmDSsKH+C9Bq40z9eQ4NJfH/779o8g=
+bp.jydev.jianyu360.cn/BP/jynsq v0.0.0-20220222052708-ebc43af90698/go.mod h1:ojo/AUH9Yr1wzarEjOaNMkj1Cet/9r8IgLyba64Z52E=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/RoaringBitmap/roaring v1.5.0/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/aws/aws-sdk-go v1.35.20/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME=
+github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coscms/tagfast v0.0.0-20150925144250-2b69b2496250/go.mod h1:zX8vynptAghuV/KG8BOZlDeo4DsTKWfBQ154RWlkay0=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/donnie4w/go-logger v0.0.0-20170827050443-4740c51383f4/go.mod h1:L7S4x0R7vv3xoOhGuyAJyCO2MYzWOpccM4Isn8jIUgY=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogf/gf/v2 v2.5.6 h1:a1UK1yUP3s+l+vPxmV91+8gTarAP9b1IEOw0W7LNl6E=
+github.com/gogf/gf/v2 v2.5.6/go.mod h1:17K/gBYrp0bHGC3XYC7bSPoywmZ6MrZHrZakTfh4eIQ=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0=
+github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
+github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/olivere/elastic v6.2.37+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
+github.com/olivere/elastic/v7 v7.0.22/go.mod h1:VDexNy9NjmtAkrjNoI7tImv7FR4tf5zUA3ickqu5Pc8=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
+go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
+go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.10.3 h1:XDQEvmh6z1EUsXuIkXE9TaVeqHw6SwS1uf93jFs0HBA=
+go.mongodb.org/mongo-driver v1.10.3/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
+go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
+go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY=
+go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
+go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
+go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
+go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220531201128-c960675eff93/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.0.5/go.mod h1:N1OIhHAIhx5SunkMGqWbGFVeh4yTNWKmMo1GOAsohLI=
+gorm.io/gorm v1.21.3/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061 h1:xfmoVsDEqVv1XzAVxgGEDi+W9ojHBJc6OmTbg3b1tP0=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

data_mgo_to_tidb_project/init.go

@@ -0,0 +1,211 @@
+package main
+
+import (
+	"data_mgo_to_tidb_project/config"
+	"fmt"
+	"os"
+	"regexp"
+
+	"github.com/gogf/gf/v2/util/gconv"
+	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+)
+
+var (
+	regLetter = regexp.MustCompile("[a-z]*")
+)
+
+var ( //通用
+	MongoP       *mongodb.MongodbSim
+	MysqlTool    *mysqldb.Mysql
+	FCriteria    map[string]interface{}
+	AreaCode     = make(map[string]string, 5000)
+	TopTypeCode  = make(map[string]string, 10)
+	SubTypeCode  = make(map[string]string, 40)
+	BuyerCode    = make(map[string]string, 100)
+	TopScopeCode = make(map[string]interface{}, 20)
+	SubScopeCode = make(map[string]interface{}, 70)
+)
+var ( //标讯
+	saveSize       = 200
+	saveBasePool   = make(chan map[string]interface{}, 5000)
+	saveBaseSp     = make(chan bool, 1)
+	saveExpandPool = make(chan map[string]interface{}, 5000)
+	saveExpandSp   = make(chan bool, 1)
+	saveDetailPool = make(chan map[string]interface{}, 5000)
+	saveDetailSp   = make(chan bool, 1)
+	saveAttrPool   = make(chan map[string]interface{}, 5000)
+	saveAttrSp     = make(chan bool, 1)
+	saveIntentPool = make(chan map[string]interface{}, 5000)
+	saveIntentSp   = make(chan bool, 1)
+	saveBidderPool = make(chan map[string]interface{}, 5000)
+	saveBidderSp   = make(chan bool, 1)
+	saveGoodsPool  = make(chan map[string]interface{}, 5000)
+	saveGoodsSp    = make(chan bool, 1)
+
+	BaseField   []string
+	BaseVMap    map[string]interface{}
+	ExpandField []string
+	ExpandVMap  map[string]interface{}
+	DetailField = []string{"s_info_id", "s_detail", "s_contenthtml", "d_updatetime", "d_createtime"}
+	AttrField   = []string{"s_info_id", "s_file_id", "s_file_text", "i_is_bidding_doc", "d_updatetime", "d_createtime"}
+	IntentField = []string{"s_info_id", "s_intention_name", "s_intention_demand", "s_item", "s_buyer_id", "f_totalprice", "s_expurasingtime", "s_reserved_amount", "d_updatetime", "d_createtime"}
+	BidderField = []string{"s_info_id", "i_package_id", "s_bidder_id", "s_bidder_name", "s_other_bidder_id", "s_other_bidder_name", "i_is_bidders", "i_is_winner", "d_updatetime", "d_createtime"}
+	GoodsField  = []string{"s_info_id", "i_package_id", "s_purchasing_tag", "s_itemname", "s_brand", "s_model", "s_specs", "i_num", "f_unit", "f_totalprice", "d_updatetime", "d_createtime"}
+)
+var ( //项目
+	saveProPool      = make(chan map[string]interface{}, 5000)
+	saveProSp        = make(chan bool, 1)
+	saveProbPool     = make(chan map[string]interface{}, 5000)
+	saveProbSp       = make(chan bool, 1)
+	saveProTagPool   = make(chan map[string]interface{}, 5000)
+	saveProTagSp     = make(chan bool, 1)
+	saveRelationPool = make(chan map[string]interface{}, 5000)
+	saveRelationSp   = make(chan bool, 1)
+	saveErrPool      = make(chan map[string]interface{}, 5000)
+	saveErrSp        = make(chan bool, 1)
+	ProField         []string
+	ProVMap          map[string]interface{}
+	ProBusField      []string
+	ProBusVMap       map[string]interface{}
+	ProTagsField     = []string{"projectid", "labelcode", "labelvalues", "labelweight", "createtime"}
+	RelationField    = []string{"projectid", "infoid", "name_id", "contact_id", "identity_type", "createtime"}
+	FileTypeArr      = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"}
+)
+
+// InitLog @Description
+// @Author J 2022/7/26 15:30
+func InitLog() {
+	logcfg := config.Conf.Log
+	err := log.InitLog(
+		log.Path(logcfg.LogPath),
+		log.Level(logcfg.LogLevel),
+		log.Compress(logcfg.Compress),
+		log.MaxSize(logcfg.MaxSize),
+		log.MaxBackups(logcfg.MaxBackups),
+		log.MaxAge(logcfg.MaxAge),
+		log.Format(logcfg.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func InitMgo() {
+
+	MongoP = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoP.Addr,
+		DbName:      config.Conf.DB.MongoP.Dbname,
+		Size:        config.Conf.DB.MongoP.Size,
+		UserName:    config.Conf.DB.MongoP.User,
+		Password:    config.Conf.DB.MongoP.Password,
+	}
+	MongoP.InitPool()
+}
+
+func InitMysql() {
+	dbcfg := config.Conf.DB.Mysql
+	MysqlTool = &mysqldb.Mysql{
+		Address:  dbcfg.Addr,
+		DBName:   dbcfg.Dbname,
+		UserName: dbcfg.User,
+		PassWord: dbcfg.Password,
+	}
+	MysqlTool.Init()
+}
+
+func InitRedis() {
+	//172.17.4.238:1479
+	redis.InitRedisByDb(config.Conf.DB.Redis.Address, 10, 10, 1, 300)
+}
+
+func InitField() {
+	util.ReadConfig("./field-criteria.json", &FCriteria)
+	if len(FCriteria) > 0 {
+		//项目相关-字段限制暂时注释
+		if m, o := FCriteria["dws_f_project_baseinfo"].(map[string]interface{}); o {
+			ProField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
+			ProVMap = m["field_criteria"].(map[string]interface{})
+		} else {
+			log.Error("InitField", zap.String("field_array", "dws_f_project_baseinfo"))
+			panic("dws_f_project_baseinfo")
+		}
+		if m, o := FCriteria["dws_f_project_business"].(map[string]interface{}); o {
+			ProBusField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
+			ProBusVMap = m["field_criteria"].(map[string]interface{})
+		} else {
+			log.Error("InitField", zap.String("field_array", "dws_f_project_business"))
+			panic("dws_f_project_business")
+		}
+	} else {
+		log.Error("InitField, 未找到field-criteria.json文件")
+		panic("InitField, 未找到field-criteria.json文件")
+	}
+}
+
+func InitCode() {
+	//加载地区
+	data := MysqlTool.SelectBySql(`select * from global_common_data.code_area;`)
+	if data == nil || len(*data) == 0 {
+		log.Info("code_area代码表加载失败")
+		panic("code_area代码表加载失败")
+	}
+	for _, v := range *data {
+		code := gconv.String(v["code"])
+		area := gconv.String(v["area"])
+		city := gconv.String(v["city"])
+		district := gconv.String(v["district"])
+		if city == "" && district == "" {
+			//加载省份
+			AreaCode[area] = code
+		}
+		if city != "" && district == "" {
+			//加载城市
+			AreaCode[fmt.Sprintf("%s,%s", area, city)] = code
+		}
+		if city != "" && district != "" {
+			//加载区县
+			AreaCode[fmt.Sprintf("%s,%s,%s", area, city, district)] = code
+		}
+	}
+
+	//加载采购单位
+	buyerclassData := MysqlTool.SelectBySql(`select * from global_common_data.code_buyerclass;`)
+	if buyerclassData == nil || len(*buyerclassData) == 0 {
+		log.Info("code_buyerclass代码表加载失败")
+		panic("code_buyerclass代码表加载失败")
+	}
+	for _, v := range *buyerclassData {
+		code := gconv.String(v["code"])
+		name := gconv.String(v["name"])
+		BuyerCode[name] = code
+	}
+
+	//加载
+	bidtopsubtypeData := MysqlTool.Find("code_bidtopsubtype", nil, "", "", -1, -1)
+	if bidtopsubtypeData == nil || len(*bidtopsubtypeData) == 0 {
+		log.Info("code_bidtopsubtype代码表加载失败")
+		panic("code_bidtopsubtype代码表加载失败")
+	}
+	for _, m := range *bidtopsubtypeData {
+		if util.IntAll(m["level"]) == 1 {
+			TopTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
+		}
+		if util.IntAll(m["level"]) == 2 {
+			SubTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
+		}
+	}
+}
+
+func InitNsq() {
+	if config.Conf.Alarm.Address != "" {
+		AlertRemind.Addr = config.Conf.Alarm.Address
+		AlertRemind.Toppic = config.Conf.Alarm.Toppic
+		AlertRemind.Id = config.Conf.Alarm.Id
+	}
+}

data_mgo_to_tidb_project/main.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"data_mgo_to_tidb_project/config"
+
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
+)
+
+var (
+	UdpClient udp.UdpClient
+)
+
+func init() {
+	config.Init("./common.toml")
+	InitLog()
+	InitMgo()
+	InitMysql()
+	InitField()
+	InitCode()
+	InitRedis()
+	InitNsq()
+	log.Info("init success")
+}
+
+func main() {
+
+	Projectset()
+
+	select {}
+}

data_mgo_to_tidb_project/nsq.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+
+	"data_mgo_to_tidb_project/config"
+
+	"bp.jydev.jianyu360.cn/BP/jynsq/gonsq"
+)
+
+//异常告警
+type AlertNsq struct {
+	Addr   string
+	Toppic string
+	Id     string //用于标识这个告警分组，及告警方式、告警人
+}
+
+var AlertRemind AlertNsq
+
+//Alert 异常告警通知
+func Alert(msg string) {
+	//自定义需要的字段
+	errMap := map[string]interface{}{
+		"异常": msg,
+	}
+	//nsq
+	m := &gonsq.Msg{AlertRemind.Id, config.Conf.Alarm.Title, config.Conf.Alarm.Text, errMap}
+	bs, _ := json.Marshal(m)
+	p, _ := gonsq.NewProducer(AlertRemind.Addr, AlertRemind.Toppic, false) //测试环境地址
+	err := p.Publish(bs)
+	defer p.P.Stop()
+	if err != nil {
+		log.Println("nsq连接失败")
+	}
+}

data_mgo_to_tidb_project/pkg.go

@@ -0,0 +1,225 @@
+package main
+
+import (
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+)
+
+// 标讯分包基本信息-标讯版
+func CPBaseInfoFromBidding(tmp map[string]interface{}, tmpid string) map[string]interface{} {
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	info["s_packagename"] = util.ObjToString(tmp["projectname"])
+	if tmp["budget"] != nil {
+		info["f_budget"] = util.Float64All(tmp["budget"])
+	}
+	if tmp["bidamount"] != nil {
+		info["f_bidamount"] = util.Float64All(tmp["bidamount"])
+	}
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	return info
+}
+
+// 投标人相关信息-标讯版
+func CPBidderBiddingBaseInfo(tmp map[string]interface{}, tmpid string, pid int64) {
+	s_winner := util.ObjToString(tmp["s_winner"])
+	if s_winner == "" {
+		return
+	}
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	info["i_package_id"] = pid
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	wins := strings.Split(s_winner, ",")
+	bidder_id, bidder_name, other_bidder_id := getWinsNameId(wins)
+	info["s_bidder_id"] = bidder_id
+
+	info["i_is_winner"] = 1
+	if len(wins) > 1 {
+		info["s_bidder_name"] = bidder_name
+		info["s_other_bidder_name"] = s_winner
+		info["s_other_bidder_id"] = other_bidder_id
+		info["i_is_bidders"] = 1
+	}
+	//保存服务...
+	saveBidderPool <- info
+	//候选人相关情况
+	o_win_ids, o_win_names := getWinerorder(tmp, wins[0])
+	for k, v := range o_win_ids {
+		o_info := map[string]interface{}{}
+		o_info["s_info_id"] = tmpid
+		o_info["i_package_id"] = pid
+		o_info["s_bidder_id"] = v
+		o_info["s_bidder_name"] = o_win_names[k]
+		o_info["i_is_winner"] = 0
+		o_info["d_updatetime"] = time.Now().Format(util.Date_Full_Layout)
+		//保存服务...
+		saveBidderPool <- info
+	}
+}
+
+// 标讯分包标的物信息
+func CPBiddingPackageGoodsBaseInfo(tmp map[string]interface{}, tmpid string, pid int64) []map[string]interface{} {
+	purchasinglist := IsMarkInterfaceMap(tmp["purchasinglist"])
+	new_purlist := []map[string]interface{}{}
+	for _, v := range purchasinglist {
+		info := map[string]interface{}{}
+		info["s_info_id"] = tmpid
+		info["i_package_id"] = pid
+		info["s_itemname"] = cleanLengthName(util.ObjToString(v["itemname"]), 255)
+		info["s_brand"] = cleanLengthName(util.ObjToString(v["brandname"]), 255)
+		info["s_model"] = cleanLengthName(util.ObjToString(v["model"]), 255)
+		if number := util.IntAll(v["number"]); number > 0 && number < 1000000000 {
+			info["i_num"] = number
+		}
+		if unitprice := util.Float64All(v["unitprice"]); unitprice > 0.0 && unitprice < 1000000000.0 {
+			info["f_unit"] = unitprice
+		}
+		if totalprice := util.Float64All(v["totalprice"]); totalprice > 0.0 && totalprice < 1000000000.0 {
+			info["f_totalprice"] = totalprice
+		}
+		info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+		new_purlist = append(new_purlist, info)
+	}
+	return new_purlist
+}
+
+// 标讯分包基本信息-分包版
+func CPBaseInfoFromPackage(pinfos map[string]interface{}, tmpid string) map[string]interface{} {
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	for k, v := range pinfos {
+		info["s_packagecode"] = k
+		if pkg := util.ObjToMap(v); pkg != nil {
+			info["s_packagename"] = util.ObjToString((*pkg)["name"])
+			if (*pkg)["budget"] != nil {
+				info["f_budget"] = util.Float64All((*pkg)["budget"])
+			}
+			if (*pkg)["bidamount"] != nil {
+				info["f_bidamount"] = util.Float64All((*pkg)["bidamount"])
+			}
+			return info
+		}
+	}
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	return info
+}
+
+// 投标人相关信息-分包版
+func CPBidderPackageBaseInfo(pinfos map[string]interface{}, tmp map[string]interface{}, tmpid string, pid int64, isF bool) {
+	for _, v := range pinfos {
+		pinfo := *util.ObjToMap(v)
+		winner := util.ObjToString(pinfo["winner"])
+		info := map[string]interface{}{}
+		info["s_info_id"] = tmpid
+		info["i_package_id"] = pid
+		info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+		wins := strings.Split(winner, ",")
+		bidder_id, bidder_name, other_bidder_id := getWinsNameId(wins)
+		info["s_bidder_id"] = bidder_id
+		info["i_is_winner"] = 1
+		if len(wins) > 1 {
+			info["s_bidder_name"] = bidder_name
+			info["s_other_bidder_name"] = winner
+			info["s_other_bidder_id"] = other_bidder_id
+			info["i_is_bidders"] = 1
+		}
+		//保存服务...
+		saveBidderPool <- info
+		//候选人相关情况
+		if isF {
+			o_win_ids, o_win_names := getWinerorder(tmp, wins[0])
+			for k, v := range o_win_ids {
+				o_info := map[string]interface{}{}
+				o_info["s_info_id"] = tmpid
+				o_info["s_bidder_id"] = v
+				o_info["s_bidder_name"] = o_win_names[k]
+				o_info["i_is_winner"] = 0
+				o_info["d_updatetime"] = time.Now().Format(util.Date_Full_Layout)
+				//保存服务...
+				saveBidderPool <- info
+			}
+		}
+	}
+}
+
+// 获取所有投标人的相关标识
+func getWinsNameId(wins []string) (string, string, string) {
+	win_id, win_name, win_ids := "", "", []string{}
+	for k, v := range wins {
+		code := GetNameId(v)
+		if k == 0 {
+			win_id = code
+			win_name = v
+		}
+		win_ids = append(win_ids, code)
+	}
+	return win_id, win_name, strings.Join(win_ids, ",")
+}
+
+// 获取候选人相关信息
+func getWinerorder(tmp map[string]interface{}, win string) ([]string, []string) {
+	o_win_names, o_win_ids := []string{}, []string{}
+	winnerorder := IsMarkInterfaceMap(tmp["winnerorder"])
+	for k, v := range winnerorder {
+		if k >= 5 {
+			break
+		}
+		entname := util.ObjToString(v["entname"])
+		if entname != win {
+			o_win_ids = append(o_win_ids, GetNameId(entname))
+			o_win_names = append(o_win_names, entname)
+		}
+	}
+	return o_win_ids, o_win_names
+}
+
+// 分包信息清洗与过滤...
+func filterPackageInfos(tmp map[string]interface{}) []map[string]interface{} {
+	new_packages := []map[string]interface{}{}
+	s_winner := util.ObjToString(tmp["s_winner"])
+	if s_winner == "" {
+		return new_packages
+	}
+	winArr := strings.Split(s_winner, ",")
+	if packages := util.ObjToMap(tmp["package"]); packages != nil {
+		for k, v := range *packages {
+			if pkg := util.ObjToMap(v); pkg != nil && k != "" {
+				if win := util.ObjToString((*pkg)["winner"]); win != "" {
+					if isExistsWins(winArr, win) {
+						new_packages = append(new_packages, map[string]interface{}{k: v})
+					}
+				}
+			}
+		}
+	}
+	return new_packages
+}
+
+// 是否存在单位
+func isExistsWins(winArr []string, win string) bool {
+	win = cleaname(win)
+	for _, v := range winArr {
+		if v == win {
+			return true
+		}
+	}
+	return false
+}
+
+// 简易清洗...
+func cleaname(name string) string {
+	return name
+}
+
+// 通用长度清洗
+func cleanLengthName(name string, l int) string {
+	new_name := name
+	if utf8.RuneCountInString(name) > l {
+		new_name = string(([]rune(new_name))[:l])
+	}
+	return new_name
+}

data_mgo_to_tidb_project/save.go

@@ -0,0 +1,45 @@
+package main
+
+import "time"
+
+var ( //项目
+	saveProjectBaseInfoPool          = make(chan map[string]interface{}, 5000)
+	saveProjectBaseInfoSp            = make(chan bool, 1)
+	dwd_f_bid_project_baseinfo_field = []string{"s_projectid", "s_projectcode", "s_projectname", "s_projectname", "s_area_code", "s_city_code", "s_district_code", "s_bidstatus", "s_buyer_id", "s_agency_id", "s_bidtype", "d_bidopentime", "d_firsttime", "d_zbtime", "d_jgtime", "d_lasttime", "f_budget", "f_bidamount", "i_multipackage", "d_updatetime", "d_createtime"}
+)
+
+//项目基础信息
+func dwd_f_bid_project_baseinfo1() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveProjectBaseInfoPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveProjectBaseInfoSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveProjectBaseInfoSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_project_baseinfo", dwd_f_bid_project_baseinfo_field, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveProjectBaseInfoSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveProjectBaseInfoSp
+					}()
+					MysqlTool.InsertBulk("dwd_f_bid_project_baseinfo", BaseField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

data_mgo_to_tidb_project/service.go

@@ -0,0 +1,280 @@
+package main
+
+import (
+	"data_mgo_to_tidb_project/config"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"go.uber.org/zap"
+
+	"database/sql"
+
+	"app.yhyue.com/moapp/jybase/date"
+	"app.yhyue.com/moapp/jybase/mongodb"
+	"github.com/gogf/gf/v2/util/gconv"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+)
+
+//projectset同步
+func Projectset() {
+	sess := MongoP.GetMgoConn()
+	defer MongoP.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Info.Ch)
+	wg := &sync.WaitGroup{}
+
+	query := map[string]interface{}{}
+	if config.Conf.Info.ProjectsetIdGt != "" {
+		query = map[string]interface{}{
+			"_id": map[string]interface{}{
+				"$gt": mongodb.StringTOBsonId(config.Conf.Info.ProjectsetIdGt),
+				"$lt": mongodb.StringTOBsonId(config.Conf.Info.ProjectsetIdLt),
+			},
+		}
+	}
+	log.Info("info", zap.Any("query", query))
+	it := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.Info.TableName).Find(query).Sort("-_id").Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(tmp); count++ {
+		if count%20000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d ，%s", count, tmp["_id"]))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			MysqlTool.ExecTx("项目同步", func(tx *sql.Tx) bool {
+				ok1 := dwd_f_bid_project_baseinfo(tx, tmp)
+				ok2 := dwd_f_bid_project_record(tx, tmp)
+				ok3 := dwd_f_bid_project_final_package(tx, tmp)
+				ok4 := dwd_f_bid_project_contacts(tx, tmp)
+				ok5 := dwd_f_bid_bidunit_record(tx, tmp)
+				if !(ok1 && ok2 && ok3 && ok4 && ok5) {
+					log.Info(fmt.Sprintf("current --- %d ，%s", count, tmp["_id"]))
+				}
+				return ok1 && ok2 && ok3 && ok4 && ok5
+			})
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count))
+}
+
+//dwd_f_bid_project_baseinfo 项目基础数据存储
+func dwd_f_bid_project_baseinfo(tx *sql.Tx, tmp map[string]interface{}) bool {
+	saveMap := map[string]interface{}{}
+	_id := mongodb.BsonIdToSId(tmp["_id"])
+	//项目id
+	saveMap["s_projectid"] = _id
+	//项目编号
+	saveMap["s_projectcode"] = gconv.String(tmp["projectcode"])
+	//项目名称
+	saveMap["s_projectname"] = gconv.String(tmp["projectname"])
+	//s_area_code
+	area := gconv.String(tmp["area"])
+	saveMap["s_area_code"] = AreaCode[area]
+	//s_city_code
+	city := gconv.String(tmp["city"])
+	saveMap["s_city_code"] = AreaCode[fmt.Sprintf("%s,%s", area, city)]
+	//s_district_code
+	district := gconv.String(tmp["district"])
+	saveMap["s_district_code"] = AreaCode[fmt.Sprintf("%s,%s,%s", area, city, district)]
+	//项目状态
+	saveMap["s_bidstatus"] = gconv.String(tmp["bidstatus"])
+	//采购单位 采购单位取的name_id
+	saveMap["s_buyer_id"] = GetNameId(gconv.String(tmp["buyer"]))
+	//代理机构
+	saveMap["s_agency_id"] = GetNameId(gconv.String(tmp["agency"]))
+	//招标分类
+	saveMap["s_bidtype"] = gconv.String(tmp["bidtype"])
+	//时间相关
+	//开标时间  首条标讯公告时间  招标时间  结果时间  最后一条公告发布时间
+	TransferDateTimeInfo(tmp, &saveMap, []string{"bidopentime", "firsttime", "zbtime", "jgtime", "lasttime"})
+	//金额相关  预算 中标金额
+	TransferMoneyRateInfo(tmp, &saveMap, []string{"budget", "bidamount"}, []float64{1000000000.0, 1000000000.0})
+	//是否多包
+	saveMap["i_multipackage"] = gconv.Int(tmp["multipackage"])
+	//
+	saveMap["d_updatetime"] = time.Now().Format(date.Date_Full_Layout)
+	saveMap["d_createtime"] = time.Now().Format(date.Date_Full_Layout)
+	//存储
+	return InsertGlobalMysqlData(tx, T_dwd_f_bid_project_baseinfo, saveMap, _id) > 0
+}
+
+//dwd_f_bid_project_record 项目标讯关系表
+func dwd_f_bid_project_record(tx *sql.Tx, tmp map[string]interface{}) bool {
+	_id := mongodb.BsonIdToSId(tmp["_id"])
+
+	//遍历list 获取infoid
+	list, ok := tmp["list"].([]interface{})
+	if !ok {
+		return true
+	}
+	listMap := gconv.Maps(list)
+	if len(listMap) > 0 {
+		for _, v := range listMap {
+			saveMap := map[string]interface{}{}
+			//
+			infoid := gconv.String(v["infoid"])
+			saveMap["s_infoid"] = infoid
+			saveMap["s_projectid"] = _id
+			saveMap["d_createtime"] = time.Now().Format(date.Date_Full_Layout)
+			//存储
+			if id := InsertGlobalMysqlData(tx, T_dwd_f_bid_project_record, saveMap, _id); id <= 0 {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+//dwd_f_bid_project_final_package 项目子包信息(最终状态包信息)
+func dwd_f_bid_project_final_package(tx *sql.Tx, tmp map[string]interface{}) bool {
+	//package
+	_id := mongodb.BsonIdToSId(tmp["_id"])
+
+	//遍历list 获取infoid
+	packageList := gconv.Map(tmp["package"])
+
+	if len(packageList) > 0 {
+		for _, v := range packageList {
+			for _, vv := range gconv.Maps(v) {
+				saveMap := map[string]interface{}{}
+				//
+				saveMap["s_projectid"] = _id
+				saveMap["s_packagecode"] = gconv.String(vv["origin"]) //标(包)段编号
+				saveMap["s_packagename"] = gconv.String(vv["name"])   //标段名称
+				saveMap["s_detail"] = gconv.String(vv["text"])        //标段内容
+				//
+				TransferMoneyRateInfo(vv, &saveMap, []string{"budget", "bidamount"}, []float64{1000000000.0, 1000000000.0})
+				//
+				saveMap["d_createtime"] = time.Now().Format(date.Date_Full_Layout)
+				saveMap["d_updatetime"] = time.Now().Format(date.Date_Full_Layout)
+				saveMap["s_winner"] = GetNameId(gconv.String(vv["winner"]))
+				//存储
+				if id := InsertGlobalMysqlData(tx, T_dwd_f_bid_project_final_package, saveMap, _id); id <= 0 {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}
+
+//dwd_f_bid_project_contacts 项目通讯录
+func dwd_f_bid_project_contacts(tx *sql.Tx, tmp map[string]interface{}) bool {
+	now := time.Now().Format(date.Date_Full_Layout)
+	_id := mongodb.BsonIdToSId(tmp["_id"])
+	for _, v := range []int{1, 2, 3} {
+		saveMap := map[string]interface{}{
+			"s_projectid":  _id,
+			"d_updatetime": now,
+			"d_createtime": now,
+		}
+		if v == 1 { //采购单位
+			//联系电话
+			buyertel := gconv.String(tmp["buyertel"])
+			if buyertel == "" {
+				continue
+			}
+			if VarcharCheck(buyertel, 50) {
+				continue
+			}
+			//采购单位
+			buyer := gconv.String(tmp["buyer"])
+			saveMap["s_name_id"] = GetNameId(buyer)
+			saveMap["s_contact_name"] = buyer
+			saveMap["s_contact_tel"] = buyertel
+			saveMap["identity_type"] = v
+			if id := InsertGlobalMysqlData(tx, T_dwd_f_bid_project_contacts, saveMap, _id); id <= 0 {
+				return false
+			}
+		} else if v == 2 { //中标单位
+			// 中标单位需要遍历list
+			listMap := gconv.Maps(tmp["list"])
+			winnerMap := map[string]string{}
+			for _, vv := range listMap {
+				s_winner := gconv.String(vv["s_winner"])
+				winnertel := gconv.String(vv["winnertel"])
+				if winnertel == "" {
+					continue
+				}
+				if VarcharCheck(winnertel, 50) {
+					continue
+				}
+				for _, vvv := range strings.Split(s_winner, ",") {
+					winnerMap[vvv] = winnertel
+				}
+			}
+			//获取到所有的中标单位
+			for winner, winnertel := range winnerMap {
+				saveMap["s_name_id"] = GetNameId(winner)
+				saveMap["s_contact_name"] = winner
+				saveMap["s_contact_tel"] = winnertel
+				saveMap["identity_type"] = v
+				if id := InsertGlobalMysqlData(tx, T_dwd_f_bid_project_contacts, saveMap, _id); id <= 0 {
+					return false
+				}
+			}
+		} else if v == 3 { //代理机构
+			//联系电话
+			agencytel := gconv.String(tmp["agencytel"])
+			if agencytel == "" {
+				continue
+			}
+			if VarcharCheck(agencytel, 50) {
+				continue
+			}
+			//代理机构
+			agency := gconv.String(tmp["agency"])
+			saveMap["s_name_id"] = GetNameId(agency)
+			saveMap["s_contact_name"] = agency
+			saveMap["s_contact_tel"] = agencytel
+			saveMap["identity_type"] = v
+			if id := InsertGlobalMysqlData(tx, T_dwd_f_bid_project_contacts, saveMap, _id); id <= 0 {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+//dwd_f_bid_bidunit_record 标讯采购人、投标人关系记录
+func dwd_f_bid_bidunit_record(tx *sql.Tx, tmp map[string]interface{}) bool {
+	//package
+	_id := mongodb.BsonIdToSId(tmp["_id"])
+	//凭安库的id 需要查company_id
+	for _, first_cooperation := range gconv.Strings(tmp["first_cooperation"]) {
+		//
+		s_winner_id := GetNameIdByCompanyId(first_cooperation)
+		saveMap := map[string]interface{}{
+			"s_buyer_id":   GetNameId(gconv.String(tmp["buyer"])),
+			"s_winner_id":  s_winner_id,
+			"d_createtime": time.Now().Format(date.Date_Full_Layout),
+			"s_projectid":  _id,
+		}
+		if id := InsertGlobalMysqlData(tx, T_dwd_f_bid_bidunit_record, saveMap, _id); id <= 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+// 插入并打印信息
+func InsertGlobalMysqlData(tx *sql.Tx, name string, data map[string]interface{}, mark string) int64 {
+	id := MysqlTool.InsertByTx(tx, name, data)
+	if id == -1 {
+		log.Info("插入数据异常", zap.String(name, mark))
+		Alert(fmt.Sprintf("%s表 检测到异常数据同步，id：%s </br>", name, mark))
+
+	}
+	return id
+}

data_mgo_to_tidb_project/util.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/mgo.v2/bson"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+)
+
+// @Description 字段验证、处理
+// @Author J 2022/10/8 17:16
+func verifyF(f string, v interface{}, tmp1 interface{}) (interface{}, bool) {
+	tmp := tmp1.(map[string]interface{})
+	stype := util.ObjToString(tmp["stype"])
+	vt := reflect.TypeOf(v).String()
+	if strings.Contains(vt, stype) {
+		switch stype {
+		case "string":
+			v1 := util.ObjToString(v)
+			lens := util.IntAll(tmp["length"])
+			if tmp["intercept"].(bool) {
+				if utf8.RuneCountInString(v1) > lens {
+					return string(([]rune(v1))[:lens]), true
+				} else {
+					return v1, false
+				}
+			} else {
+				if utf8.RuneCountInString(v1) > lens {
+					return "", true
+				} else {
+					return v1, false
+				}
+			}
+		case "float":
+			v1 := util.Float64All(v)
+			min := util.Float64All(tmp["mix"])
+			max := util.Float64All(tmp["max"])
+			if min != 0 && v1 < min {
+				return nil, true
+			}
+			if max != 0 && v1 >= max {
+				return nil, true
+			}
+			dm := util.IntAll(tmp["decimal"]) //小数点位数
+			v2, err := util.FormatFloat(v1, dm)
+			if err != nil {
+				return nil, true
+			}
+			return v2, false
+
+		default:
+			break
+		}
+	}
+	return nil, true
+}
+
+func IsMarkInterfaceArr(t interface{}) []string {
+	sub_list := []string{}
+	if list_3, ok_3 := t.([]string); ok_3 {
+		sub_list = list_3
+		return sub_list
+	}
+	if list_1, ok_1 := t.(primitive.A); ok_1 {
+		sub_list = util.ObjArrToStringArr(list_1)
+	} else {
+		if list_2, ok_2 := t.([]interface{}); ok_2 {
+			sub_list = util.ObjArrToStringArr(list_2)
+		}
+	}
+	return sub_list
+}
+
+func IsMarkInterfaceMap(t interface{}) []map[string]interface{} {
+	p_list := []map[string]interface{}{}
+	if yl_list_1, ok_1 := t.(primitive.A); ok_1 {
+		p_list = util.ObjArrToMapArr(yl_list_1)
+	} else {
+		if yl_list_2, ok_2 := t.([]interface{}); ok_2 {
+			p_list = util.ObjArrToMapArr(yl_list_2)
+		}
+	}
+	return p_list
+}
+
+//获取企业的nameId
+
+//GetRedisKey 获取entname的key
+func GetRedisKey(name string) string {
+	return fmt.Sprintf("nameId_%s", name)
+}
+
+//
+func GetNameId(name string) string {
+	nameId := redis.GetStr("other", GetRedisKey(name))
+	if nameId != "" {
+		return nameId
+	}
+	info := MysqlTool.FindOne("dws_f_ent_baseinfo", map[string]interface{}{"name": name}, "name_id", "")
+	if info != nil && (*info)["name_id"] != nil {
+		name_id := util.ObjToString((*info)["name_id"])
+		//存储redis
+		redis.Put("other", GetRedisKey(name), name_id, 600)
+		return name_id
+	} else {
+		return ""
+	}
+}
+
+func GetNameIdByCompanyId(comapnyId string) string {
+	info := MysqlTool.FindOne("dws_f_ent_baseinfo", map[string]interface{}{"company_id": comapnyId}, "name_id", "")
+	if info != nil && (*info)["name_id"] != nil {
+		return util.ObjToString((*info)["name_id"])
+	} else {
+		return ""
+	}
+}
+
+// 转换时间
+func TransferDateTimeInfo(tmp map[string]interface{}, info *map[string]interface{}, keys []string) {
+	for _, v := range keys {
+		if t := util.Int64All(tmp[v]); t > 0 {
+			key := "d_" + v
+			(*info)[key] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+		}
+	}
+}
+
+// 转换金额费用
+func TransferMoneyRateInfo(tmp map[string]interface{}, info *map[string]interface{}, keys []string, maxs []float64) {
+	for k, v := range keys {
+		key := "f_" + v
+		if value, b := CleanFloat64(util.Float64All(tmp[v]), maxs[k]); tmp[v] != nil && b {
+			(*info)[key] = value
+		}
+	}
+}
+
+// 通用长度清洗
+func CleanLength(name string, l int) string {
+	if l == -1 {
+		return name
+	}
+	new_name := name
+	if utf8.RuneCountInString(name) > l {
+		new_name = string(([]rune(new_name))[:l])
+	}
+	return new_name
+}
+
+// 通用浮点清洗
+func CleanFloat64(value float64, max float64) (float64, bool) {
+	if value <= max {
+		return value, true
+	}
+	return float64(0), false
+}
+
+func GetObjectId() {
+	sid := bson.NewObjectIdWithTime(time.Unix(1700537100, 0))
+	eid := bson.NewObjectIdWithTime(time.Unix(1700580300, 0))
+	log.Println(sid)
+	log.Println(eid)
+	return
+}
+
+//判断是否varchar类型超过字符串
+func VarcharCheck(str string, length int) bool {
+	return utf8.RuneCountInString(str) > length
+}

data_project/common.toml

@@ -1,7 +1,7 @@
 [serve]
 udp = ":1782"
 thread = 1
-loadStart = 0
+loadStart = 1
 validdays = 150
 statusdays = 15
 backupFlag = true

data_project/init.go

@@ -280,7 +280,7 @@ type Info struct {
 	City                string                   `json:"city"`
 	District            string                   `json:"district"`
 	Infoformat          int                      `json:"infoformat"`
-	ReviewExperts       []string                 `json:"review_experts"`
+	ReviewExperts       string                   `json:"review_experts"`
 	Purchasing          string                   `json:"purchasing"`
 	WinnerOrder         []map[string]interface{} `json:"winnerorder"`
 	ProjectScale        string                   `json:"project_scale"`
@@ -369,7 +369,7 @@ type Project struct {
 	Buyertel           string                 `json:"buyertel"`                //采购联系人电话
 	Bidstatus          string                 `json:"bidstatus"`               //
 	Bidtype            string                 `json:"bidtype"`                 //
-	ReviewExperts      []string               `json:"review_experts"`          // 项目评审专家
+	ReviewExperts      string                 `json:"review_experts"`          // 项目评审专家
 	Purchasing         string                 `json:"purchasing"`              // 标的物
 	Package            map[string]interface{} `json:"package,omitempty"`       //分包的对比对象
 	Buyerclass         string                 `json:"buyerclass"`              //采购单位分类

data_project/load_data.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"go.uber.org/zap"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
@@ -11,7 +12,7 @@ import (
 	"time"
 )
 
-//  初始加载数据，默认加载最近6个月的数据
+// 初始加载数据，默认加载最近6个月的数据
 func (p *ProjectTask) loadData(starttime int64) {
 	log.Info("load project start..", zap.Int64("starttime", starttime))
 	p.findLock.Lock()
@@ -87,9 +88,9 @@ func (p *ProjectTask) loadData(starttime int64) {
 	ch := make(chan bool, 3)
 	wg := &sync.WaitGroup{}
 	for tmp := make(map[string]interface{}); it.Next(tmp); count++ {
-		//if count%20000 == 0 {
-		//	log.Println(fmt.Sprintf("iter --- %d", count))
-		//}
+		if count%20000 == 0 {
+			log.Info(fmt.Sprintf("iter --- %d", count))
+		}
 		ch <- true
 		wg.Add(1)
 		go func(tmp map[string]interface{}) {
@@ -193,7 +194,7 @@ func saveFiled(p *ProjectTask, res map[string]interface{}, tmp *ProjectCache) {
 	//tmp.InfoFiled = tmpMap
 }
 
-//  加载spidercode数据，isflow字段
+// 加载spidercode数据，isflow字段
 func (p *ProjectTask) loadSpiderCode() {
 	log.Info("load spider code start..")
 	p.findLock.Lock()

data_project/main.go

@@ -79,15 +79,15 @@ func DealSign() {
 	}
 }
 
-func mainT() {
+func main() {
 
 	P_QL.loadSpiderCode()
 	P_QL.loadSite()
 	if config.Conf.Serve.LoadStart > 0 {
 		P_QL.loadData(config.Conf.Serve.LoadStart)
 	}
-	//go checkMapJob()
-	//go P_QL.nsqMethod()
+	go checkMapJob()
+	go P_QL.nsqMethod()
 
 	for {
 		mapinfo, ok := <-UdpChan
@@ -167,9 +167,9 @@ func mainT() {
 	}
 }
 
-func main() {
-	sid = "626cccaa631ff1ac3d29289e"
-	eid = "640aa55d8aea8786d1cd0247"
+func mainT() {
+	sid = "64f5d956513e65bb3f0e4d4f"
+	eid = "64f5d956513e65bb3f0e4d50"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()
@@ -190,7 +190,6 @@ func main() {
 	if config.Conf.Serve.LoadStart > 0 {
 		P_QL.loadData(config.Conf.Serve.LoadStart)
 	}
-	P_QL.loadSite()
 	P_QL.currentType = mapinfo["stype"].(string)
 	P_QL.pici = time.Now().Unix()
 	P_QL.taskQl(mapinfo)

data_project/project.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"encoding/json"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
@@ -10,9 +12,6 @@ import (
 	"sort"
 	"strings"
 	"time"
-
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
 /**
@@ -516,7 +515,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 		set["multipackage"] = 0
 	}
 	//项目评审专家
-	if len(thisinfo.ReviewExperts) > 0 {
+	if thisinfo.ReviewExperts != "" {
 		set["review_experts"] = thisinfo.ReviewExperts
 	}
 	//标的物
@@ -922,7 +921,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 
 	//项目评审专家
-	if len(thisinfo.ReviewExperts) > 0 {
+	if thisinfo.ReviewExperts != "" {
 		set["review_experts"] = thisinfo.ReviewExperts
 		project.ReviewExperts = thisinfo.ReviewExperts
 	}

data_project/task.go

@@ -340,12 +340,13 @@ func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
 		log.Info("does not exist modifyinfo ---," + infoid)
 		return
 	}
-	client := Es.GetEsConn()
-	defer Es.DestoryEsConn(client)
-	esquery := `{"query": {"bool": {"must": [{"match": {"ids": "` + infoid + `"}}]}}}`
-	data := Es.Get(config.Conf.DB.Es.Index, esquery)
+	//client := Es.GetEsConn()
+	//defer Es.DestoryEsConn(client)
+	//esquery := `{"query": {"bool": {"must": [{"match": {"ids": "` + infoid + `"}}]}}}`
+	//data := Es.Get(config.Conf.DB.Es.Index, esquery)
+	data, _ := MgoP.FindOne(config.Conf.DB.MongoP.Coll, bson.M{"ids": infoid})
 	if len(*data) > 0 {
-		pid := util.ObjToString(((*data)[0])["_id"])
+		pid := util.ObjToString((*data)["_id"])
 		p.updateJudge(*infoMap, pid)
 	} else {
 		log.Info("not find project---," + infoid)
@@ -411,12 +412,13 @@ func (p *ProjectTask) delInfoPro(udpInfo map[string]interface{}) {
 	if infoid == "" {
 		return
 	}
-	client := Es.GetEsConn()
-	defer Es.DestoryEsConn(client)
-	esquery := `{"query": {"bool": {"must": [{"term": {"ids": "` + infoid + `"}}]}}}`
-	data := Es.Get(config.Conf.DB.Es.Index, esquery)
+	//client := Es.GetEsConn()
+	//defer Es.DestoryEsConn(client)
+	//esquery := `{"query": {"bool": {"must": [{"term": {"ids": "` + infoid + `"}}]}}}`
+	//data := Es.Get(config.Conf.DB.Es.Index, esquery)
+	data, _ := MgoP.FindOne(config.Conf.DB.MongoP.Coll, bson.M{"ids": infoid})
 	if len(*data) > 0 {
-		pid := util.ObjToString(((*data)[0])["_id"])
+		pid := util.ObjToString((*data)["_id"])
 		p.delJudge(infoid, pid)
 	} else {
 		log.Info("not find project---," + infoid)
@@ -494,7 +496,7 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 		c, _ := sess.DB(db).C(coll).Find(q).Count()
 		log.Info(fmt.Sprintf("共查询: %d条", c))
 	}
-	ms := sess.DB(db).C(coll).Find(q).Select(fields).Sort("publishtime")
+	ms := sess.DB(db).C(coll).Find(q).Select(fields)
 	query := ms.Iter()
 	var lastid interface{}
 L:
@@ -675,7 +677,7 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 		thisinfo.Buyer = ""
 	}
 	//清理评审专家名单
-	if len(thisinfo.ReviewExperts) > 0 {
+	if thisinfo.ReviewExperts != "" {
 		thisinfo.ReviewExperts = ClearRp(thisinfo.ReviewExperts)
 	}
 	//winners整理、清理
@@ -893,9 +895,9 @@ func deleteSlice(arr []string, v, stype string) []string {
 }
 
 // 校验评审专家
-func ClearRp(tmp []string) []string {
+func ClearRp(tmp string) string {
 	arrTmp := []string{}
-	for _, v := range tmp {
+	for _, v := range strings.Split(tmp, ",") {
 		// 汉字过滤(全汉字，2-4个字)
 		if ok, _ := regexp.MatchString("^[\\p{Han}]{2,4}$", v); !ok {
 			continue
@@ -906,7 +908,7 @@ func ClearRp(tmp []string) []string {
 		}
 		arrTmp = append(arrTmp, v)
 	}
-	return arrTmp
+	return strings.Join(arrTmp, ",")
 }
 
 func QyFilter(name, stype string) string {

data_tidb/common.toml

@@ -22,6 +22,12 @@ dbname = "qfw_data"
 size = 5
 user = "root"
 password = "root"
+[db.es]
+addr = "127.0.0.1"
+index = "bidding"
+size = 10
+user = "es_all"
+password = "TopJkO2E_d1x"
 
 [mail]
 send = false

data_tidb/config/conf.go

@@ -51,6 +51,7 @@ type log struct {
 }
 
 type db struct {
+	Es     es
 	MongoB mgo
 	MongoP mgo
 	Mysql  mysql
@@ -72,6 +73,14 @@ type mysql struct {
 	Password string
 }
 
+type es struct {
+	Addr     string
+	Index    string
+	Size     int
+	User     string
+	Password string
+}
+
 type duration struct {
 	time.Duration
 }

data_tidb/go.mod

@@ -4,9 +4,10 @@ go 1.16
 
 require (
 	github.com/BurntSushi/toml v1.2.0
+	github.com/olivere/elastic/v7 v7.0.32
 	github.com/shopspring/decimal v1.3.1
 	github.com/spf13/cobra v1.5.0
 	go.mongodb.org/mongo-driver v1.10.3
 	go.uber.org/zap v1.23.0
-	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67
 )

data_tidb/go.sum

@@ -22,6 +22,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
@@ -58,6 +59,7 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -66,10 +68,12 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -214,5 +218,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061 h1:xfmoVsDEqVv1XzAVxgGEDi+W9ojHBJc6OmTbg3b1tP0=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67 h1:5rabE3IQRjI/NXfgJyquRTMdzSy7SfufFhmREixLZXg=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

data_tidb/init.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"go.uber.org/zap"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
@@ -14,6 +15,7 @@ import (
 var (
 	MongoB, MongoP *mongodb.MongodbSim
 	MysqlTool      *mysqldb.Mysql
+	Es             *elastic.Elastic
 
 	FCriteria map[string]interface{}
 
@@ -120,6 +122,16 @@ func InitMgo() {
 	MongoP.InitPool()
 }
 
+func InitEs() {
+	Es = &elastic.Elastic{
+		S_esurl:  config.Conf.DB.Es.Addr,
+		I_size:   config.Conf.DB.Es.Size,
+		Username: config.Conf.DB.Es.User,
+		Password: config.Conf.DB.Es.Password,
+	}
+	Es.InitElasticSize()
+}
+
 func InitMysql() {
 	dbcfg := config.Conf.DB.Mysql
 	MysqlTool = &mysqldb.Mysql{

data_tidb/main.go

@@ -29,6 +29,7 @@ func init() {
 	InitField()
 
 	redis.InitRedis1("qyxy_id=127.0.0.1:8379", 1)
+	redis.InitRedis1("project_ids=127.0.0.1:8379", 0)
 	//redis.InitRedis1("qyxy_id=192.168.3.166:4379", 1)
 	log.Info("init success")
 }
@@ -48,7 +49,7 @@ func main() {
 
 	rootCmd := &cobra.Command{Use: "my cmd"}
 	rootCmd.AddCommand(bidding())
-	rootCmd.AddCommand(project())
+	rootCmd.AddCommand(relation())
 	rootCmd.AddCommand(projectAdd())
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println("rootCmd.Execute failed", err.Error())
@@ -317,25 +318,24 @@ func bidding() *cobra.Command {
 	return cmdClient
 }
 
-// @Description 项目数据
+// @Description 企业联系人关系表
 // @Author J 2022/9/20 17:52
-func project() *cobra.Command {
+func relation() *cobra.Command {
 	cmdClient := &cobra.Command{
-		Use:   "project",
-		Short: "Start processing project data",
+		Use:   "relation",
+		Short: "Start processing relation data",
 		Run: func(cmd *cobra.Command, args []string) {
-			//go SaveProFunc()
-			//go SaveProTagFunc()
-			//go SaveProbFunc()
+
+			InitEs()
+
 			go SaveRelationFunc()
-			taskP()
+			taskR()
 		},
 	}
-	//cmdClient.Flags().StringVarP(&cfg, "conf", "c", "", "server config [toml]")
 	return cmdClient
 }
 
-// @Description 项目数据
+// @Description 项目数据(目前仅关系表数据)
 // @Author J 2022/9/20 17:52
 func projectAdd() *cobra.Command {
 	var pici int64
@@ -788,7 +788,7 @@ func SaveRelationFunc() {
 					defer func() {
 						<-saveRelationSp
 					}()
-					MysqlTool.InsertBulk("dws_f_bpmc_relation", RelationField, arru...)
+					MysqlTool.InsertBulk("dws_f_bpmc_relation_new", RelationField, arru...)
 				}(arru)
 				arru = make([]map[string]interface{}, saveSize)
 				indexu = 0
@@ -800,7 +800,7 @@ func SaveRelationFunc() {
 					defer func() {
 						<-saveRelationSp
 					}()
-					MysqlTool.InsertBulk("dws_f_bpmc_relation", RelationField, arru...)
+					MysqlTool.InsertBulk("dws_f_bpmc_relation_new", RelationField, arru...)
 				}(arru[:indexu])
 				arru = make([]map[string]interface{}, saveSize)
 				indexu = 0

data_tidb/project.go

@@ -1,8 +1,11 @@
 package main
 
 import (
+	"context"
 	"data_tidb/config"
+	"encoding/json"
 	"fmt"
+	es "github.com/olivere/elastic/v7"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
@@ -14,38 +17,63 @@ import (
 	"time"
 )
 
-func taskP() {
-	sess := MongoP.GetMgoConn()
-	defer MongoP.DestoryMongoConn(sess)
+func taskR() {
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
 
-	ch := make(chan bool, 20)
 	wg := &sync.WaitGroup{}
 
-	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("64e5a63855d5406905c574e6")}
-	query := sess.DB(config.Conf.DB.MongoP.Dbname).C("projectset_20230407").Find(nil).Sort("-_id").Skip(100000).Iter()
-	count := 0
-	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
-		if count%20000 == 0 {
-			log.Info(fmt.Sprintf("current --- %d", count))
+	query := es.NewBoolQuery()
+	//Must(es.NewTermsQuery("id", "64e7685255d5406905c94a64"))
+	//Must(es.NewRangeQuery("comeintime").Gte(1688140800).Lte(1690444635)).
+	//Must(es.NewExistsQuery("yuceendtime"))
+
+	util.Debug(Es.Count("bidding", query))
+	countDocs := 0
+	res, err := client.Scroll().Index("bidding").Query(query).Scroll("5m").Size(2000).Do(context.Background()) //查询一条获取游标
+	if err == nil {
+		taskInfoA(res, wg, &countDocs)
+		scrollId := res.ScrollId
+		for {
+			searchResult, err := client.Scroll("1m").Index("bidding").ScrollId(scrollId).Size(2000).Do(context.TODO()) //查询
+			if err != nil {
+				util.Debug("Es Search Data Error：", err.Error())
+				break
+			}
+			taskInfoA(searchResult, wg, &countDocs)
+			scrollId = searchResult.ScrollId
 		}
-		ch <- true
+		wg.Wait()
+		util.Debug("over---", countDocs)
+		_, _ = client.ClearScroll().ScrollId(scrollId).Do(context.Background()) //清理游标
+	} else {
+		util.Debug(err)
+	}
+}
+
+func taskInfoA(searchResult *es.SearchResult, wg *sync.WaitGroup, countDocs *int) {
+	ch := make(chan bool, 1)
+	for _, hit := range searchResult.Hits.Hits {
+		//开始处理数据
 		wg.Add(1)
-		go func(tmp map[string]interface{}) {
+		ch <- true
+		go func(tmpHit *es.SearchHit) {
 			defer func() {
 				<-ch
 				wg.Done()
 			}()
-			//taskPro(tmp)
-			//taskBusiness(tmp)
-			//taskProTag(tmp)
 
-			taskRelation(tmp)
+			tmp := make(map[string]interface{})
+			if json.Unmarshal(tmpHit.Source, &tmp) == nil {
+				taskRelation(tmp)
+			}
+		}(hit)
+		*countDocs += 1
 
-		}(tmp)
-		tmp = make(map[string]interface{})
+		if *countDocs%10000 == 0 {
+			util.Debug("Current：", *countDocs)
+		}
 	}
-	wg.Wait()
-	log.Info(fmt.Sprintf("over --- %d", count))
 }
 
 func taskPAdd(pici int64) {
@@ -56,7 +84,7 @@ func taskPAdd(pici int64) {
 	wg := &sync.WaitGroup{}
 
 	q := bson.M{"pici": bson.M{"$gt": pici}}
-	query := sess.DB(config.Conf.DB.MongoP.Dbname).C("projectset_20230407").Find(q).Iter()
+	query := sess.DB(config.Conf.DB.MongoB.Dbname).C("bidding_back").Find(q).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
 		if count%20000 == 0 {
@@ -277,89 +305,94 @@ func taskProTag(tmp map[string]interface{}) {
 // @Description 关系表
 // @Author J 2022/9/30 13:56
 func taskRelation(tmp map[string]interface{}) {
-	pid := mongodb.BsonIdToSId(tmp["_id"])
-	if tmp["ids"] == nil {
-		log.Info("taskRelation ids err", zap.Any("id", pid))
-		return
+	id := util.ObjToString(tmp["id"])
+	pid := ""
+	if str := redis.GetStr("project_ids", id); str == "" {
+		info, _ := MongoP.FindOneByField("projectset_20230407", bson.M{"ids": id}, bson.M{"ids": 1})
+		if len(*info) > 0 {
+			pid = mongodb.BsonIdToSId((*info)["_id"])
+			for _, s := range util.ObjArrToStringArr((*info)["ids"].([]interface{})) {
+				redis.PutCKV("project_ids", s, mongodb.BsonIdToSId((*info)["_id"]))
+			}
+		} else {
+			//log.Info("taskRelation pid err", zap.Any("id", id))
+			//return
+		}
+	} else {
+		pid = str
 	}
-	ids := util.ObjArrToStringArr(tmp["ids"].([]interface{}))
-	lid := ids[len(ids)-1]
+	if b := util.ObjToString(tmp["buyer"]); b != "" {
+		saveM := make(map[string]interface{})
 
-	//if b := util.ObjToString(tmp["buyer"]); b != "" {
-	//	saveM := make(map[string]interface{})
-	//
-	//	saveM["projectid"] = pid
-	//	saveM["infoid"] = lid
-	//	saveM["identity_type"] = 1
-	//	saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
-	//	if code := redis.GetStr("qyxy_id", b); code != "" {
-	//		saveM["name_id"] = code
-	//		if util.ObjToString(tmp["buyertel"]) != "" {
-	//			q := make(map[string]interface{})
-	//			q["name_id"] = code
-	//			q["identity_type"] = 1
-	//			q["contact_tel"] = util.ObjToString(tmp["buyertel"])
-	//			if util.ObjToString(tmp["buyerperson"]) != "" {
-	//				q["contact_name"] = util.ObjToString(tmp["buyerperson"])
-	//			}
-	//			cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
-	//			if cinfo != nil && len(*cinfo) > 0 {
-	//				saveM["contact_id"] = (*cinfo)["id"]
-	//				saveRelationPool <- saveM
-	//			}
-	//		}
-	//	}
-	//}
+		saveM["projectid"] = pid
+		saveM["infoid"] = id
+		saveM["identity_type"] = 1
+		saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
+		if code := redis.GetStr("qyxy_id", b); code != "" {
+			saveM["name_id"] = code
+			if util.ObjToString(tmp["buyertel"]) != "" {
+				q := make(map[string]interface{})
+				q["name_id"] = code
+				q["identity_type"] = 1
+				q["contact_tel"] = util.ObjToString(tmp["buyertel"])
+				if util.ObjToString(tmp["buyerperson"]) != "" {
+					q["contact_name"] = util.ObjToString(tmp["buyerperson"])
+				}
+				cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+				if cinfo != nil && len(*cinfo) > 0 {
+					saveM["contact_id"] = (*cinfo)["id"]
+					saveRelationPool <- saveM
+				}
+			}
+		}
+	}
 
-	//if a := util.ObjToString(tmp["agency"]); a != "" {
-	//	saveM := make(map[string]interface{})
-	//	saveM["projectid"] = pid
-	//	saveM["infoid"] = lid
-	//	saveM["identity_type"] = 4
-	//	saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
-	//	if code := redis.GetStr("qyxy_id", a); code != "" {
-	//		saveM["name_id"] = code
-	//		if util.ObjToString(tmp["agencytel"]) != "" {
-	//			q := make(map[string]interface{})
-	//			q["name_id"] = code
-	//			q["identity_type"] = 4 // 100
-	//			q["contact_tel"] = util.ObjToString(tmp["agencytel"])
-	//			if util.ObjToString(tmp["agencyperson"]) != "" {
-	//				q["contact_name"] = util.ObjToString(tmp["agencyperson"])
-	//			}
-	//			cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
-	//			if cinfo != nil && len(*cinfo) > 0 {
-	//				saveM["contact_id"] = (*cinfo)["id"]
-	//				saveRelationPool <- saveM
-	//			}
-	//		}
-	//	}
-	//}
+	if a := util.ObjToString(tmp["agency"]); a != "" {
+		saveM := make(map[string]interface{})
+		saveM["projectid"] = pid
+		saveM["infoid"] = id
+		saveM["identity_type"] = 4
+		saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
+		if code := redis.GetStr("qyxy_id", a); code != "" {
+			saveM["name_id"] = code
+			if util.ObjToString(tmp["agencytel"]) != "" {
+				q := make(map[string]interface{})
+				q["name_id"] = code
+				q["identity_type"] = 4 // 100
+				q["contact_tel"] = util.ObjToString(tmp["agencytel"])
+				if util.ObjToString(tmp["agencyperson"]) != "" {
+					q["contact_name"] = util.ObjToString(tmp["agencyperson"])
+				}
+				cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+				if cinfo != nil && len(*cinfo) > 0 {
+					saveM["contact_id"] = (*cinfo)["id"]
+					saveRelationPool <- saveM
+				}
+			}
+		}
+	}
 
-	for _, item := range tmp["list"].([]interface{}) {
-		item1 := item.(map[string]interface{})
-		sw := util.ObjToString(item1["s_winner"])
-		if !strings.Contains(sw, ",") {
-			if code := redis.GetStr("qyxy_id", sw); code != "" {
-				saveM := make(map[string]interface{})
-				saveM["projectid"] = pid
-				saveM["infoid"] = lid
-				saveM["identity_type"] = 2
-				saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
-				saveM["name_id"] = code
-				if util.ObjToString(item1["winnertel"]) != "" {
-					q := make(map[string]interface{})
-					q["name_id"] = code
-					q["identity_type"] = 2 // 010
-					q["contact_tel"] = util.ObjToString(item1["winnertel"])
-					if util.ObjToString(item1["winnerperson"]) != "" {
-						q["contact_name"] = util.ObjToString(item1["winnerperson"])
-					}
-					cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
-					if cinfo != nil && len(*cinfo) > 0 {
-						saveM["contact_id"] = (*cinfo)["id"]
-						saveRelationPool <- saveM
-					}
+	sw := util.ObjToString(tmp["s_winner"])
+	if !strings.Contains(sw, ",") {
+		if code := redis.GetStr("qyxy_id", sw); code != "" {
+			saveM := make(map[string]interface{})
+			saveM["projectid"] = pid
+			saveM["infoid"] = id
+			saveM["identity_type"] = 2
+			saveM["createtime"] = time.Now().Format(util.Date_Full_Layout)
+			saveM["name_id"] = code
+			if util.ObjToString(tmp["winnertel"]) != "" {
+				q := make(map[string]interface{})
+				q["name_id"] = code
+				q["identity_type"] = 2 // 010
+				q["contact_tel"] = util.ObjToString(tmp["winnertel"])
+				if util.ObjToString(tmp["winnerperson"]) != "" {
+					q["contact_name"] = util.ObjToString(tmp["winnerperson"])
+				}
+				cinfo := MysqlTool.FindOne("dws_f_ent_contact", q, "", "")
+				if cinfo != nil && len(*cinfo) > 0 {
+					saveM["contact_id"] = (*cinfo)["id"]
+					saveRelationPool <- saveM
 				}
 			}
 		}

field_py/go.mod

@@ -4,7 +4,7 @@ go 1.16
 
 require (
 	github.com/BurntSushi/toml v1.2.0
-	go.mongodb.org/mongo-driver v1.11.7 // indirect
+	go.mongodb.org/mongo-driver v1.11.7
 	go.uber.org/zap v1.22.0
 	google.golang.org/grpc v1.56.0
 	google.golang.org/protobuf v1.30.0

field_py/task.go

@@ -32,7 +32,6 @@ func getIntention(mapinfo map[string]interface{}) {
 	defer util.Catch()
 	gtid, _ := mapinfo["gtid"].(string)
 	lteid, _ := mapinfo["lteid"].(string)
-
 	sess := MgoB.GetMgoConn()
 	defer MgoB.DestoryMongoConn(sess)
 	ch := make(chan bool, config.Conf.Serve.Thread)
@@ -76,6 +75,7 @@ func getIntention(mapinfo map[string]interface{}) {
 					update["review_experts"] = strings.Join(util.ObjArrToStringArr(r[id].([]interface{})), ",")
 				}
 			}
+
 			if result1 != nil && len(result1) > 0 {
 				clock.Lock()
 				if result1["purchasinglist"] != nil && len(result1["purchasinglist"].([]interface{})) > 0 {
@@ -260,11 +260,11 @@ func rpcGetFieldP(reqStr, id string) (map[string]interface{}, error) {
 	cancel()
 	if err != nil {
 		//_, _ = ipClient.Release(context.Background(), &service.ApplyRepData{Ip: ip, Port: int32(port)})
-		return nil, err
+		return nil, errors.Join(err, errors.New(fmt.Sprintf("%s:%d", ip, port)))
 	}
 	result := map[string]interface{}{}
 	if json.Unmarshal([]byte(resp.Goods), &result) != nil {
-		//_, _ = ipClient.Release(context.Background(), &service.ApplyRepData{Ip: ip, Port: int32(port)})
+		_, _ = ipClient.Release(context.Background(), &service.ApplyRepData{Ip: ip, Port: int32(port)})
 		return nil, errors.New("Json Unmarshal Error")
 	}
 	// 服务中心释放服务
@@ -417,7 +417,7 @@ func SaveErrorInfo() {
 					defer func() {
 						<-SP
 					}()
-					MgoB.SaveBulk("bidding_warn_1", savearr...)
+					MgoB.SaveBulk("bidding_py_err", savearr...)
 				}(savearr)
 				savearr = make([]map[string]interface{}, 200)
 				indexh = 0
@@ -429,7 +429,7 @@ func SaveErrorInfo() {
 					defer func() {
 						<-SP
 					}()
-					MgoB.SaveBulk("bidding_warn_1", savearr...)
+					MgoB.SaveBulk("bidding_py_err", savearr...)
 				}(savearr[:indexh])
 				savearr = make([]map[string]interface{}, 200)
 				indexh = 0

field_sync/common.toml

@@ -9,7 +9,8 @@ fields = ["buyerzipcode", "winnertel", "winnerperson", "contractcode", "winnerad
     "funds", "review_experts", "bidmethod", "bidendtime", "bidopenaddress", "docamount", "bidway", "agencyrate", "agencyfee", "getdocmethod", "purchasing_tag",
     "package", "history_updatetime", "total_investment", "owner", "projecttype", "project_person", "project_phone", "approvedept", "construction_area",
     "floor_area", "bidstarttime", "docendtime", "docstarttime", "signendtime", "signstarttime", "issue_quota", "bidopen_shape", "quote_mode", "is_acquire_tender",
-    "is_payment_deposit", "is_joint_bidding", "est_purchase_time", "est_purchase_amount", "s_project_startdate", "s_project_completedate"
+    "is_payment_deposit", "is_joint_bidding", "est_purchase_time", "est_purchase_amount", "s_project_startdate", "s_project_completedate", "enterprise_qualification",
+    "personnel_qualification", "performance_qualification", "enterprise_credit", "is_exist_pakage"
 ]
 
 [udp]

field_sync/main.go

@@ -45,7 +45,7 @@ func init() {
 	InitFileInfo()
 	InitLog()
 	InitMgo()
-	inits()
+	//inits()
 	redis.InitRedis1(config.Conf.DB.Redis.Addr, config.Conf.DB.Redis.DbIndex)
 
 	log.Info("init success")
@@ -53,7 +53,7 @@ func init() {
 
 func main() {
 	go checkMapJob()
-	go nsqMethod()
+	//go nsqMethod()
 
 	go UpdateBidding()
 	go UpdateExtract()

field_sync/task.go

@@ -30,8 +30,7 @@ func biddingTask(data []byte, mapInfo map[string]interface{}) {
 
 	stype := util.ObjToString(mapInfo["stype"])
 	if stype == "bidding" {
-		uq := bson.M{"gtid": bson.M{"$gte": util.ObjToString(mapInfo["gtid"])},
-			"lteid": bson.M{"$lte": util.ObjToString(mapInfo["lteid"])}}
+		uq := bson.M{"gtid": util.ObjToString(mapInfo["gtid"]), "lteid": util.ObjToString(mapInfo["lteid"])}
 		MgoB.Update("bidding_processing_ids", uq, bson.M{"$set": bson.M{"dataprocess": 7, "updatetime": time.Now().Unix()}}, false, true)
 	}
 	//领域标签处理的数据 id段
@@ -282,9 +281,6 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 			for _, k := range config.Conf.Serve.FieldS {
 				v1 := compare[k] //extract
 				v2 := tmp[k]     //bidding
-				if k == "project_startdate" || k == "project_completedate" {
-					util.Debug(v1, v2)
-				}
 				if v2 == nil && v1 != nil {
 					update[k] = v1
 				} else if v2 != nil && v1 != nil {
@@ -297,13 +293,11 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 					} else if k == "city" || k == "district" {
 						update[k] = ""
 					} else {
-						if k == "project_startdate" || k == "project_completedate" {
-							util.Debug(v1, v2)
-						}
 						del[k] = 1
 					}
 				}
 			}
+			// 附件重采，数据同步时不更新判重标识
 			if util.IntAll(compare["repeat"]) == 1 {
 				update["extracttype"] = -1
 				update["dataprocess"] = 7
@@ -334,9 +328,9 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		//------------------对比结束
 
 		//处理key descript
-		if bkey == "" {
-			DealInfo(&tmp, &update)
-		}
+		//if bkey == "" {
+		//	DealInfo(&tmp, &update)
+		//}
 		// entidlist
 		extractMap := make(map[string]interface{})
 		if update["s_winner"] != "" {
@@ -379,8 +373,6 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 			}
 		}
 		if len(update) > 0 {
-			util.Debug(update)
-			util.Debug(del)
 			if len(del) > 0 {
 				bidUpdate = append(bidUpdate, []map[string]interface{}{{
 					"_id": tmp["_id"],

monitor/common.toml

@@ -1,6 +1,6 @@
 [serve]
 fileWarn = 10000
-pyWarn = 50000
+pyWarn = 1000
 
 [db]
 [db.mongoB]

monitor/go.mod

@@ -8,5 +8,5 @@ require (
 	github.com/spf13/cobra v1.6.1
 	go.mongodb.org/mongo-driver v1.11.0
 	go.uber.org/zap v1.23.0
-	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67 // indirect
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67
 )

monitor/go.sum

@@ -3,12 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
-github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
-github.com/antlabs/strsim v0.0.3 h1:J9AHxnybJZHKBoxeup1VZNWt3ST8QD+ieDJsm/nEpRo=
-github.com/antlabs/strsim v0.0.3/go.mod h1:bIcymn+2jtt01korFun0bs8PsYZeQa82aHoYMi7cm30=
 github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -19,20 +15,14 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
 github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
-github.com/donnie4w/go-logger v0.0.0-20170827050443-4740c51383f4 h1:T9PR91sjTtrA1HmZB4G+M7OLCelch0f6rIEY7Mm1T4U=
-github.com/donnie4w/go-logger v0.0.0-20170827050443-4740c51383f4/go.mod h1:L7S4x0R7vv3xoOhGuyAJyCO2MYzWOpccM4Isn8jIUgY=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
-github.com/go-ego/gse v0.70.2 h1:y2UMOHJMtI+0b2GjxTtQfKON5DMmlyX1hOQHTo8UVVs=
-github.com/go-ego/gse v0.70.2/go.mod h1:kesekpZfcFQ/kwd9b27VZHUOH5dQUjaaQUZ4OGt4Hj4=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -49,16 +39,15 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
@@ -76,7 +65,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
@@ -106,9 +94,6 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/vcaesar/cedar v0.20.1 h1:cDOmYWdprO7ZW8cngJrDi8Zivnscj9dA/y8Y+2SB1P0=
-github.com/vcaesar/cedar v0.20.1/go.mod h1:iMDweyuW76RvSrCkQeZeQk4iCbshiPzcCvcGCtpM7iI=
-github.com/vcaesar/tt v0.20.0/go.mod h1:GHPxQYhn+7OgKakRusH7KJ0M5MhywoeLb8Fcffs/Gtg=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
@@ -153,9 +138,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -213,12 +196,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/olivere/elastic.v2 v2.0.61 h1:7cpl3MW8ysa4GYFBXklpo5mspe4NK0rpZTdyZ+QcD4U=
-gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
@@ -227,11 +207,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20221205033056-885644941005 h1:AEEi+8ao9pTVqPIh6uVvjxBby/i43fFj7DwVo+feDAE=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20221205033056-885644941005/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
 jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67 h1:5rabE3IQRjI/NXfgJyquRTMdzSy7SfufFhmREixLZXg=
 jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
-jygit.jydev.jianyu360.cn/data_processing/field_info_tag v1.1.1 h1:ouJx+s77O8N69RALYyBkOPNyVeJ8WZL+2Ot/eZSEHTw=
-jygit.jydev.jianyu360.cn/data_processing/field_info_tag v1.1.1/go.mod h1:i9NKOqhWmcDl0Cl5qHBBBx46Da0Qzys7L0J+3my/Hbw=

monitor/main.go

@@ -4,25 +4,23 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
-	"monitor/config"
-	"net/http"
-	"time"
-
 	"github.com/robfig/cron"
 	"github.com/spf13/cobra"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
-	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"io"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"monitor/config"
+	"net/http"
+	"time"
 )
 
 var (
 	WebUrl  = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=45962efc-ca87-4996-9ffa-08bf6608ab7a"
 	WebUrl1 = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=1594c58d-c279-4fdc-8d44-c201160e3731"
 
-	WarningStr  = "数据采集bidding_file表数据，已累计%d条数据未处理"
-	WarningStr1 = "标的物等字段识别提醒，bidding表数据已积累%d条未处理"
+	WarningStr  = "数据采集bidding_file表数据，已累计%d条数据待处理"
+	WarningStr1 = "标的物等字段识别提醒，bidding_normal表数据已积累%d条待处理"
 	WarningStr2 = "项目合并表数据，超过一个1小时(查询时间点: %d)未有合并新数据"
 )
 
@@ -76,11 +74,7 @@ func taskFile() {
 }
 
 func taskPy() {
-	info, _ := MgoB.Find("bidding_processing_ids", bson.M{"dataprocess": 1}, nil, bson.M{"count": 1, "dataprocess": 1}, false, -1, -1)
-	count := 0
-	for _, m := range *info {
-		count += util.IntAll(m["count"])
-	}
+	count := MgoB.Count("bidding_nomal", bson.M{"bid_completetime": bson.M{"$exists": false}})
 	if count > config.Conf.Serve.PyWarn {
 		SendMsg(fmt.Sprintf(WarningStr1, count))
 	} else {

processing_ids/go.mod

@@ -3,8 +3,8 @@ module processing_ids
 go 1.16
 
 require (
-	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061
 	github.com/robfig/cron v1.2.0
-	go.mongodb.org/mongo-driver v1.11.0 // indirect
-	go.uber.org/zap v1.23.0
+	go.mongodb.org/mongo-driver v1.11.0
+	go.uber.org/zap v1.23.0 // indirect
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061
 )

processing_ids/go.sum

@@ -1,25 +1,59 @@
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
 github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -27,16 +61,23 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
 github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -56,10 +97,12 @@ github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgk
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.mongodb.org/mongo-driver v1.11.0 h1:FZKhBSTydeuffHj9CBjXlR8vQLee1cQyTWYPA6/tqiE=
 go.mongodb.org/mongo-driver v1.11.0/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
@@ -71,35 +114,55 @@ go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -107,6 +170,25 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -114,9 +196,13 @@ gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061 h1:xfmoVsDEqVv1XzAVxgGEDi+W9ojHBJc6OmTbg3b1tP0=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

processing_ids/main.go

@@ -5,9 +5,7 @@ import (
 	"fmt"
 	"github.com/robfig/cron"
 	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"go.uber.org/zap"
-	"io/ioutil"
+	"io"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
@@ -19,19 +17,24 @@ var (
 	MgoBid          *mongodb.MongodbSim
 	lastId, startId string
 
-	mail_to  = "zhangjinkun@topnet.net.cn,wangjianghan@topnet.net.cn,maxiaoshan@topnet.net.cn,zhengkun@topnet.net.cn"
+	mail_to  = "wangjianghan@topnet.net.cn,zhangjinkun@topnet.net.cn,maxiaoshan@topnet.net.cn,zhengkun@topnet.net.cn"
 	mail_api = "http://172.17.145.179:19281/_send/_mail"
+
+	skip        = 0
+	MgoBulkSize = 200
 )
 
 func init() {
 	MgoBid = &mongodb.MongodbSim{
-		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
+		MongodbAddr: "172.17.189.140:27080,172.17.189.141:27081",
 		DbName:      "qfw",
 		Size:        5,
 		UserName:    "SJZY_RWESBid_Other",
 		Password:    "SJZY@O17t8herB3B",
 	}
 	MgoBid.InitPool()
+
+	log.InitLog()
 }
 
 func main() {
@@ -46,7 +49,8 @@ func main() {
 
 func TimeTask() {
 	crn := cron.New()
-	cronstr := "0 */5 * * * ?" // 每5min执行一次
+	//cronstr := "0 */1 * * * ?" // 每2min执行一次
+	cronstr := "*/20 * * * * ?" // 每20s执行一次
 	ct := 0
 	_ = crn.AddFunc(cronstr, func() {
 		ct += 1
@@ -60,28 +64,30 @@ func TimeTask() {
 }
 
 func taskinfo() {
-	currentTime := time.Now()
-	m, _ := time.ParseDuration("-5m") // 5分钟之前
-	rtime := currentTime.Add(m)
-	eid := primitive.NewObjectIDFromTimestamp(rtime)
-	qfid := bson.M{"_id": bson.M{"$lte": eid}}
-	info, _ := MgoBid.Find("bidding", qfid, `{"_id": -1}`, `{"_id": 1}`, true, -1, -1)
-	if info != nil {
+	//currentTime := time.Now()
+	//m, _ := time.ParseDuration("5m") // 5分钟之前
+	//rtime := currentTime.Add(m)
+	//eid := primitive.NewObjectIDFromTimestamp(rtime)
+	//qfid := bson.M{"_id": bson.M{"$lte": eid}}
+	info, _ := MgoBid.Find("bidding", nil, `{"_id": -1}`, `{"_id": 1}`, true, -1, -1)
+	if len(*info) > 0 && lastId != mongodb.BsonIdToSId((*info)[0]["_id"]) {
 		lastId = mongodb.BsonIdToSId((*info)[0]["_id"])
 	} else {
-		sendMail("bidding表id查询失败")
+		if skip >= 30 {
+			sendMail(fmt.Sprintf("bidding表id段数据查询失败，%s", lastId))
+			skip = 0
+		} else {
+			skip++
+		}
+		log.Debug(fmt.Sprintf("skip: %d, lastid: %s", skip, lastId))
 		return
 	}
+	skip = 0
 	info1, _ := MgoBid.Find("bidding_processing_ids", `{"dataprocess": 0}`, `{"_id": -1}`, nil, false, -1, 2)
 	if len(*info1) > 1 {
 		startId = util.ObjToString((*info1)[0]["gtid"])
 		q := bson.M{"_id": bson.M{"$gt": mongodb.StringTOBsonId(startId), "$lte": mongodb.StringTOBsonId(lastId)}}
 		count := MgoBid.Count("bidding", q)
-		ids := fmt.Sprintf("%s-%s", startId, lastId)
-		if count <= 0 {
-			sendMail(fmt.Sprintf("bidding表id段数据查询失败，%s", ids))
-			return
-		}
 		if count > 10000 {
 			startId = util.ObjToString((*info1)[0]["lteid"])
 			q = bson.M{"_id": bson.M{"$gt": mongodb.StringTOBsonId(startId), "$lte": mongodb.StringTOBsonId(lastId)}}
@@ -95,7 +101,7 @@ func taskinfo() {
 			save["dataprocess"] = 0
 			save["createtime"] = now
 			save["updatetime"] = now
-			MgoBid.Save("bidding_processing_ids", save)
+			saveFuc(save, util.Int64All((*info1)[0]["last_autoid"]))
 			startId = lastId
 		} else {
 			update := make(map[string]interface{})
@@ -103,17 +109,13 @@ func taskinfo() {
 			startId = util.ObjToString((*info1)[0]["gtid"])
 			update["count"] = count
 			update["updatetime"] = time.Now().Unix()
-			MgoBid.UpdateById("bidding_processing_ids", (*info1)[0]["_id"], map[string]interface{}{"$set": update})
+			updateFuc((*info1)[0]["_id"], update, util.Int64All((*info1)[0]["last_autoid"]), util.ObjToString((*info1)[0]["lteid"]))
 			startId = lastId
 		}
 	} else {
 		q := bson.M{"_id": bson.M{"$gt": mongodb.StringTOBsonId(startId), "$lte": mongodb.StringTOBsonId(lastId)}}
 		count := MgoBid.Count("bidding", q)
-		ids := fmt.Sprintf("%s-%s", startId, lastId)
-		if count <= 0 {
-			sendMail(fmt.Sprintf("bidding表id段数据查询失败，%s", ids))
-			return
-		}
+		info2, _ := MgoBid.Find("bidding_processing_ids", nil, `{"_id": -1}`, nil, true, -1, 1)
 		save := make(map[string]interface{})
 		save["gtid"] = startId
 		save["lteid"] = lastId
@@ -122,16 +124,77 @@ func taskinfo() {
 		save["dataprocess"] = 0
 		save["createtime"] = now
 		save["updatetime"] = now
-		MgoBid.Save("bidding_processing_ids", save)
+		saveFuc(save, util.Int64All((*info2)[0]["last_autoid"]))
 		startId = lastId
 	}
 }
 
+func saveFuc(save map[string]interface{}, autoid int64) {
+	conn := MgoBid.GetMgoConn()
+	defer MgoBid.DestoryMongoConn(conn)
+	gtid := util.ObjToString(save["gtid"])
+	lteid := util.ObjToString(save["lteid"])
+	q := bson.M{"_id": bson.M{"$gt": mongodb.StringTOBsonId(gtid), "$lte": mongodb.StringTOBsonId(lteid)}}
+	result := conn.DB(MgoBid.DbName).C("bidding").Find(q).Select(bson.M{"_id": 1}).Iter()
+	var bidUpdate [][]map[string]interface{}
+	for tmp := make(map[string]interface{}); result.Next(tmp); {
+		autoid++
+		bidUpdate = append(bidUpdate, []map[string]interface{}{{
+			"_id": tmp["_id"],
+		},
+			{"$set": bson.M{"autoid": autoid}},
+		})
+
+		if len(bidUpdate) >= MgoBulkSize {
+			tmps := bidUpdate
+			MgoBid.UpdateBulk("bidding", tmps...)
+			bidUpdate = [][]map[string]interface{}{}
+		}
+	}
+	if len(bidUpdate) > 0 {
+		tmps := bidUpdate
+		MgoBid.UpdateBulk("bidding", tmps...)
+		bidUpdate = [][]map[string]interface{}{}
+	}
+	save["last_autoid"] = autoid
+	MgoBid.Save("bidding_processing_ids", save)
+}
+
+func updateFuc(id interface{}, update map[string]interface{}, autoid int64, gtid string) {
+	conn := MgoBid.GetMgoConn()
+	defer MgoBid.DestoryMongoConn(conn)
+	lteid := util.ObjToString(update["lteid"])
+	q := bson.M{"_id": bson.M{"$gt": mongodb.StringTOBsonId(gtid), "$lte": mongodb.StringTOBsonId(lteid)}}
+	result := conn.DB(MgoBid.DbName).C("bidding").Find(q).Select(bson.M{"_id": 1}).Iter()
+	var bidUpdate [][]map[string]interface{}
+	for tmp := make(map[string]interface{}); result.Next(tmp); {
+		autoid++
+		bidUpdate = append(bidUpdate, []map[string]interface{}{{
+			"_id": tmp["_id"],
+		},
+			{"$set": bson.M{"autoid": autoid}},
+		})
+
+		if len(bidUpdate) >= MgoBulkSize {
+			tmps := bidUpdate
+			MgoBid.UpdateBulk("bidding", tmps...)
+			bidUpdate = [][]map[string]interface{}{}
+		}
+	}
+	if len(bidUpdate) > 0 {
+		tmps := bidUpdate
+		MgoBid.UpdateBulk("bidding", tmps...)
+		bidUpdate = [][]map[string]interface{}{}
+	}
+	update["last_autoid"] = autoid
+	MgoBid.UpdateById("bidding_processing_ids", id, map[string]interface{}{"$set": update})
+}
+
 func sendMail(content string) {
 	res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", mail_api, mail_to, "processing_ids-send-fail", content))
 	if err == nil {
 		defer res.Body.Close()
-		read, err := ioutil.ReadAll(res.Body)
-		log.Info("send mail ...", zap.String("r:", string(read)), zap.Any("err:", err))
+		read, _ := io.ReadAll(res.Body)
+		util.Debug("send mail ..." + string(read))
 	}
 }