xg

README.md

@@ -1,4 +1,4 @@
-## 领域化产品-数据处理项目
+## 产品-数据处理项目
 
 ###  fieldproject_com(公共数据处理模块)
 ####  org_data
@@ -7,30 +7,30 @@
 
 #### 处理逻辑
 + 医疗机构表信息处理，同步法人基本信息到法人表
-  + 医疗机构数据分为三部分信息：
-  + 1、凭安数据找到company_id直接同步更新
-  + 2、找
+    + 医疗机构数据分为三部分信息：
+    + 1、凭安数据找到company_id直接同步更新
+    + 2、找
 + 基于bidding表医疗领域标签，处理标讯数据的企业信息
-  + mark_id = 1，直接入法人库
-  + mark_id = 2，待处理的信息入法人库
-  + mark_id = 3，暂时不进法人库
-  + mark_id = 4, 进法人信息库
+    + mark_id = 1，直接入法人库
+    + mark_id = 2，待处理的信息入法人库
+    + mark_id = 3，暂时不进法人库
+    + mark_id = 4, 进法人信息库
 
 ### fieldproject_inc_data 增量数据
 + 配合任务调度平台定时发送udp任务
 + 标讯处理完之后，处理es领域字段、项目数据领域字段
-  + 
+    +
 
-### fieldproject_medical(医疗数据处理模块) 
+### fieldproject_medical(医疗数据处理模块)
 + data_preparation (基础信息~数据准备~)
-  + 1、企业产品信息  
-  + 2、医院相关信息  
-  + 3、产品相关整合  
-  + 4、代码表
+    + 1、企业产品信息
+    + 2、医院相关信息
+    + 3、产品相关整合
+    + 4、代码表
 
 + data_service (服务)
-  + 1、标的物相关信息  
-  + 2、相关服务
+    + 1、标的物相关信息
+    + 2、相关服务
 
 ###  proposed_project(拟在建数据项目合并)
 ####  拟在建数据处理分为三个阶段
@@ -41,15 +41,40 @@
 ####  拟在建关联
 1. 建立关联关系，拟建项目与在建项目
 2. 关联数据更新到tidb库
-   + 基本信息表的跟进数量、项目阶段
-   + 跟进记录表的信息
+    + 基本信息表的跟进数量、项目阶段
+    + 跟进记录表的信息
 3. 关联数据增量处理
-   + 拟建项目增量处理
-   + 在建项目增量数据处理
+    + 拟建项目增量处理
+    + 在建项目增量数据处理
 
 
 ### medical_project(医疗领域数据合并)
-
 #### 处理逻辑
-+ 
+
+###  tieta_data(铁塔数据)
+#### 铁塔数据处理
+1. 导出标讯数据
+2. 导出捏合数据(项目数据)
+3. 项目合并
+
+###  data_credible(标讯数据可信度)
+#### 标讯数据可信度分析
+1. 标注数据平台数据
+2. 旧版标注数据结构  
+   credible old -d dbname -c coll
+
+###  buyer_data(采购单位数据)
+#### 采购单位数据处理
+1. 标注数据平台数据
+2. 旧版标注数据结构
+
+
+
+
+
+
+
+
+
+
 

buyer_data/common.toml

@@ -0,0 +1,41 @@
+[serve]
+thread = 3
+
+[db]
+
+[db.mongo]
+addr = "192.168.3.71:29099"
+dbname = "wjh"
+coll = "buyer_enterprise"
+saveColl = "buyer_directory"
+size = 5
+user = ""
+password = ""
+
+[db.mysql]
+addr = "192.168.3.14:4000"
+dbname = "global_common_data"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+
+[redis]
+addr = "192.168.3.207:2679"
+code = "buyer_data"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"

buyer_data/config/conf.go

@@ -0,0 +1,84 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	// Conf crocodile conf
+	Conf *conf
+)
+
+// Init Config
+func Init(conf string) {
+	_, err := toml.DecodeFile(conf, &Conf)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+}
+
+type conf struct {
+	Serve serve
+	DB    db
+	Log   log
+}
+
+type serve struct {
+	Thread int
+}
+
+// Log Config
+type log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type db struct {
+	Mongo mgo
+	Mysql mysql
+	Redis redis
+}
+
+type mgo struct {
+	Addr     string
+	Dbname   string
+	Coll     string
+	SaveColl string
+	Size     int
+	User     string
+	Password string
+}
+
+type mysql struct {
+	Addr     string
+	Dbname   string
+	Size     int
+	User     string
+	Password string
+}
+
+type redis struct {
+	Addr string
+	Code string
+}
+
+type duration struct {
+	time.Duration
+}
+
+// UnmarshalText parse 10s to time.Time
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}

buyer_data/config/conf_test.go

@@ -0,0 +1,66 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var confs = `# log
+[log]
+logpath = ""
+maxsize = 10
+compress = true
+maxage =  7
+maxbackups = 10
+loglevel  = "info"
+format = "text"
+
+[db]
+[db.mongo]
+addr = "192.168.3.207:27092"
+dbname = "qfw"
+size = 10
+user = ""
+password = ""
+[db.mongo1]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+size = 5
+user = ""
+password = ""
+[db.mysql]
+addr = "192.168.3.217:4000"
+dbnameBasic = "global_common_data"
+dbnameMedical = "medical_fileld_data"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+
+[db.es]
+addr = "http://192.168.3.206:9800"
+size = 5
+indexm = "medical_institution_v1"
+typem = "medical_institution"
+indexs = "supplier_product_v1"
+types = "supplier_product"
+
+[db.es.fieldarrm]
+id = "int"
+company_id = "string"
+mi_name = "string"
+alias = "string"
+area_code = "string"
+level_code = "string"
+mi_type_code = "string"
+business_type = "int"
+sdequipment = "string"
+`
+
+func TestInit(t *testing.T) {
+	testfile := "/tmp/crocodile.toml"
+	ioutil.WriteFile(testfile, []byte(confs), 0644)
+	Init(testfile)
+	t.Logf("%+v", Conf.DB)
+	os.Remove(testfile)
+}

buyer_data/go.mod

@@ -0,0 +1,37 @@
+module buyer_data
+
+go 1.20
+
+require (
+	github.com/BurntSushi/toml v1.3.2
+	github.com/google/uuid v1.1.2
+	github.com/spf13/cobra v1.7.0
+	go.mongodb.org/mongo-driver v1.12.1
+	go.uber.org/zap v1.22.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/gomodule/redigo v1.8.9 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+)

buyer_data/go.sum

@@ -0,0 +1,226 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE=
+go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c h1:fgH3lMOi2jG3KCejzC3d0otkJB6mJ9eaq1HecMoHG4c=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

buyer_data/init.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"buyer_data/config"
+	"fmt"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
+	"os"
+)
+
+var (
+	MongoTool *mongodb.MongodbSim
+	MysqlTool *mysqldb.Mysql
+
+	Coll string
+
+	saveSize   int
+	savePool   chan map[string]interface{}
+	saveSp     chan bool
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+)
+
+func init() {
+	config.Init("./common.toml")
+	InitLog()
+	InitMgo()
+
+	saveSize = 200
+	savePool = make(chan map[string]interface{}, 5000)
+	saveSp = make(chan bool, 2)
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 2)
+}
+
+func InitMgo() {
+	MongoTool = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.Mongo.Addr,
+		Size:        config.Conf.DB.Mongo.Size,
+		DbName:      config.Conf.DB.Mongo.Dbname,
+		UserName:    config.Conf.DB.Mongo.User,
+		Password:    config.Conf.DB.Mongo.Password,
+	}
+	MongoTool.InitPool()
+}
+
+func InitMysql() {
+	dbcfg := config.Conf.DB.Mysql
+	MysqlTool = &mysqldb.Mysql{
+		Address:  dbcfg.Addr,
+		DBName:   dbcfg.Dbname,
+		UserName: dbcfg.User,
+		PassWord: dbcfg.Password,
+	}
+	MysqlTool.Init()
+}
+
+func InitLog() {
+	logcfg := config.Conf.Log
+
+	err := log.InitLog(
+		log.Path(logcfg.LogPath),
+		log.Level(logcfg.LogLevel),
+		log.Compress(logcfg.Compress),
+		log.MaxSize(logcfg.MaxSize),
+		log.MaxBackups(logcfg.MaxBackups),
+		log.MaxAge(logcfg.MaxAge),
+		log.Format(logcfg.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+}

buyer_data/main.go

@@ -0,0 +1,174 @@
+package main
+
+import (
+	"buyer_data/config"
+	"fmt"
+	"github.com/spf13/cobra"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+	"time"
+)
+
+func main() {
+	rootCmd := &cobra.Command{Use: "my cmd"}
+	rootCmd.AddCommand(buyerEnt())      // buyer_enterprise
+	rootCmd.AddCommand(buyerErr())      // buyer_err
+	rootCmd.AddCommand(buyerTidb())     // dws_f_ent_baseinfo
+	rootCmd.AddCommand(deduplication()) // 去重
+	rootCmd.AddCommand(buyerPy())       // buyer_detail_1019
+	rootCmd.AddCommand(buyerTask())     // reliability不存在的数据
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println("rootCmd.Execute failed", err.Error())
+	}
+
+	c := make(chan bool, 1)
+	<-c
+}
+
+func buyerEnt() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "buyer-mongo",
+		Short: "Start dispose buyer data",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			go SaveMethod()
+			taskInfo1()
+		},
+	}
+	return cmdClient
+}
+
+func buyerErr() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "buyer-err",
+		Short: "Start dispose buyer data",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			go SaveMethod()
+			taskInfo2()
+		},
+	}
+	return cmdClient
+}
+
+func buyerTidb() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "buyer-tidb",
+		Short: "Start dispose buyer-tidb data",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitMysql()
+
+			go SaveMethod()
+			go updateMethod()
+			taskMysql()
+		},
+	}
+	return cmdClient
+}
+
+func deduplication() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "buyer-dep",
+		Short: "Start deduplication data",
+		Run: func(cmd *cobra.Command, args []string) {
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Code, config.Conf.DB.Redis.Addr), 8)
+			taskInfo5()
+		},
+	}
+	return cmdClient
+}
+
+func buyerPy() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "buyer-py",
+		Short: "Start dispose buyer-python data",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			go SaveMethod()
+			go updateMethod()
+			taskInfo4()
+		},
+	}
+	return cmdClient
+}
+
+func buyerTask() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "buyer-err",
+		Short: "Start dispose buyer data",
+		Run: func(cmd *cobra.Command, args []string) {
+			go updateMethod()
+			taskInfo3()
+		},
+	}
+	return cmdClient
+}
+
+func SaveMethod() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MongoTool.SaveBulk(config.Conf.DB.Mongo.SaveColl, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MongoTool.SaveBulk(config.Conf.DB.Mongo.SaveColl, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func updateMethod() {
+	arru := make([][]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoTool.UpSertBulk(config.Conf.DB.Mongo.SaveColl, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1 * time.Second):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoTool.UpSertBulk(config.Conf.DB.Mongo.SaveColl, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

buyer_data/task.go

@@ -0,0 +1,456 @@
+package main
+
+import (
+	"buyer_data/config"
+	"database/sql"
+	"fmt"
+	"github.com/google/uuid"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
+	"strings"
+	"sync"
+	"time"
+)
+
+func taskInfo1() {
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	field := bson.M{"buyer_name": 1, "buyerclass": 1, "area": 1, "city": 1, "district": 1}
+	query := sess.DB(config.Conf.DB.Mongo.Dbname).C("buyer_enterprise").Find(nil).Select(field).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%2000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			save := make(map[string]interface{})
+			name := util.ObjToString(tmp["buyer_name"])
+			info, _ := MongoTool.FindOne("qyxy_std", bson.M{"company_name": name})
+			if len(*info) > 0 {
+				save["name"] = name
+				save["name_id"] = util.ObjToString((*info)["_id"])
+				save["name_id_source"] = 1
+				save["type"] = util.ObjToString((*info)["company_type_old"])
+				save["buyerclass"] = util.ObjToString(tmp["buyerclass"])
+				if area := util.ObjToString(tmp["area"]); area != "" {
+					save["area"] = area
+					if city := util.ObjToString(tmp["city"]); city != "" {
+						save["city"] = city
+					}
+					if district := util.ObjToString(tmp["district"]); district != "" {
+						save["city"] = district
+					}
+				} else {
+					if area = util.ObjToString((*info)["company_area"]); area != "" {
+						save["area"] = area
+						if city := util.ObjToString((*info)["company_city"]); city != "" {
+							save["city"] = city
+						}
+						if district := util.ObjToString((*info)["company_district"]); district != "" {
+							save["city"] = district
+						}
+					}
+				}
+				save["reliability"] = 1
+				save["legal_person"] = util.ObjToString((*info)["legal_person"])
+				if arr := hisNameFuc(*info); arr != nil && len(arr) > 0 {
+					save["historyname"] = arr
+				}
+				save["status"] = util.ObjToString((*info)["company_status"])
+				save["comeintime"] = time.Now().Unix()
+			} else {
+				save["name"] = name
+				save["name_id"] = strings.ReplaceAll(uuid.New().String(), "-", "")
+				save["name_id_source"] = 4
+				save["comeintime"] = time.Now().Unix()
+			}
+			savePool <- save
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+}
+
+func taskInfo2() {
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	query := sess.DB(config.Conf.DB.Mongo.Dbname).C("buyer_err").Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%2000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			save := make(map[string]interface{})
+			name := util.ObjToString(tmp["name"])
+			info, _ := MongoTool.FindOne("qyxy_std", bson.M{"company_name": name})
+			if len(*info) > 0 {
+				save["name"] = name
+				save["name_id"] = util.ObjToString((*info)["_id"])
+				save["name_id_source"] = 1
+				save["type"] = util.ObjToString((*info)["company_type_old"])
+				save["buyerclass"] = util.ObjToString(tmp["buyerclass"])
+				if area := util.ObjToString(tmp["area"]); area != "" {
+					save["area"] = area
+					if city := util.ObjToString(tmp["city"]); city != "" {
+						save["city"] = city
+					}
+					if district := util.ObjToString(tmp["district"]); district != "" {
+						save["city"] = district
+					}
+				} else {
+					if area = util.ObjToString((*info)["company_area"]); area != "" {
+						save["area"] = area
+						if city := util.ObjToString((*info)["company_city"]); city != "" {
+							save["city"] = city
+						}
+						if district := util.ObjToString((*info)["company_district"]); district != "" {
+							save["city"] = district
+						}
+					}
+				}
+				save["reliability"] = 1
+				save["legal_person"] = util.ObjToString((*info)["legal_person"])
+				if arr := hisNameFuc(*info); arr != nil && len(arr) > 0 {
+					save["historyname"] = arr
+				}
+				save["status"] = util.ObjToString((*info)["company_status"])
+				save["comeintime"] = time.Now().Unix()
+			} else {
+				save["name"] = name
+				save["name_id"] = strings.ReplaceAll(uuid.New().String(), "-", "")
+				save["name_id_source"] = 4
+				save["comeintime"] = time.Now().Unix()
+			}
+			savePool <- save
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+}
+
+func hisNameFuc(tmp map[string]interface{}) []string {
+	var nameArr []string
+	if names, ok := tmp["history_names"].([]interface{}); ok && len(names) > 0 {
+		for _, n := range names {
+			nameArr = append(nameArr, util.ObjToString(n))
+		}
+	}
+	if hisname := util.ObjToString(tmp["history_name"]); hisname != "" {
+		for _, s := range strings.Split(hisname, ",") {
+			if !strings.Contains(strings.Join(nameArr, ","), s) {
+				nameArr = append(nameArr, s)
+			}
+		}
+	}
+	return nameArr
+}
+
+func taskMysql() {
+	pool := make(chan bool, 2) //控制线程数
+	wg := &sync.WaitGroup{}
+
+	finalId := 0
+	lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT id FROM %s where identity_type = 1 ORDER BY id DESC LIMIT 1 ", "dws_f_ent_baseinfo"))
+	if len(*lastInfo) > 0 {
+		finalId = util.IntAll((*lastInfo)[0]["id"])
+	}
+	log.Info("查询最后id---", zap.Int("finally id: ", finalId))
+	lastid, count := 0, 0
+	for {
+		log.Info("重新查询，lastid---", zap.Int("lastid: ", lastid))
+		q := fmt.Sprintf("SELECT * FROM %s WHERE id > %d and identity_type = 1 ORDER BY id ASC limit 10000", "dws_f_ent_baseinfo", lastid)
+		var stmtOut *sql.Stmt
+		var tx *sql.Tx
+		var err error
+		if tx == nil {
+			stmtOut, err = MysqlTool.DB.Prepare(q)
+		} else {
+			stmtOut, err = tx.Prepare(q)
+		}
+		rows, err := stmtOut.Query()
+		if err != nil {
+			log.Error("mysql query err ", zap.Error(err))
+		}
+		columns, err := rows.Columns()
+		if finalId == lastid {
+			log.Info("----finish-----", zap.Int("count: ", count))
+			break
+		}
+		for rows.Next() {
+			scanArgs := make([]interface{}, len(columns))
+			values := make([]interface{}, len(columns))
+			ret := make(map[string]interface{})
+			for k := range values {
+				scanArgs[k] = &values[k]
+			}
+			err = rows.Scan(scanArgs...)
+			if err != nil {
+				log.Error("mysql scan err ", zap.Error(err))
+				break
+			}
+			for i, col := range values {
+				if col == nil {
+					ret[columns[i]] = nil
+				} else {
+					switch val := (*scanArgs[i].(*interface{})).(type) {
+					case byte:
+						ret[columns[i]] = val
+						break
+					case []byte:
+						v := string(val)
+						switch v {
+						case "\x00": // 处理数据类型为bit的情况
+							ret[columns[i]] = 0
+						case "\x01": // 处理数据类型为bit的情况
+							ret[columns[i]] = 1
+						default:
+							ret[columns[i]] = v
+							break
+						}
+						break
+					case time.Time:
+						if val.IsZero() {
+							ret[columns[i]] = nil
+						} else {
+							ret[columns[i]] = val.Format("2006-01-02 15:04:05")
+						}
+						break
+					default:
+						ret[columns[i]] = val
+					}
+				}
+			}
+			lastid = util.IntAll(ret["id"])
+			count++
+			if count%2000 == 0 {
+				log.Info("current----", zap.Int("count: ", count), zap.Int("lastid: ", lastid))
+			}
+			pool <- true
+			wg.Add(1)
+			go func(tmp map[string]interface{}) {
+				defer func() {
+					<-pool
+					wg.Done()
+				}()
+				name := util.ObjToString(tmp["name"])
+				save := make(map[string]interface{})
+				if cid := util.ObjToString(tmp["company_id"]); cid != "" {
+					bfo, _ := MongoTool.FindOne(config.Conf.DB.Mongo.SaveColl, bson.M{"name_id": cid})
+					if len(*bfo) > 0 {
+						return
+					}
+					info, _ := MongoTool.FindOne("qyxy_std", bson.M{"company_name": name})
+					save["name"] = name
+					save["name_id"] = cid
+					save["name_id_source"] = 1
+					save["type"] = util.ObjToString((*info)["company_type_old"])
+					if area := util.ObjToString((*info)["company_area"]); area != "" {
+						save["area"] = area
+						if city := util.ObjToString((*info)["company_city"]); city != "" {
+							save["city"] = city
+						}
+						if district := util.ObjToString((*info)["company_district"]); district != "" {
+							save["city"] = district
+						}
+					}
+					save["reliability"] = 1
+					save["legal_person"] = util.ObjToString((*info)["legal_person"])
+					if arr := hisNameFuc(*info); arr != nil && len(arr) > 0 {
+						save["historyname"] = arr
+					}
+					save["status"] = util.ObjToString((*info)["company_status"])
+					updatePool <- []map[string]interface{}{
+						{"name_id": cid},
+						{"$set": save},
+					}
+				} else {
+					save["name"] = name
+					save["name_id"] = strings.ReplaceAll(uuid.New().String(), "-", "")
+					save["name_id_source"] = 4
+					save["comeintime"] = time.Now().Unix()
+					savePool <- save
+				}
+			}(ret)
+			ret = make(map[string]interface{})
+		}
+		_ = rows.Close()
+		stmtOut.Close()
+		wg.Wait()
+	}
+	log.Info(fmt.Sprintf("over --- %d", count))
+}
+
+func taskInfo3() {
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	f := bson.M{"reliability": nil}
+	query := sess.DB(config.Conf.DB.Mongo.Dbname).C(config.Conf.DB.Mongo.SaveColl).Find(f).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%2000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			save := make(map[string]interface{})
+			name := util.ObjToString(tmp["buyer_name"])
+			info, _ := MongoTool.FindOne("qyxy_std", bson.M{"company_name": name})
+			if len(*info) > 0 {
+				save["name"] = name
+				save["name_id"] = util.ObjToString((*info)["_id"])
+				save["name_id_source"] = 1
+				save["type"] = util.ObjToString((*info)["company_type_old"])
+				save["buyerclass"] = util.ObjToString(tmp["buyerclass"])
+				if area := util.ObjToString(tmp["area"]); area != "" {
+					save["area"] = area
+					if city := util.ObjToString(tmp["city"]); city != "" {
+						save["city"] = city
+					}
+					if district := util.ObjToString(tmp["district"]); district != "" {
+						save["city"] = district
+					}
+				} else {
+					if area = util.ObjToString((*info)["company_area"]); area != "" {
+						save["area"] = area
+						if city := util.ObjToString((*info)["company_city"]); city != "" {
+							save["city"] = city
+						}
+						if district := util.ObjToString((*info)["company_district"]); district != "" {
+							save["city"] = district
+						}
+					}
+				}
+				save["reliability"] = 1
+				save["legal_person"] = util.ObjToString((*info)["legal_person"])
+				if arr := hisNameFuc(tmp); arr != nil && len(arr) > 0 {
+					save["historyname"] = arr
+				}
+				save["status"] = util.ObjToString((*info)["company_status"])
+				save["comeintime"] = time.Now().Unix()
+			} else {
+				save["name"] = name
+				save["name_id"] = strings.ReplaceAll(uuid.New().String(), "-", "")
+				save["name_id_source"] = 4
+				save["comeintime"] = time.Now().Unix()
+			}
+			savePool <- save
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+}
+
+func taskInfo4() {
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	query := sess.DB(config.Conf.DB.Mongo.Dbname).C("buyer_detail_1019").Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%20000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			name := util.ObjToString(tmp["accurate_entity_name"])
+			info, _ := MongoTool.FindOne(config.Conf.DB.Mongo.SaveColl, bson.M{"name": name})
+			if len(*info) > 0 {
+				update := make(map[string]interface{})
+				update["reliability"] = 0
+				update["legal_person"] = util.ObjToString(tmp["fddbr"])
+				update["status"] = util.ObjToString((*info)["company_status"])
+				updatePool <- []map[string]interface{}{
+					{"_id": (*info)["_id"]},
+					{"$set": update},
+				}
+			} else {
+				save := make(map[string]interface{})
+				save["name"] = name
+				save["name_id"] = strings.ReplaceAll(uuid.New().String(), "-", "")
+				save["name_id_source"] = 4
+				save["reliability"] = 0
+				save["legal_person"] = util.ObjToString(tmp["fddbr"])
+				save["status"] = util.ObjToString((*info)["company_status"])
+				save["comeintime"] = time.Now().Unix()
+				savePool <- save
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	log.Info(fmt.Sprintf("over --- %d", count))
+}
+
+func taskInfo5() {
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	query := sess.DB(config.Conf.DB.Mongo.Dbname).C(config.Conf.DB.Mongo.SaveColl).Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%20000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			name := util.ObjToString(tmp["name"])
+			if ok, _ := redis.Exists(config.Conf.DB.Redis.Code, name); ok {
+				MongoTool.UpdateById(config.Conf.DB.Mongo.SaveColl, tmp["_id"], bson.M{"$set": bson.M{"del": true}})
+			} else {
+				redis.PutCKV(config.Conf.DB.Redis.Code, name, 1)
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	log.Info(fmt.Sprintf("over --- %d", count))
+}

customer_project/common.toml

@@ -0,0 +1,45 @@
+[serve]
+locport = ":1782"
+jyhref = "https://www.jianyu360.cn/article/content/%s.html"
+rpcserver = ":7979"
+
+[db]
+
+[db.mysql]
+addr = "192.168.3.14:4000"
+dbname = "bi_service"
+coll = "customer_data_dev"
+pcoll = "customer_data_yys_project_dev"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+[db.ent]
+addr = "192.168.3.14:4000"
+dbname = "global_common_data"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+
+[db.es]
+addr = "http://192.168.3.241:9205"
+user = ""
+password = ""
+size = 5
+index = "ttbid"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"

customer_project/config/conf.go

@@ -0,0 +1,92 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	// Conf crocodile conf
+	Conf *conf
+)
+
+// Init Config
+func Init(conf string) {
+	_, err := toml.DecodeFile(conf, &Conf)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+}
+
+type conf struct {
+	Serve serve
+	DB    db
+	Log   log
+}
+
+type serve struct {
+	JyHref    string
+	LocPort   string
+	RpcServer string
+}
+
+// Log Config
+type log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type db struct {
+	Mongo mgo
+	Mysql mysql
+	Ent   mysql
+	Es    es
+}
+
+type mgo struct {
+	Addr     string
+	Dbname   string
+	Coll     string
+	Pcoll    string
+	Size     int
+	User     string
+	Password string
+}
+
+type mysql struct {
+	Addr     string
+	Dbname   string
+	Coll     string
+	Pcoll    string
+	Size     int
+	User     string
+	Password string
+}
+
+type es struct {
+	Addr     string
+	User     string
+	Password string
+	Size     int
+	Index    string
+}
+
+type duration struct {
+	time.Duration
+}
+
+// UnmarshalText parse 10s to time.Time
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}

customer_project/config/conf_test.go

@@ -0,0 +1,66 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var confs = `# log
+[log]
+logpath = ""
+maxsize = 10
+compress = true
+maxage =  7
+maxbackups = 10
+loglevel  = "info"
+format = "text"
+
+[db]
+[db.mongo]
+addr = "192.168.3.207:27092"
+dbname = "qfw"
+size = 10
+user = ""
+password = ""
+[db.mongo1]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+size = 5
+user = ""
+password = ""
+[db.mysql]
+addr = "192.168.3.217:4000"
+dbnameBasic = "global_common_data"
+dbnameMedical = "medical_fileld_data"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+
+[db.es]
+addr = "http://192.168.3.206:9800"
+size = 5
+indexm = "medical_institution_v1"
+typem = "medical_institution"
+indexs = "supplier_product_v1"
+types = "supplier_product"
+
+[db.es.fieldarrm]
+id = "int"
+company_id = "string"
+mi_name = "string"
+alias = "string"
+area_code = "string"
+level_code = "string"
+mi_type_code = "string"
+business_type = "int"
+sdequipment = "string"
+`
+
+func TestInit(t *testing.T) {
+	testfile := "/tmp/crocodile.toml"
+	ioutil.WriteFile(testfile, []byte(confs), 0644)
+	Init(testfile)
+	t.Logf("%+v", Conf.DB)
+	os.Remove(testfile)
+}

customer_project/go.mod

@@ -0,0 +1,38 @@
+module customer_project
+
+go 1.20
+
+require (
+	app.yhyue.com/moapp/jybase v0.0.0-20231222072535-dca21a393e0e
+	github.com/BurntSushi/toml v1.3.2
+	github.com/google/uuid v1.4.0
+	github.com/robfig/cron/v3 v3.0.1
+	go.mongodb.org/mongo-driver v1.13.0
+	go.uber.org/zap v1.26.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+)

customer_project/go.sum

@@ -0,0 +1,427 @@
+app.yhyue.com/moapp/esv1 v0.0.0-20220414031211-3da4123e648d/go.mod h1:91/lSD/hS+ckMVP3WdidRzDhC60lLMdyce9QHy0cSMA=
+app.yhyue.com/moapp/jybase v0.0.0-20231222072535-dca21a393e0e h1:3SOZl1ZdYiC/Midaf92FjtF77eNjUWDusLTn7xzoPXE=
+app.yhyue.com/moapp/jybase v0.0.0-20231222072535-dca21a393e0e/go.mod h1:fjaD11Z3FIk9EyfOpdKAMQNs7nPZCpT/qKIw1oVsb9w=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/RoaringBitmap/roaring v1.5.0/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/aws/aws-sdk-go v1.35.20/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coscms/tagfast v0.0.0-20150925144250-2b69b2496250/go.mod h1:zX8vynptAghuV/KG8BOZlDeo4DsTKWfBQ154RWlkay0=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/donnie4w/go-logger v0.0.0-20170827050443-4740c51383f4/go.mod h1:L7S4x0R7vv3xoOhGuyAJyCO2MYzWOpccM4Isn8jIUgY=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic v6.2.37+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
+github.com/olivere/elastic/v7 v7.0.22/go.mod h1:VDexNy9NjmtAkrjNoI7tImv7FR4tf5zUA3ickqu5Pc8=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
+go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
+go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.13.0 h1:67DgFFjYOCMWdtTEmKFpV3ffWlFnh+CYZ8ZS/tXWUfY=
+go.mongodb.org/mongo-driver v1.13.0/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
+go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220531201128-c960675eff93/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.0.5/go.mod h1:N1OIhHAIhx5SunkMGqWbGFVeh4yTNWKmMo1GOAsohLI=
+gorm.io/gorm v1.21.3/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c h1:fgH3lMOi2jG3KCejzC3d0otkJB6mJ9eaq1HecMoHG4c=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

customer_project/init.go

@@ -0,0 +1,291 @@
+package main
+
+import (
+	"customer_project/config"
+	"fmt"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
+	"os"
+	"sync"
+)
+
+var (
+	MongoTool           *mongodb.MongodbSim
+	MysqlTool, MysqlEnt *mysqldb.Mysql
+	Es                  *elastic.Elastic
+
+	findLock sync.Mutex
+	//mapPb, mapPn, mapPc map[string]*Key
+	mapEnt        map[int64]*EntMap
+	AllIdsMapLock sync.Mutex
+	AllIdsMap     map[string]*ID
+
+	//仅初始化使用
+	compareNoPass = map[string]bool{}
+	compareAB     = map[string]bool{}
+	compareAB2D   = map[string]bool{}
+	compareABD    = map[string]bool{}
+	compareAB2CD  = map[string]bool{}
+	compareABCD   = map[string]bool{}
+)
+
+type EntMap struct {
+	mapPb map[string]*Key
+	mapPn map[string]*Key
+	mapPc map[string]*Key
+}
+
+type ID struct {
+	Id   string
+	Lock sync.Mutex
+	P    *Project
+}
+
+type Key struct {
+	Arr  []string
+	Lock sync.Mutex
+}
+
+func init() {
+
+	wg = sync.WaitGroup{}
+	AllIdsMap = make(map[string]*ID, 5000000)
+	//mapPb = make(map[string]*Key, 1500000)
+	//mapPn = make(map[string]*Key, 5000000)
+	//mapPc = make(map[string]*Key, 5000000)
+	mapEnt = make(map[int64]*EntMap, 5000)
+
+	//加载项目数据
+	//---不能通过
+	vm := []string{"C", "D"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				compareNoPass[key] = true
+				//fmt.Println(key)
+			}
+		}
+	}
+	//fmt.Println("-------------------")
+
+	//三个元素一致 [AB][AB][AB]，分值最高
+	vm = []string{"A", "B"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				compareAB[key] = true
+				//fmt.Println(key)
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareAB))
+	//---至少两个一致，其他可能不存在
+	//[AB][AB][ABD]
+	//[AB][ABD][AB]
+	vm = []string{"A", "B"}
+	vm2 := []string{"A", "B", "D"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 3; k++ {
+				key := vm[i] + vm[j] + vm2[k]
+				if !compareAB[key] {
+					compareAB2D[key] = true
+					//fmt.Println(key)
+
+				}
+			}
+		}
+	}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 3; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm2[j] + vm[k]
+				if !compareAB[key] {
+					compareAB2D[key] = true
+					//fmt.Println(key)
+
+				}
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareAB2D))
+	//---至少一个一致，其他可能不存在
+	//[ABD][ABD][ABD] //已经删除DDD
+	vm = []string{"A", "B", "D"}
+	for i := 0; i < 3; i++ {
+		for j := 0; j < 3; j++ {
+			for k := 0; k < 3; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] {
+					compareABD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareABD))
+
+	//[AB][ABCD][AB]
+	//[AB][AB][ABCD]
+	vm = []string{"A", "B"}
+	vm2 = []string{"A", "B", "C", "D"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 4; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm2[j] + vm[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
+					compareAB2CD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 4; k++ {
+				key := vm[i] + vm[j] + vm2[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] {
+					compareAB2CD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+	//fmt.Println("-------------------", len(compareAB2CD))
+	//[ABECD][ABECD][ABECD]  //已经删除[CD][CD][CD]   //这个要重点讨论
+	vm = []string{"A", "B", "C", "D"}
+	for i := 0; i < 4; i++ {
+		for j := 0; j < 4; j++ {
+			for k := 0; k < 4; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				if !compareAB[key] && !compareAB2D[key] && !compareABD[key] && !compareNoPass[key] && !compareAB2CD[key] {
+					compareABCD[key] = true
+					//fmt.Println(key)
+				}
+			}
+		}
+	}
+}
+
+func InitMysql() {
+	dbcfg := config.Conf.DB.Mysql
+	MysqlTool = &mysqldb.Mysql{
+		Address:  dbcfg.Addr,
+		DBName:   dbcfg.Dbname,
+		UserName: dbcfg.User,
+		PassWord: dbcfg.Password,
+	}
+	MysqlTool.Init()
+
+	MysqlEnt = &mysqldb.Mysql{
+		Address:  config.Conf.DB.Ent.Addr,
+		DBName:   config.Conf.DB.Ent.Dbname,
+		UserName: config.Conf.DB.Ent.User,
+		PassWord: config.Conf.DB.Ent.Password,
+	}
+	MysqlEnt.Init()
+}
+
+func InitLog() {
+	logcfg := config.Conf.Log
+
+	err := log.InitLog(
+		log.Path(logcfg.LogPath),
+		log.Level(logcfg.LogLevel),
+		log.Compress(logcfg.Compress),
+		log.MaxSize(logcfg.MaxSize),
+		log.MaxBackups(logcfg.MaxBackups),
+		log.MaxAge(logcfg.MaxAge),
+		log.Format(logcfg.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+type Project struct {
+	Id          int64  `json:"msg_id"`
+	EntId       int64  `json:"ent_id"`
+	ProjectId   string `json:"projectId"`
+	FirstTime   string `json:"firsttime,omitempty"`   //项目的最早时间
+	LastTime    string `json:"lasttime,omitempty"`    //项目的最后时间
+	ProjectName string `json:"projectname,omitempty"` //项目名称
+	ProjectCode string `json:"projectcode,omitempty"` //项目代码
+	Buyer       string `json:"buyer,omitempty"`       //采购单位
+	Agency      string `json:"agency"`                //代理机构
+	//Winners       string  `json:"s_winner,omitempty"`    //中标人
+	Area          string  `json:"area"`                //地区
+	City          string  `json:"city"`                //地市
+	District      string  `json:"district"`            //区县
+	Bidamount     float64 `json:"bidamount,omitempty"` //中标金额
+	Budget        float64 `json:"budget,omitempty"`    //预算
+	Bidtype       string  `json:"bidtype"`
+	Bidstatus     string  `json:"bidstatus"`
+	score         int
+	comStr        string
+	resVal, pjVal int
+	Topscopeclass []string `json:"topscopeclass,omitempty"`
+	Subscopeclass []string `json:"subscopeclass,omitempty"` //子行业分类
+	Buyerperson   string   `json:"buyerperson"`             //采购联系人
+	Buyertel      string   `json:"buyertel"`                //采购联系人电话
+	AgencyPerson  string   `json:"agencyperson"`
+	AgencyTel     string   `json:"agencytel"`
+	//WinnerPerson  string   `json:"winnerperson"`
+	//WinnerTel     string   `json:"winnertel"`
+	Buyerclass  string `json:"buyerclass"`            //采购单位分类
+	Bidopentime int64  `json:"bidopentime,omitempty"` //开标时间
+	Jgtime      int64  `json:"jgtime"`                //结果中标时间
+	Zbtime      int64  `json:"zbtime"`                //招标时间
+
+	Industry  string `json:"industry"`
+	Region    string `json:"region"`
+	WinnerTag string `json:"winner_tag"`
+	TagName   string `json:"tagname"`  //招标单位一级标签
+	TagName2  string `json:"tagname2"` //招标单位二级行业
+	TagName3  string `json:"tagname3"` //中标单位行业归类
+}
+
+type Info struct {
+	Id           int64   `json:"id"`
+	EntId        int64   `json:"eid"`
+	RuleName     string  `json:"rulename"`
+	TagName      string  `json:"tagname"`
+	TagName2     string  `json:"tagname2"`
+	TagName3     string  `json:"tagname3"`
+	Href         string  `json:"href"` //源地址
+	JybxHref     string  `json:"jybxhref"`
+	Publishtime  string  `json:"publishtime"`
+	Title        string  `json:"title"`
+	TopType      string  `json:"toptype"`
+	SubType      string  `json:"subtype"`
+	ProjectName  string  `json:"projectname"`
+	ProjectCode  string  `json:"projectcode"`
+	Buyer        string  `json:"buyer"`
+	Buyerperson  string  `json:"buyerperson"`
+	Buyertel     string  `json:"buyertel"`
+	WinnerTel    string  `json:"winnertel"`
+	WinnerPerson string  `json:"winnerperson"`
+	Agency       string  `json:"agency"`
+	AgencyPerson string  `json:"agencyperson"`
+	AgencyTel    string  `json:"agencytel"`
+	Area         string  `json:"area"`
+	City         string  `json:"city"`
+	District     string  `json:"district"`
+	Buyerclass   string  `json:"buyerclass"`
+	Budget       float64 `json:"budget"`
+	Bidamount    float64 `json:"bidamount"`
+	Region       string  `json:"region"`
+	Industry     string  `json:"industry"`
+	WinnerTag    string  `json:"winner_tag"`
+
+	Winners []string
+	pnbval  int //项目名称、编号、采购单位存在的个数
+	LenPC   int //项目编号长度
+	LenPN   int //项目名称长度
+	LenPTC  int //标题抽的项目编号长度
+}

customer_project/main.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"customer_project/config"
+	"fmt"
+	"github.com/robfig/cron/v3"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"net"
+	"net/http"
+	"net/rpc"
+	"os"
+)
+
+var (
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+	saveSize   int
+)
+
+func init() {
+	config.Init("./common.toml")
+	InitLog()
+
+	InitMysql()
+}
+
+func main() {
+	loadData()
+	go RpcService()
+
+	taskProject()
+	crn := cron.New()
+	_, _ = crn.AddFunc("@every 10m", func() {
+		taskProject()
+	})
+	crn.Start()
+
+	ch := make(chan bool, 1)
+	<-ch
+}
+
+func RpcService() {
+	rpc.Register(new(JyService)) // 注册rpc服务
+	rpc.HandleHTTP()             // 采用http协议作为rpc载体
+	util.Debug(config.Conf.Serve.RpcServer)
+	lis, err := net.Listen("tcp", config.Conf.Serve.RpcServer)
+	if err != nil {
+		util.Debug("fatal error: ", err)
+	}
+	_, _ = fmt.Fprintf(os.Stdout, "%s\n", "start connection")
+	_ = http.Serve(lis, nil)
+}

customer_project/merge_comparepnc.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"strings"
+)
+
+//对比项目名称、项目编号
+
+func comparePNC(info *Info, compareProject *Project) (compareStr string, score int) {
+	if info.ProjectName != "" {
+		pns := []string{}
+		if compareProject.ProjectName != "" {
+			pns = append(pns, compareProject.ProjectName)
+		}
+		ifind := 0
+		templen := 0
+		for _, v := range pns {
+			if info.ProjectName == v {
+				ifind = 1
+				break
+			} else {
+				//if strings.Contains(info.ProjectName, v) || strings.Contains(v, info.ProjectName) ||
+				retv := CheckContain(info.ProjectName, v)
+				if retv == 1 {
+					ifind = 1
+					break
+				} else {
+					//v1 := CosineSimilar(pn, v)
+					if retv == 2 {
+						templen = len([]rune(v))
+						ifind = 2
+						break
+					}
+					//else if ifind == 0 {
+					//	ifind = 3
+					//}
+				}
+			}
+		}
+		switch ifind {
+		case 0:
+			compareStr = "D"
+		case 1:
+			compareStr = "A"
+			score += 4
+			if len([]rune(info.ProjectName)) > 18 {
+				score += 2
+			}
+		case 2:
+			compareStr = "B"
+			score += 2
+			if templen > info.LenPN {
+				templen = info.LenPN
+			}
+			if templen > 12 {
+				score += 1
+			}
+		case 3:
+			compareStr = "C"
+		}
+	} else {
+		compareStr = "D"
+	}
+
+	/*
+				项目编号 - －()（） 要注意
+				init_text = ["号","(重)","(第二次)","（重）"]
+		all_clean_mark = ["[","(","【","（","〖","]",")","】","）","〗","-","〔","〕","《","［","］","｛","｝","{","—"," ","－","﹝","﹞","–"]
+	*/
+	for index, pc := range []string{info.ProjectCode, ""} {
+		if pc != "" {
+			pcs := []string{}
+			if compareProject.ProjectCode != "" {
+				pcs = append(pcs, compareProject.ProjectCode)
+			}
+			ifind := 0
+			templen := 0
+			for _, v := range pcs {
+				if pc == v {
+					ifind = 1
+					break
+				} else {
+					// math.Abs(float64(len([]rune(pc))-len([]rune(v)))) < 6
+					//if !_numreg1.MatchString(pc) && !_zimureg1.MatchString(pc) && !_numreg1.MatchString(v) && !_zimureg1.MatchString(v)
+					if strings.Contains(pc, v) || strings.Contains(v, pc) {
+						t1 := pc
+						t2 := v
+						if len(v) > len(pc) {
+							t1 = v
+							t2 = pc
+						}
+						t3 := strings.Replace(t1, t2, "", -1)
+						t3 = _datereg.ReplaceAllString(t3, "")
+						if t3 == "" {
+							ifind = 1
+							break
+						} else {
+							ifind = 2
+							templen = len([]rune(v))
+						}
+					} else if ifind == 0 {
+						ifind = 3
+					}
+				}
+			}
+			switch ifind {
+			case 0:
+				compareStr += "D"
+			case 1:
+				compareStr += "A"
+				score += 4
+				if len([]rune(pc)) > 18 {
+					score += 2
+				}
+			case 2:
+				compareStr += "B"
+				score += 2
+				if index == 0 {
+					if templen > info.LenPC {
+						templen = info.LenPC
+					}
+					if templen > 12 {
+						score += 1
+					}
+
+				} else {
+					if templen > info.LenPTC {
+						templen = info.LenPTC
+					}
+					if templen > 12 {
+						score += 1
+					}
+				}
+
+			case 3:
+				compareStr += "C"
+			}
+
+		} else {
+			compareStr += "D"
+		}
+	}
+	return
+}
+
+func CheckContain(b1, b2 string) (res int) {
+	b1 = replaceStr.ReplaceAllString(b1, "")
+	b2 = replaceStr.ReplaceAllString(b2, "")
+
+	if b1 == b2 {
+		res = 1 //相等
+		return
+	}
+	bs1 := []rune(b1)
+	bs2 := []rune(b2)
+	tmp := ""
+	for i := 0; i < len(bs1); i++ {
+		for j := 0; j < len(bs2); j++ {
+			if bs1[i] == bs2[j] {
+				tmp += string(bs1[i])
+			} else if tmp != "" {
+				b1 = strings.Replace(b1, tmp, "", -1)
+				b2 = strings.Replace(b2, tmp, "", -1)
+				tmp = ""
+			}
+		}
+	}
+	if tmp != "" {
+		b1 = strings.Replace(b1, tmp, "", -1)
+		b2 = strings.Replace(b2, tmp, "", -1)
+	}
+	if b1 == b2 {
+		res = 1 //相等
+	} else if b1 == "" || b2 == "" {
+		res = 2 //包含
+	} else {
+		res = 3 //不相等
+	}
+	return
+}

customer_project/merge_select.go

@@ -0,0 +1,431 @@
+package main
+
+// 根据字符特征打分
+// 3为最高分，pj为评级 A AD A  AA AA AB
+func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
+	//没有可对比的项目名称、或项目编号 //评级
+	if compareNoPass[compareStr] {
+
+	} else {
+		switch compareStr {
+		case "AAA":
+			res = 3
+			pj = 3
+		case "AAB":
+			res = 3
+			pj = 3
+		case "ABA":
+			res = 3
+			pj = 3
+		case "ABB":
+			if info.LenPN > 10 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BAA":
+			if info.LenPC > 8 || info.LenPTC > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BAB":
+			if info.LenPTC > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BBA":
+			if info.LenPC > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BBB":
+			res = 2
+			pj = 2
+		case "AAD":
+			if info.LenPC > 8 || info.LenPN > 12 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "ABD":
+			if info.LenPN > 10 {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BAD":
+			if info.LenPC > 13 || info.LenPC > 8 {
+				res = 3
+				pj = 3
+			} else if info.LenPC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BBD":
+			res = 1
+			pj = 3
+		case "ADA":
+			if info.LenPN > 12 || info.LenPTC > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 2
+		case "ADB":
+			if info.LenPN > 10 {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BDA":
+			if info.LenPTC > 8 {
+				res = 3
+				pj = 2
+			} else if info.LenPTC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BDB":
+			res = 1
+			pj = 2
+		case "ADD":
+			if info.LenPN > 18 {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "BDD":
+			if CheckContain(info.ProjectName, compareInfo.ProjectName) != 3 {
+				res = 1
+				pj = 1
+			}
+		case "DAA":
+			if info.LenPTC > 8 || info.LenPC > 8 {
+				res = 3
+				pj = 2
+			} else {
+				res = 2
+				pj = 3
+			}
+		case "DAB":
+			res = 1
+			pj = 3
+		case "DAD":
+			if info.LenPC > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "DBA":
+			if info.LenPC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DBB":
+			if !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 1
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DBD":
+			if !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+				pj = 1
+			} else {
+				res = 1
+				pj = 1
+			}
+		case "DDA":
+			if info.LenPTC > 14 {
+				res = 3
+				pj = 1
+			} else if info.LenPTC > 8 {
+				res = 2
+				pj = 1
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "DDB":
+			res = 1
+			pj = 1
+		case "ACA":
+			if info.LenPN > 10 && info.LenPTC > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.LenPTC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "ACB":
+			if info.LenPN > 10 && info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BCA":
+			if info.LenPTC > 8 || info.LenPTC > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				res = 3
+				pj = 2
+			} else if info.LenPTC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BCB":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				res = 3
+				pj = 1
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "AAC":
+			if (info.LenPN > 10 && info.LenPC > 8) || info.LenPN > 14 || (info.LenPC > 10 && !StrOrNum2.MatchString(info.ProjectCode)) {
+				res = 3
+				pj = 3
+			} else {
+				res = 2
+				pj = 3
+			}
+		case "ABC":
+			if info.LenPN > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BAC":
+			if info.LenPC > 8 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BBC":
+			if !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 1
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "ACC":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPN > 16 {
+					res = 2
+					pj = 1
+				} else {
+					res = 1
+					pj = 2
+				}
+			}
+		case "ACD":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPN > 16 {
+
+					res = 2
+				} else {
+					res = 1
+
+				}
+				pj = 1
+			}
+
+		case "ADC":
+			if info.LenPN > 16 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "BCC":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				res = 1
+				pj = 1
+			}
+		case "BCD":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				res = 1
+				pj = 1
+			}
+		case "BDC":
+			res = 1
+			pj = 1
+		case "CAA":
+			if info.LenPC > 12 || info.LenPTC > 12 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 || info.LenPTC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "CAB":
+			if info.LenPC > 12 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 12 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "CAC":
+			if info.LenPC > 9 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 1
+			}
+		case "CAD":
+			if info.LenPC > 9 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "CBA":
+			if info.LenPTC > 14 {
+				res = 3
+				pj = 2
+			} else if info.LenPTC > 12 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "CBB":
+			res = 1
+			pj = 2
+		case "CBC":
+			if !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "CBD":
+			if !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "CCA":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPTC > 12 {
+					res = 2
+				} else if info.LenPTC > 5 {
+					res = 1
+				}
+				pj = 1
+			}
+		case "CCB":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				res = 1
+				pj = 1
+			}
+		case "CDA":
+			if info.LenPTC > 12 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "CDB":
+
+		case "DAC":
+			if info.LenPC > 13 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+			} else if info.LenPC > 8 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "DBC":
+			pj = 1
+		case "DCA":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPTC > 10 {
+					res = 2
+				} else {
+					res = 1
+				}
+				pj = 1
+			}
+		case "DCB":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				pj = 1
+			}
+		}
+
+	}
+	return
+}

customer_project/project.go

@@ -0,0 +1,724 @@
+package main
+
+import (
+	"app.yhyue.com/moapp/jybase/encrypt"
+	"customer_project/config"
+	"github.com/google/uuid"
+	"go.mongodb.org/mongo-driver/bson"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	wg sync.WaitGroup
+)
+
+// 从对应map中获取对比的项目id
+func getCompareIds(pn, pc, pb string, entid int64) (bpn, bpc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
+	//	p.ConCurrentLock(n1, n2, n3, n4)
+	//	defer p.ConCurrentUnLock(n1, n2, n3, n4)
+	wg.Add(1)
+	//查找到id数组
+	res = []*Key{}
+	//是否查找到，并标识位置。-1代表值为空或已经存在。
+	bpn, bpc, bpb = -1, -1, -1
+
+	var mapPn, mapPc, mapPb map[string]*Key
+	if mapEnt[entid] == nil {
+
+		mapPn = make(map[string]*Key, 50000)
+		mapPc = make(map[string]*Key, 50000)
+		mapPb = make(map[string]*Key, 50000)
+	} else {
+		mapPn = mapEnt[entid].mapPn
+		mapPc = mapEnt[entid].mapPc
+		mapPb = mapEnt[entid].mapPb
+	}
+
+	if pn != "" {
+		ids := mapPn[pn]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			mapPn[pn] = ids
+			bpn = 0
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if pc != "" {
+		ids := mapPc[pc]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			mapPc[pc] = ids
+			bpc = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if pb != "" {
+		ids := mapPb[pb]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			mapPb[pb] = ids
+			bpb = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+	repeatId := map[string]bool{}
+	idArr = []string{} //项目id
+	IDArr = []*ID{}    //项目信息
+	for _, m := range res {
+		for _, id := range m.Arr {
+			if !repeatId[id] {
+				repeatId[id] = true
+				//_, _ = strconv.ParseInt(id[0:8], 16, 64)
+				AllIdsMapLock.Lock()
+				Id := AllIdsMap[id]
+				AllIdsMapLock.Unlock()
+				if Id != nil {
+					Id.Lock.Lock()
+					idArr = append(idArr, id)
+					IDArr = append(IDArr, Id)
+				}
+			}
+		}
+	}
+
+	mapEnt[entid] = &EntMap{
+		mapPb: mapPb,
+		mapPn: mapPn,
+		mapPc: mapPc,
+	}
+	return
+}
+
+func startProjectMerge(info *Info, tmp map[string]interface{}) {
+	findLock.Lock()
+	defer findLock.Unlock()
+	//bpn, bpc, bptc, bpb 是否查找到，并标识位置。-1代表未查找到。
+	//pids 是项目id数组集合
+	//IDArr，是单个项目ID对象集合
+	bpn, bpc, bpb, pids, _, IDArr := getCompareIds(info.ProjectName, info.ProjectCode, info.Buyer, info.EntId) //, info.LenPN, info.LenPC, info.LenPTC, len([]rune(info.Buyer)))
+	defer wg.Done()
+	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock
+
+	for _, m := range pids {
+		defer m.Lock.Unlock()
+	}
+	for _, id := range IDArr {
+		defer id.Lock.Unlock()
+	}
+
+	bFindProject := false
+	findPid := ""
+	//获取完id，进行计算
+	//定义两组
+	comRes1 := []*Project{} //优先级最高的对比结果数组
+	comRes2 := []*Project{} //优化级其次
+	comRes3 := []*Project{}
+	for _, v := range IDArr {
+		comStr := ""
+		compareProject := v.P
+		compareProject.score = 0
+		//代理机构完全不相同，直接新建项目
+		if CheckContain(compareProject.Agency, info.Agency) == 3 {
+			continue
+		}
+		//地区(省、市、区)不同，直接新建项目
+		if ComparePlace(compareProject, info) {
+			continue
+		}
+
+		compareStr, score := comparePNC(info, compareProject)
+
+		resVal, pjVal := Select(compareStr, info, compareProject)
+		//---------------------------------------
+		if resVal > 0 {
+			compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount, score2 := compareBCTABB(info, compareProject, 0, score)
+
+			//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
+			comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
+			compareProject.comStr = comStr
+			compareProject.pjVal = pjVal
+			compareProject.resVal = resVal
+			//log.Println(compareProject.comStr)
+			eqV := compareResult(resVal, pjVal, score2, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount)
+			if eqV == 1 {
+				comRes1 = append(comRes1, compareProject)
+			} else if eqV == 2 {
+				comRes2 = append(comRes2, compareProject)
+			} else if eqV == 3 {
+				comRes3 = append(comRes3, compareProject)
+			}
+		}
+	}
+	//--------------------------------对比完成-----------------------
+
+	//更新数组、更新项目
+	for kv, resN := range [][]*Project{comRes1, comRes2, comRes3} {
+		if len(resN) > 0 {
+			if len(resN) > 1 {
+				sort.Slice(resN, func(i, j int) bool {
+					return resN[i].score > resN[j].score
+				})
+			}
+
+			resArr := []*Project{}
+			for _, res := range resN {
+				resArr = append(resArr, res)
+			}
+			if len(resArr) > 0 {
+				bFindProject = true
+				findPid = resArr[0].ProjectId
+				UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr)
+				for k2, bv := range []int{bpn, bpc, bpb} {
+					if bv > -1 {
+						pids[bv].Arr = append(pids[bv].Arr, findPid)
+						if k2 == 0 {
+							if resArr[0].ProjectName == "" {
+								resArr[0].ProjectName = info.ProjectName
+							}
+						} else if k2 < 3 {
+							if resArr[0].ProjectCode == "" {
+								resArr[0].ProjectCode = info.ProjectCode
+							}
+						} else {
+							if resArr[0].Buyer == "" {
+								resArr[0].Buyer = info.Buyer
+							}
+						}
+					}
+				}
+			} else {
+				bFindProject = false
+				findPid = ""
+			}
+			break
+		}
+	}
+
+	if !bFindProject {
+		id, p1 := NewProject(tmp, info)
+		AllIdsMapLock.Lock()
+		AllIdsMap[id] = &ID{Id: id, P: p1}
+		AllIdsMapLock.Unlock()
+		for _, m := range pids {
+			m.Arr = append(m.Arr, id)
+		}
+	}
+}
+
+func compareBCTABB(info *Info, cp *Project, diffTime int64, score int) (compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string, score2 int) {
+	compareBuyer = "D"
+	if len([]rune(info.Buyer)) > 3 && len([]rune(cp.Buyer)) > 3 {
+		v := CheckContain(info.Buyer, cp.Buyer)
+		if v == 1 {
+			compareBuyer = "A"
+			score += 3
+		} else {
+			//v1 := CosineSimilar(info.Buyer, cp.Buyer)
+			if v == 2 {
+				compareBuyer = "B"
+				score += 1
+			} else {
+				compareBuyer = "C"
+			}
+		}
+	}
+	//---------------------------------------
+
+	compareCity = ""
+	if info.Area != "全国" && info.Area != "" && info.Area == cp.Area {
+		compareCity += "A"
+		score += 2
+	} else if info.Area == "全国" || cp.Area == "全国" {
+		compareCity += "B"
+		score += 1
+	} else {
+		compareCity += "C"
+	}
+	if compareCity != "C" {
+		if info.City != "" && info.City == cp.City {
+			compareCity += "A"
+			score += 2
+		} else {
+			if info.Area == "全国" || cp.Area == "全国" {
+				compareCity += "B"
+			} else if info.City == compareCity {
+				compareCity += "B"
+			} else {
+				compareCity += "C"
+			}
+		}
+	} else {
+		compareCity += "C"
+	}
+	score2 = 0
+	if compareCity == "AA" {
+		if info.District != "" && info.District == cp.District {
+			score2 = 1
+		}
+	}
+
+	compareTime = "D"
+	if diffTime < 45*86400 {
+		compareTime = "A"
+		score += 2
+	} else if diffTime < 90*86400 {
+		compareTime = "B"
+		score += 1
+	}
+
+	compareAgency = "D"
+	if info.Agency != "" {
+		if info.Agency == cp.Agency {
+			compareAgency = "A"
+			score += 2
+			score2 += 1
+		} else if cp.Agency != "" {
+			if strings.Contains(info.Agency, cp.Agency) || strings.Contains(cp.Agency, info.Agency) {
+				compareAgency = "B"
+				score += 1
+				score2 += 1
+			} else {
+				compareAgency = "C"
+			}
+		}
+	}
+	compareBudget = "C"
+	if info.Budget > 0 && (info.Budget == cp.Budget || (cp.Bidamount > 0 && info.Budget > cp.Bidamount && (info.Budget-cp.Bidamount) < (0.15*info.Budget))) {
+		compareBudget = "A"
+		score += 1
+		score2 += 1
+	}
+	//	else if info.Budget == 0 && cp.Budget == 0 {
+	//		compareBudget = "B"
+	//	}
+	compareBidmount = "C"
+	if info.Bidamount > 0 && (info.Bidamount == cp.Bidamount || (cp.Budget > 0 && cp.Budget > info.Bidamount && (cp.Budget-info.Bidamount) < 0.15*cp.Budget)) {
+		compareBidmount = "A"
+		score += 1
+		score2 += 1
+	}
+	//	else if info.Bidamount == 0 && cp.Bidamount == 0 {
+	//		compareBidmount = "B"
+	//	}
+
+	cp.score = score
+	return
+}
+
+func compareResult(resVal, pjVal, score2 int, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string) int {
+	eqV := 0
+	switch resVal {
+	case 3:
+		if pjVal == 3 && comStr[3:] != "CCCDCCC" {
+			eqV = 1
+		} else if compareBuyer < "C" {
+			if pjVal > 1 {
+				eqV = 1
+			} else { //if (compareCity[1:1] != "C" || compareTime != "D") && score2 > 0
+				eqV = 2
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && (compareCity[1:1] != "C" || score2 > 0) {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+		} else {
+			if pjVal == 3 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if pjVal == 2 && compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			} else if compareCity == "AA" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+		}
+	case 2:
+		if compareBuyer < "C" {
+			if pjVal > 1 {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" || score2 > 0 {
+				eqV = 3
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+
+		} else {
+			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		}
+	case 1:
+		if compareBuyer < "C" {
+			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		} else {
+			if pjVal > 1 && compareTime == "A" && score2 > 0 && (compareBudget == "A" || compareBidmount == "A") && compareCity[1:1] != "C" {
+				eqV = 3
+			}
+		}
+	}
+	return eqV
+}
+
+func ComparePlace(project *Project, info *Info) bool {
+	if info.Area == "全国" || info.Area == "" {
+		return false
+	}
+	if project.Area == "全国" || project.Area == "" {
+		return false
+	}
+	if info.Area == project.Area {
+		if info.City == "" || project.City == "" {
+			return false
+		} else if info.City == project.City {
+			return false
+		}
+	} else {
+		return true
+	}
+	return true
+}
+
+func NewCachePinfo(thisinfo *Info, pid string) Project {
+	p1 := Project{
+		ProjectId:   pid,
+		EntId:       thisinfo.EntId,
+		ProjectName: thisinfo.ProjectName,
+		ProjectCode: thisinfo.ProjectCode,
+		Buyer:       thisinfo.Buyer,
+		Agency:      thisinfo.Agency,
+		Area:        thisinfo.Area,
+		City:        thisinfo.City,
+		District:    thisinfo.District,
+		FirstTime:   thisinfo.Publishtime,
+		LastTime:    thisinfo.Publishtime,
+		Budget:      thisinfo.Budget,
+		Bidamount:   thisinfo.Bidamount,
+	}
+	return p1
+}
+
+func NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *Project) {
+	pId := uuid.New().String()
+	pId = strings.ReplaceAll(pId, "-", "")
+	set := map[string]interface{}{}
+	set["ent_id"] = thisinfo.EntId
+	set["projectId"] = pId
+	for _, f := range FIELDS {
+		if tmp[f] != nil && tmp[f] != "" {
+			set[f] = tmp[f]
+		}
+	}
+	if util.ObjToString(tmp["projectname"]) == "" {
+		set["projectname"] = thisinfo.ProjectName
+	}
+	p1 := NewCachePinfo(thisinfo, pId)
+	bt := util.ObjToString(tmp["toptype"])
+	bs := util.ObjToString(tmp["subtype"])
+	if thisinfo.SubType == "拟建" {
+		set["bidstatus"] = "拟建"
+		bt = "拟建"
+	} else {
+		if bidtype[bs] != "" {
+			set["bidtype"] = bidtype[bs]
+		} else {
+			set["bidtype"] = "招标"
+		}
+		if bt == "招标" {
+			set["bidstatus"] = bt
+		} else {
+			if bidstatus[bs] != "" {
+				set["bidstatus"] = thisinfo.SubType
+				bt = thisinfo.SubType
+			} else if bs == "" {
+				set["bidstatus"] = ""
+				bt = ""
+			} else {
+				set["bidstatus"] = "其它"
+				bt = "其它"
+			}
+		}
+	}
+	if tmp["bidopentime"] != nil {
+		set["bidopentime"] = tmp["bidopentime"]
+	}
+	now := time.Now()
+	set["createtime"] = util.FormatDate(&now, util.Date_Full_Layout)
+	set["updatetime"] = util.FormatDate(&now, util.Date_Full_Layout)
+	set["firsttime"] = thisinfo.Publishtime
+	set["lasttime"] = thisinfo.Publishtime
+	if thisinfo.TopType == "招标" {
+		set["zbtime"] = tmp["publishtime"]
+	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
+		set["jgtime"] = tmp["publishtime"]
+	}
+	if util.ObjToString(tmp["tagname"]) != "" {
+		p1.TagName = util.ObjToString(tmp["tagname"])
+		set["tagname"] = util.ObjToString(tmp["tagname"])
+	}
+	if util.ObjToString(tmp["tagname2"]) != "" {
+		p1.TagName2 = util.ObjToString(tmp["tagname2"])
+		set["tagname2"] = util.ObjToString(tmp["tagname2"])
+	}
+	if util.ObjToString(tmp["tagname3"]) != "" {
+		p1.TagName3 = util.ObjToString(tmp["tagname3"])
+		set["tagname3"] = util.ObjToString(tmp["tagname3"])
+	}
+	if tmp["industry"] != nil {
+		set["industry"] = util.IntAll(tmp["industry"])
+	}
+	if tmp["region"] != nil {
+		set["region"] = util.IntAll(tmp["region"])
+	}
+	if tmp["winner_tag"] != nil {
+		set["winner_tag"] = util.IntAll(tmp["winner_tag"])
+	}
+
+	id := MysqlTool.Insert(config.Conf.DB.Mysql.Pcoll, set)
+	if id > 0 {
+		if len(thisinfo.Winners) > 0 {
+			for _, w := range thisinfo.Winners {
+				if w == "" {
+					continue
+				}
+				wSave := make(map[string]interface{})
+				wSave["projectId"] = pId
+				wSave["msg_id"] = id
+				wSave["winner_name"] = w
+				wSave["bidamount"] = tmp["bidamount"]
+				wSave["winner_person"] = thisinfo.WinnerPerson
+				wSave["winner_tel"] = thisinfo.WinnerTel
+				wSave["ent_id"] = thisinfo.EntId
+				einfo := MysqlEnt.FindOne("dws_f_ent_baseinfo", bson.M{"name": w}, "id,name_id,company_id", "")
+				if einfo != nil && len(*einfo) > 0 {
+					wSave["name_id"] = (*einfo)["name_id"]
+					wSave["company_id"] = (*einfo)["company_id"]
+					if cid := util.ObjToString((*einfo)["company_id"]); cid != "" {
+						wSave["encrypt_id"] = encrypt.EncodeArticleId2ByCheck(cid)
+					}
+				}
+				MysqlTool.Insert("customer_data_yys_project_winner", wSave)
+			}
+		}
+		MysqlTool.Update(config.Conf.DB.Mysql.Coll, bson.M{"id": thisinfo.Id}, bson.M{"projectId": pId})
+	}
+	return pId, &p1
+}
+
+func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *Project, weight int, comStr string) {
+	set := map[string]interface{}{}
+	//相同城市的公告才会合并到一起(全国列外)
+	if thisinfo.Area != "全国" {
+		pInfo.Area = thisinfo.Area
+		set["area"] = thisinfo.Area
+		if thisinfo.City != "" {
+			pInfo.City = thisinfo.City
+			set["city"] = thisinfo.City
+		}
+		if thisinfo.District != "" {
+			pInfo.District = thisinfo.District
+			set["district"] = thisinfo.District
+		}
+	}
+	//项目名称
+	if pInfo.ProjectName == "" && thisinfo.ProjectName != "" {
+		pInfo.ProjectName = thisinfo.ProjectName
+		set["projectname"] = thisinfo.ProjectName
+	}
+	if pInfo.ProjectCode == "" && thisinfo.ProjectCode != "" {
+		pInfo.ProjectCode = thisinfo.ProjectCode
+		set["projectcode"] = thisinfo.ProjectCode
+	}
+	//7--采购单位
+	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
+		pInfo.Buyer = thisinfo.Buyer
+		set["buyer"] = thisinfo.Buyer
+
+		pInfo.Buyerclass = thisinfo.Buyerclass
+		set["buyerclass"] = thisinfo.Buyerclass
+	}
+	//采购单位联系人
+	if thisinfo.Buyerperson != "" {
+		pInfo.Buyerperson = thisinfo.Buyerperson
+		set["buyerperson"] = thisinfo.Buyerperson
+	}
+	//采购单位電話
+	if thisinfo.Buyertel != "" {
+		pInfo.Buyertel = thisinfo.Buyertel
+		set["buyertel"] = pInfo.Buyertel
+	}
+
+	//8--代理机构
+	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
+		pInfo.Agency = thisinfo.Agency
+		set["agency"] = thisinfo.Agency
+	}
+	if util.ObjToString(tmp["agencyperson"]) != "" {
+		set["agencyperson"] = tmp["agencyperson"]
+	}
+	if util.ObjToString(tmp["agencytel"]) != "" {
+		set["agencytel"] = tmp["agencytel"]
+	}
+	//if len(thisinfo.Winners) > 0 {
+	//	var winners []string
+	//	if pInfo.Winners == "" {
+	//		for _, k := range thisinfo.Winners {
+	//			if k == "" {
+	//				continue
+	//			}
+	//			if BinarySearch(winners, k) == -1 {
+	//				winners = append(winners, k)
+	//			}
+	//		}
+	//	} else {
+	//		winners = strings.Split(pInfo.Winners, ",")
+	//		for _, k := range thisinfo.Winners {
+	//			if k == "" {
+	//				continue
+	//			}
+	//			if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+	//				if BinarySearch(winners, k) != -1 {
+	//					deleteSlice(winners, k, "")
+	//				}
+	//			} else {
+	//				if BinarySearch(winners, k) == -1 {
+	//					winners = append(winners, k)
+	//				}
+	//			}
+	//		}
+	//	}
+	//	set["s_winner"] = strings.Join(util.ObjArrToStringArr(Duplicate(winners)), ",")
+	//	if util.ObjToString(tmp["winnertel"]) != "" {
+	//		set["winnertel"] = tmp["winnertel"]
+	//	}
+	//	if util.ObjToString(tmp["winnerperson"]) != "" {
+	//		set["winnerperson"] = tmp["winnerperson"]
+	//	}
+	//}
+	bt := util.ObjToString(tmp["toptype"])
+	bs := util.ObjToString(tmp["subtype"])
+	if bt == "招标" {
+		//招标状态，更新projectscope
+		if tmp["projectscope"] != nil {
+			set["projectscope"] = util.ObjToString(tmp["projectscope"])
+		}
+		set["bidstatus"] = bt
+		pInfo.Bidstatus = bt
+		if bidtype[bs] != "" {
+			set["bidtype"] = bidtype[bs]
+			pInfo.Bidtype = bidtype[bs]
+		} else {
+			set["bidtype"] = "招标"
+			pInfo.Bidtype = "招标"
+		}
+	} else {
+		if bidstatus[bs] != "" {
+			set["bidstatus"] = thisinfo.SubType
+			pInfo.Bidstatus = thisinfo.SubType
+		} else if bs == "" && bt == "结果" {
+			if pInfo.Bidstatus == "招标" {
+				set["bidstatus"] = ""
+				pInfo.Bidstatus = ""
+			}
+		} else {
+			set["bidstatus"] = "其它"
+			pInfo.Bidstatus = "其它"
+		}
+	}
+	if tmp["bidopentime"] != nil {
+		set["bidopentime"] = tmp["bidopentime"]
+	}
+	set["lasttime"] = thisinfo.Publishtime
+	if thisinfo.TopType == "招标" {
+		set["zbtime"] = tmp["publishtime"]
+	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
+		set["jgtime"] = tmp["publishtime"]
+	}
+	if util.ObjToString(tmp["tagname"]) != "" && pInfo.TagName != "" {
+		pInfo.TagName = util.ObjToString(tmp["tagname"])
+		set["tagname"] = util.ObjToString(tmp["tagname"])
+	}
+	if util.ObjToString(tmp["tagname2"]) != "" && pInfo.TagName2 != "" {
+		pInfo.TagName2 = util.ObjToString(tmp["tagname2"])
+		set["tagname2"] = util.ObjToString(tmp["tagname2"])
+	}
+	if util.ObjToString(tmp["tagname3"]) != "" && pInfo.TagName3 != "" {
+		pInfo.TagName3 = util.ObjToString(tmp["tagname3"])
+		set["tagname3"] = util.ObjToString(tmp["tagname3"])
+	}
+	if tmp["industry"] != nil {
+		set["industry"] = thisinfo.Industry
+	}
+	if tmp["region"] != nil {
+		set["region"] = thisinfo.Region
+	}
+	if tmp["winning_tag"] != nil {
+		set["winning_tag"] = thisinfo.WinnerTag
+	}
+
+	if thisinfo.Budget > 0 {
+		set["budget"] = thisinfo.Budget
+		pInfo.Budget = thisinfo.Budget
+	}
+	if thisinfo.Bidamount > 0 {
+		set["bidamount"] = thisinfo.Bidamount
+		pInfo.Bidamount = thisinfo.Bidamount
+	}
+	now := time.Now()
+	set["updatetime"] = util.FormatDate(&now, util.Date_Full_Layout)
+	b := MysqlTool.Update(config.Conf.DB.Mysql.Pcoll, bson.M{"projectId": pInfo.ProjectId}, set)
+	if b {
+		if len(thisinfo.Winners) > 0 {
+			for _, w := range thisinfo.Winners {
+				if w == "" {
+					continue
+				}
+				if info := MysqlTool.FindOne("customer_data_yys_project_winner", bson.M{"projectId": pInfo.ProjectId, "winner_name": w}, "", ""); info != nil && len(*info) > 0 {
+					continue
+				}
+				wSave := make(map[string]interface{})
+				wSave["projectId"] = pInfo.ProjectId
+				wSave["msg_id"] = pInfo.Id
+				wSave["winner_name"] = w
+				wSave["bidamount"] = tmp["bidamount"]
+				wSave["winner_person"] = thisinfo.WinnerPerson
+				wSave["winner_tel"] = thisinfo.WinnerTel
+				wSave["ent_id"] = thisinfo.EntId
+				einfo := MysqlEnt.FindOne("dws_f_ent_baseinfo", bson.M{"name": w}, "id,name_id,company_id", "")
+				if einfo != nil && len(*einfo) > 0 {
+					wSave["name_id"] = (*einfo)["name_id"]
+					wSave["company_id"] = (*einfo)["company_id"]
+					if cid := util.ObjToString((*einfo)["company_id"]); cid != "" {
+						wSave["encrypt_id"] = encrypt.EncodeArticleId2ByCheck(cid)
+					}
+				}
+				MysqlTool.Insert("customer_data_yys_project_winner", wSave)
+			}
+		}
+		MysqlTool.Update(config.Conf.DB.Mysql.Coll, bson.M{"id": thisinfo.Id}, bson.M{"projectId": pInfo.ProjectId})
+	}
+}

customer_project/service.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"app.yhyue.com/moapp/jybase/date"
+	"go.mongodb.org/mongo-driver/bson"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+)
+
+type JyService struct {
+}
+
+type DelRequest struct {
+	Eid    int64
+	InfoId int64
+}
+
+type DelResponse struct {
+	Status int
+	Msg    string
+}
+
+func (j *JyService) InfoDel(req DelRequest, res *DelResponse) error {
+	if req.InfoId <= 0 || req.Eid <= 0 {
+		res.Status = -1
+		res.Msg = "参数异常"
+	}
+	util.Debug(req)
+	info := MysqlTool.FindOne("customer_data", bson.M{"eid": req.Eid, "id": req.InfoId}, "", "")
+	if info != nil && len(*info) > 0 {
+		pid := util.ObjToString((*info)["projectId"])
+		if c := MysqlTool.Count("customer_data", bson.M{"projectId": pid}); c > 1 {
+			MysqlTool.Update("customer_data", bson.M{"eid": req.Eid, "id": req.InfoId}, bson.M{"projectId": nil})
+		} else {
+			MysqlTool.Delete("customer_data_yys_project", bson.M{"projectId": pid})
+			MysqlTool.Update("customer_data", bson.M{"eid": req.Eid, "id": req.InfoId}, bson.M{"projectId": nil})
+		}
+	} else {
+		res.Status = -1
+		res.Msg = "信息不存在"
+	}
+	return nil
+}
+
+type RevRequest struct {
+	Eid  int64
+	Pici int64 // 批次时间
+
+}
+
+type RevResponse struct {
+	Status int
+	Msg    string
+}
+
+func (j *JyService) DelByP(req RevRequest, res *RevResponse) error {
+	if req.Pici <= 0 || req.Eid <= 0 {
+		res.Status = -1
+		res.Msg = "参数异常"
+	}
+	util.Debug(req)
+	query := make(map[string]interface{})
+	query["pici"] = date.FormatDateByInt64(&req.Pici, date.Date_Full_Layout)
+	query["entid"] = req.Eid
+	binfo := MysqlTool.Find("customer_data", query, "id", "", -1, -1)
+	for _, info := range *binfo {
+		pid := util.ObjToString((info)["projectId"])
+		if c := MysqlTool.Count("customer_data", bson.M{"projectId": pid}); c > 1 {
+			MysqlTool.Update("customer_data", bson.M{"eid": req.Eid, "id": info["id"]}, bson.M{"projectId": nil})
+		} else {
+			MysqlTool.Delete("customer_data_yys_project", bson.M{"projectId": pid})
+			MysqlTool.Delete("customer_data_yys_monitoring", bson.M{"projectId": pid})
+			MysqlTool.Update("customer_data", bson.M{"eid": req.Eid, "id": info["id"]}, bson.M{"projectId": nil})
+		}
+	}
+
+	return nil
+}

customer_project/task.go

@@ -0,0 +1,275 @@
+package main
+
+import (
+	"customer_project/config"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+var (
+	queryClose     = make(chan bool)
+	queryCloseOver = make(chan bool)
+	pool           = make(chan bool, 2)
+
+	StrOrNum   = regexp.MustCompile("^[0-9_-]{1,4}$|^[a-zA-Z_-]{1,4}$")
+	_datereg   = regexp.MustCompile("20[0-2][0-9][年-][0-9]{1,2}[月-][0-9]{1,2}[日-]([0-9]{1,2}时[0-9]{0,2})?")
+	StrOrNum2  = regexp.MustCompile("^[0-9_-]+$|^[a-zA-Z_-]+$")
+	replaceStr = regexp.MustCompile("(工程|采购|项目|[?！、【】（）—()－-]|栏标价|中标候选人|招标代理)")
+)
+
+func loadData() {
+	finalId := 0
+	lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT msg_id FROM %s ORDER BY msg_id DESC LIMIT 1", config.Conf.DB.Mysql.Pcoll))
+	if len(*lastInfo) > 0 {
+		finalId = util.IntAll((*lastInfo)[0]["msg_id"])
+	}
+	log.Debug("loadData---", zap.Any("finally id", finalId))
+	lastid, count := 0, 0
+	for {
+		log.Debug("重新查询", zap.Any("lastid---", lastid))
+		q := fmt.Sprintf("SELECT * FROM %s WHERE msg_id > %d ORDER BY msg_id ASC limit 100000", config.Conf.DB.Mysql.Pcoll, lastid)
+		rows, err := MysqlTool.DB.Query(q)
+		if err != nil {
+			log.Error("loadData---", zap.Error(err))
+		}
+		columns, err := rows.Columns()
+		if finalId == lastid {
+			log.Debug("---loadData-finish----" + fmt.Sprint(count))
+			break
+		}
+		for rows.Next() {
+			scanArgs := make([]interface{}, len(columns))
+			values := make([]interface{}, len(columns))
+			ret := make(map[string]interface{})
+			for k := range values {
+				scanArgs[k] = &values[k]
+			}
+			err = rows.Scan(scanArgs...)
+			if err != nil {
+				log.Error("loadData---", zap.Error(err))
+				break
+			}
+			for i, col := range values {
+				if v, ok := col.([]uint8); ok {
+					ret[columns[i]] = string(v)
+				} else {
+					ret[columns[i]] = col
+				}
+			}
+			lastid = util.IntAll(ret["msg_id"])
+			count++
+			if count%20000 == 0 {
+				log.Info("loadData current-------", zap.Any("count", count), zap.Any("lastid", lastid))
+			}
+			pool <- true
+			wg.Add(1)
+			func(result map[string]interface{}) {
+				defer func() {
+					<-pool
+					wg.Done()
+				}()
+				bys, _ := json.Marshal(result)
+				var pro *Project
+				_ = json.Unmarshal(bys, &pro)
+
+				eid := pro.EntId
+				id := pro.ProjectId
+				var mapPn, mapPc, mapPb map[string]*Key
+				if mapEnt[eid] != nil {
+					mapPn = mapEnt[eid].mapPn
+					mapPc = mapEnt[eid].mapPc
+					mapPb = mapEnt[eid].mapPb
+					if v := pro.ProjectName; v != "" {
+						k := mapPn[v]
+						if k == nil {
+							k = &Key{Arr: []string{id}}
+							mapPn[v] = k
+						} else {
+							k.Arr = append(k.Arr, id)
+						}
+					}
+					if v := pro.ProjectCode; v != "" {
+						k := mapPc[v]
+						if k == nil {
+							k = &Key{Arr: []string{id}}
+							mapPc[v] = k
+						} else {
+							k.Arr = append(k.Arr, id)
+						}
+					}
+					if pro.Buyer != "" && len([]rune(pro.Buyer)) > 2 {
+						k := mapPb[pro.Buyer]
+						if k == nil {
+							k = &Key{Arr: []string{id}}
+							mapPb[pro.Buyer] = k
+						} else {
+							k.Arr = append(k.Arr, id)
+						}
+					}
+					AllIdsMap[id] = &ID{Id: id, P: pro}
+				} else {
+					mapPn = make(map[string]*Key, 1500000)
+					mapPb = make(map[string]*Key, 5000000)
+					mapPc = make(map[string]*Key, 5000000)
+					if v := pro.ProjectName; v != "" {
+						k := mapPn[v]
+						if k == nil {
+							k = &Key{Arr: []string{id}}
+							mapPn[v] = k
+						} else {
+							k.Arr = append(k.Arr, id)
+						}
+					}
+					if v := pro.ProjectCode; v != "" {
+						k := mapPc[v]
+						if k == nil {
+							k = &Key{Arr: []string{id}}
+							mapPc[v] = k
+						} else {
+							k.Arr = append(k.Arr, id)
+						}
+					}
+					if pro.Buyer != "" && len([]rune(pro.Buyer)) > 2 {
+						k := mapPb[pro.Buyer]
+						if k == nil {
+							k = &Key{Arr: []string{id}}
+							mapPb[pro.Buyer] = k
+						} else {
+							k.Arr = append(k.Arr, id)
+						}
+					}
+					AllIdsMap[id] = &ID{Id: id, P: pro}
+				}
+				mapEnt[eid] = &EntMap{
+					mapPb: mapPb,
+					mapPn: mapPc,
+					mapPc: mapPn,
+				}
+			}(ret)
+			ret = make(map[string]interface{})
+		}
+		_ = rows.Close()
+		wg.Wait()
+	}
+	log.Info("load project over..", zap.Int("n", count))
+}
+
+func taskProject() {
+	pool := make(chan bool, 1) //控制线程数
+	wg := &sync.WaitGroup{}
+
+	finalId := 0
+	lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT id FROM %s where projectId is null and status = 1 ORDER BY id DESC LIMIT 1", config.Conf.DB.Mysql.Coll))
+	if len(*lastInfo) > 0 {
+		finalId = util.IntAll((*lastInfo)[0]["id"])
+	}
+	log.Debug("taskProject---", zap.Any("finalId", finalId))
+	lastid, count := 0, 0
+	for {
+		log.Debug("重新查询，", zap.Any("lastid", lastid))
+		q := fmt.Sprintf("SELECT * FROM %s WHERE id > %d And projectId is null and status = 1 ORDER BY id ASC limit 10000", config.Conf.DB.Mysql.Coll, lastid)
+		var stmtOut *sql.Stmt
+		var tx *sql.Tx
+		var err error
+		if tx == nil {
+			stmtOut, err = MysqlTool.DB.Prepare(q)
+		} else {
+			stmtOut, err = tx.Prepare(q)
+		}
+		rows, err := stmtOut.Query()
+		if err != nil {
+			log.Error("taskProject---", zap.Error(err))
+		}
+		columns, err := rows.Columns()
+		if finalId == lastid {
+			log.Debug("----finish----------", zap.Int("count", count))
+			break
+		}
+		for rows.Next() {
+			scanArgs := make([]interface{}, len(columns))
+			values := make([]interface{}, len(columns))
+			ret := make(map[string]interface{})
+			for k := range values {
+				scanArgs[k] = &values[k]
+			}
+			err = rows.Scan(scanArgs...)
+			if err != nil {
+				log.Error("taskProject---", zap.Error(err))
+				break
+			}
+			for i, col := range values {
+				if v, ok := col.([]uint8); ok {
+					ret[columns[i]] = string(v)
+				} else {
+					ret[columns[i]] = col
+				}
+			}
+			lastid = util.IntAll(ret["id"])
+			count++
+			if count%2000 == 0 {
+				log.Debug("current-------", zap.Any("count", count), zap.Any("lastid", lastid))
+			}
+			pool <- true
+			wg.Add(1)
+			go func(tmp map[string]interface{}) {
+				defer func() {
+					<-pool
+					wg.Done()
+				}()
+				info := ParseInfo(tmp)
+				startProjectMerge(info, tmp)
+			}(ret)
+			ret = make(map[string]interface{})
+		}
+		_ = rows.Close()
+		stmtOut.Close()
+		wg.Wait()
+	}
+
+	log.Info("所有线程执行完成...", zap.Int("count:", count))
+}
+
+func ParseInfo(tmp map[string]interface{}) (info *Info) {
+	bys, _ := json.Marshal(tmp)
+	var thisinfo *Info
+	_ = json.Unmarshal(bys, &thisinfo)
+	if thisinfo == nil {
+		return nil
+	}
+	thisinfo.Budget, _ = strconv.ParseFloat(util.ObjToString(tmp["budget"]), 64)
+	thisinfo.Bidamount, _ = strconv.ParseFloat(util.ObjToString(tmp["bidamount"]), 64)
+	if thisinfo.ProjectName == "" {
+		thisinfo.ProjectName = thisinfo.Title
+	}
+	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
+		thisinfo.pnbval++
+	}
+	if thisinfo.ProjectCode != "" {
+		if thisinfo.ProjectCode != "" {
+			if thisinfo.pnbval == 0 && len([]rune(thisinfo.ProjectCode)) < 5 {
+				thisinfo.ProjectCode = StrOrNum.ReplaceAllString(thisinfo.ProjectCode, "")
+			}
+		}
+		thisinfo.pnbval++
+	}
+	if thisinfo.Buyer != "" && len([]rune(thisinfo.Buyer)) > 2 {
+		thisinfo.pnbval++
+	} else {
+		thisinfo.Buyer = ""
+	}
+	//winners整理、清理
+	winner := util.ObjToString(tmp["s_winner"])
+	thisinfo.Winners = strings.Split(winner, ",")
+
+	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
+	thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
+	return thisinfo
+}

customer_project/tool.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"log"
+	"reflect"
+	"sort"
+	"time"
+)
+
+var DateTimeSelect = []string{"bidopentime", "bidendtime", "signaturedate", "comeintime"}
+
+// 项目中的字段
+var FIELDS = []string{
+	"area",
+	"city",
+	"district",
+	"projectname",
+	"projectcode",
+	"buyer",
+	"buyerclass",
+	"buyerperson",
+	"buyertel",
+	//"winner",
+	//"winnerperson",
+	//"winnertel",
+	"agency",
+	"agencyperson",
+	"agencytel",
+	"budget",
+	"bidamount",
+	"tagname",
+	"tagname_2",
+	"tagname_3",
+}
+
+// 招标信息字段
+var INFOFIELDS = []string{
+	"projectname",
+	"projectcode",
+	"title",
+	"publishtime",
+	"comeintime",
+	"bidopentime",
+	"toptype",
+	"subtype",
+	"buyer",
+	"buyerclass",
+	"agency",
+	"agencyperson",
+	"agencytel",
+	"winner",
+	"winnertel",
+	"winnerperson",
+	"budget",
+	"bidamount",
+	"buyerperson",
+	"buyertel",
+	"area",
+	"city",
+	"district",
+}
+
+var bidstatus = map[string]string{
+	"预告": "预告",
+	"中标": "中标",
+	"成交": "成交",
+	"废标": "废标",
+	"流标": "流标",
+	"合同": "合同",
+}
+
+var bidtype = map[string]string{
+	"招标": "招标",
+	"邀标": "邀标",
+	"询价": "询价",
+	"单一": "单一",
+	"竞价": "竞价",
+	"竞谈": "竞谈",
+}
+
+func Duplicate(a interface{}) (ret []interface{}) {
+	va := reflect.ValueOf(a)
+	for i := 0; i < va.Len(); i++ {
+		if i > 0 && reflect.DeepEqual(va.Index(i-1).Interface(), va.Index(i).Interface()) {
+			continue
+		}
+		ret = append(ret, va.Index(i).Interface())
+	}
+	return ret
+}
+
+// 二分字符串查找
+func BinarySearch(s []string, k string) int {
+	sort.Strings(s)
+	lo, hi := 0, len(s)-1
+	for lo <= hi {
+		m := (lo + hi) >> 1
+		if s[m] < k {
+			lo = m + 1
+		} else if s[m] > k {
+			hi = m - 1
+		} else {
+			return m
+		}
+	}
+	return -1
+}
+
+// 从数组中删除元素
+func deleteSlice(arr []string, v, stype string) []string {
+	for k, v1 := range arr {
+		if v1 == v {
+			ts := time.Now().Unix()
+			arr = append(arr[:k], arr[k+1:]...)
+			rt := time.Now().Unix() - ts
+			if rt > 0 {
+				log.Println("deleteSlice", stype, rt, v, len(arr))
+			}
+			return arr
+		}
+	}
+	return arr
+}

data_credible/common.toml

@@ -0,0 +1,56 @@
+[serve]
+thread = 3
+jyhref = "https://www.jianyu360.cn/article/content/%s.html"
+
+fields = ["agency", "agencyaddr", "agencyfee", "agencyperson", "agencyrate", "agencytel", "bid_bond", "bidamount", "biddiscount",
+    "bidendtime", "bidmethod", "bidopen_shape", "bidopenaddress", "bidopentime", "bidstarttime", "budget", "buyer", "buyeraddr",
+    "buyerperson", "buyertel", "area", "city", "district", "construction_area", "floor_area", "contractcode", "docamount", "docendtime",
+    "docstarttime", "est_purchase_amount", "est_purchase_time", "funds", "is_joint_bidding", "payway", "project_person", "project_phone",
+    "project_scale", "projectaddr", "projectcode", "projectname", "projectperiod", "projectscope", "project_duration", "project_timeunit",
+    "quote_mode", "signaturedate", "signendtime", "signstarttime", "total_investment", "s_winner", "winneraddr", "winnerperson", "winnertel",
+    "basicClass", "enterprise_qualification", "personnel_qualification", "performance_qualification", "enterprise_credit", "package",
+    "purchasinglist", "procurementlist", "winnerorder"
+]
+
+[db]
+[db.mongo]
+addr = "192.168.3.71:29099"
+dbname = "wjh"
+coll = "bidding_credible"
+size = 5
+user = ""
+password = ""
+[db.mongoBz]
+addr = "192.168.3.71:29099"
+dbname = "wjh"
+coll = "A_20230908"
+size = 5
+[db.mongoE]
+addr = "192.168.3.71:29099"
+dbname = "wjh"
+coll = "result_20220218"
+size = 5
+[db.mongoS]
+addr = "192.168.3.71:29099"
+dbname = "wjh"
+coll = "site"
+size = 5
+user = ""
+password = ""
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"

data_credible/config/conf.go

@@ -0,0 +1,73 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	// Conf crocodile conf
+	Conf *conf
+)
+
+// Init Config
+func Init(conf string) {
+	_, err := toml.DecodeFile(conf, &Conf)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+}
+
+type conf struct {
+	Serve serve
+	DB    Db
+	Log   Log
+}
+
+type serve struct {
+	Thread int
+	JyHref string
+	Fields []string
+}
+
+// Log Config
+type Log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type Db struct {
+	Mongo   mgo
+	MongoBz mgo
+	MongoE  mgo
+	MongoS  mgo
+}
+
+type mgo struct {
+	Addr     string
+	Dbname   string
+	Coll     string
+	Size     int
+	User     string
+	Password string
+}
+
+type duration struct {
+	time.Duration
+}
+
+// UnmarshalText parse 10s to time.Time
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}

data_credible/config/conf_test.go

@@ -0,0 +1,66 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var confs = `# log
+[log]
+logpath = ""
+maxsize = 10
+compress = true
+maxage =  7
+maxbackups = 10
+loglevel  = "info"
+format = "text"
+
+[db]
+[db.mongo]
+addr = "192.168.3.207:27092"
+dbname = "qfw"
+size = 10
+user = ""
+password = ""
+[db.mongo1]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+size = 5
+user = ""
+password = ""
+[db.mysql]
+addr = "192.168.3.217:4000"
+dbnameBasic = "global_common_data"
+dbnameMedical = "medical_fileld_data"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+
+[db.es]
+addr = "http://192.168.3.206:9800"
+size = 5
+indexm = "medical_institution_v1"
+typem = "medical_institution"
+indexs = "supplier_product_v1"
+types = "supplier_product"
+
+[db.es.fieldarrm]
+id = "int"
+company_id = "string"
+mi_name = "string"
+alias = "string"
+area_code = "string"
+level_code = "string"
+mi_type_code = "string"
+business_type = "int"
+sdequipment = "string"
+`
+
+func TestInit(t *testing.T) {
+	testfile := "/tmp/crocodile.toml"
+	ioutil.WriteFile(testfile, []byte(confs), 0644)
+	Init(testfile)
+	t.Logf("%+v", Conf.DB.Mongo.Dbname)
+	os.Remove(testfile)
+}

data_credible/go.mod

@@ -0,0 +1,35 @@
+module data_credible
+
+go 1.20
+
+require (
+	github.com/BurntSushi/toml v1.3.2
+	github.com/spf13/cobra v1.7.0
+	go.mongodb.org/mongo-driver v1.10.1
+	go.uber.org/zap v1.22.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+)

data_credible/go.sum

@@ -0,0 +1,210 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1 h1:MbtiS3BwDnB7uRjHFMCBe7/oW+aLaGMeQNiQa3qvd4c=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

data_credible/init.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"data_credible/config"
+	"fmt"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"os"
+)
+
+var (
+	Mongo, MongoS, MongoE, MongoBz *mongodb.MongodbSim
+
+	SiteMap map[string]interface{}
+	CodeMap map[string]interface{}
+
+	Dbname string
+	Coll   string
+
+	savePool chan map[string]interface{}
+	saveSp   chan bool
+	saveSize int
+)
+
+func init() {
+	config.Init("./common.toml")
+	InitLog()
+	Mongo = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.Mongo.Addr,
+		Size:        config.Conf.DB.Mongo.Size,
+		DbName:      config.Conf.DB.Mongo.Dbname,
+		UserName:    config.Conf.DB.Mongo.User,
+		Password:    config.Conf.DB.Mongo.Password,
+	}
+	Mongo.InitPool()
+	MongoBz = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoBz.Addr,
+		Size:        config.Conf.DB.MongoBz.Size,
+		DbName:      config.Conf.DB.MongoBz.Dbname,
+	}
+	MongoBz.InitPool()
+	MongoE = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoE.Addr,
+		Size:        config.Conf.DB.MongoE.Size,
+		DbName:      config.Conf.DB.MongoE.Dbname,
+	}
+	MongoE.InitPool()
+	MongoS = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoS.Addr,
+		Size:        config.Conf.DB.MongoS.Size,
+		DbName:      config.Conf.DB.MongoS.Dbname,
+	}
+	MongoS.InitPool()
+
+	SiteMap = make(map[string]interface{}, 50000)
+	CodeMap = make(map[string]interface{}, 80000)
+
+	savePool = make(chan map[string]interface{}, 5000)
+	saveSp = make(chan bool, 1)
+	saveSize = 200
+}
+
+// InitLog @Description
+// @Author J 2022/7/26 15:30
+func InitLog() {
+	logcfg := config.Conf.Log
+
+	err := log.InitLog(
+		log.Path(logcfg.LogPath),
+		log.Level(logcfg.LogLevel),
+		log.Compress(logcfg.Compress),
+		log.MaxSize(logcfg.MaxSize),
+		log.MaxBackups(logcfg.MaxBackups),
+		log.MaxAge(logcfg.MaxAge),
+		log.Format(logcfg.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+}

data_credible/main.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"data_credible/config"
+	"fmt"
+	"github.com/spf13/cobra"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"sync"
+	"time"
+)
+
+func main() {
+	loadSite()
+	go saveQueue()
+
+	rootCmd := &cobra.Command{Use: "my cmd"}
+	rootCmd.AddCommand(taskOld())
+	rootCmd.AddCommand(taskNew())
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println("rootCmd.Execute failed", err.Error())
+	}
+
+	c := make(chan bool, 1)
+	<-c
+}
+
+func taskOld() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "old",
+		Short: "Start dispose task",
+		Run: func(cmd *cobra.Command, args []string) {
+			sess := MongoS.GetMgoConn()
+			defer MongoS.DestoryMongoConn(sess)
+			ch := make(chan bool, config.Conf.Serve.Thread)
+			wg := &sync.WaitGroup{}
+
+			query := sess.DB(Dbname).C(Coll).Find(nil).Select(nil).Iter()
+			count := 0
+			for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+				if count%5000 == 0 {
+					log.Info(fmt.Sprintf("current --- %d", count))
+				}
+				ch <- true
+				wg.Add(1)
+				go func(temp map[string]interface{}) {
+					defer func() {
+						<-ch
+						wg.Done()
+					}()
+					task1(temp)
+				}(tmp)
+				tmp = make(map[string]interface{})
+			}
+			wg.Wait()
+			log.Info(fmt.Sprintf("over --- %d", count))
+		},
+	}
+	cmdClient.Flags().StringVarP(&Dbname, "dbname", "d", "", "库名")
+	cmdClient.Flags().StringVarP(&Coll, "coll", "c", "", "表名")
+	return cmdClient
+}
+
+func taskNew() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "new",
+		Short: "Start dispose task",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			sess := MongoBz.GetMgoConn()
+			defer MongoBz.DestoryMongoConn(sess)
+			ch := make(chan bool, config.Conf.Serve.Thread)
+			wg := &sync.WaitGroup{}
+
+			query := sess.DB(config.Conf.DB.MongoBz.Dbname).C(config.Conf.DB.MongoBz.Coll).Find(nil).Select(nil).Iter()
+			count := 0
+			for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+				if count%5000 == 0 {
+					log.Info(fmt.Sprintf("current --- %d", count))
+				}
+				ch <- true
+				wg.Add(1)
+				go func(temp map[string]interface{}) {
+					defer func() {
+						<-ch
+						wg.Done()
+					}()
+					if temp["v_taginfo"] != nil {
+						taskInfo(temp)
+					}
+				}(tmp)
+				tmp = make(map[string]interface{})
+			}
+			wg.Wait()
+			log.Info(fmt.Sprintf("over --- %d", count))
+		},
+	}
+	return cmdClient
+}
+
+func loadSite() {
+	f := bson.M{"site": 1, "site_type": 1, "second_type": 1, "platform": 1}
+	info, _ := MongoS.Find("site", nil, nil, f, false, -1, -1)
+	if len(*info) > 0 {
+		for _, m := range *info {
+			SiteMap[util.ObjToString(m["site"])] = fmt.Sprintf("%s-%s-%s", util.ObjToString(m["site_type"]),
+				util.ObjToString(m["second_type"]), util.ObjToString(m["platform"]))
+		}
+	}
+	log.Info("loadSite", zap.Int("SiteMap", len(SiteMap)))
+
+	info1, _ := MongoS.Find("luaconfig", nil, nil, bson.M{"code": 1, "platform": 1}, false, -1, -1)
+	if len(*info1) > 0 {
+		for _, m := range *info1 {
+			CodeMap[util.ObjToString(m["code"])] = util.ObjToString(m["platform"])
+		}
+	}
+	log.Info("loadSite", zap.Int("CodeMap", len(CodeMap)))
+}
+
+func saveQueue() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					Mongo.SaveBulk(config.Conf.DB.Mongo.Coll, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1 * time.Second):
+			if indexu > 0 {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					Mongo.SaveBulk(config.Conf.DB.Mongo.Coll, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

data_credible/task.go

@@ -0,0 +1,321 @@
+package main
+
+import (
+	"data_credible/config"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"strings"
+)
+
+func task1(tmp map[string]interface{}) {
+	id := util.ObjToString(tmp["id"])
+	save := make(map[string]interface{})
+	if id == "" {
+		id = mongodb.BsonIdToSId(tmp["_id"])
+	}
+	save["infoid"] = id
+	save["data_tag"] = 2
+	site := util.ObjToString(tmp["site"])
+	save["site"] = site
+	if SiteMap[site] != nil {
+		s1 := strings.Split(util.ObjToString(SiteMap[site]), "-")
+		if s1[0] != "" {
+			save["site_toptype"] = s1[0]
+		}
+		if s1[1] != "" {
+			save["site_subtype"] = s1[1]
+		}
+	} else {
+		//log.Error("siteMap not find", zap.String("site", site))
+	}
+	if CodeMap[util.ObjToString(tmp["spidercode"])] != nil {
+		save["collection_platform"] = CodeMap[util.ObjToString(tmp["spidercode"])]
+	} else {
+		//log.Error("codeMap not find", zap.String("spidercode", util.ObjToString(tmp["spidercode"])))
+	}
+	save["href"] = tmp["href"]
+	if tmp["jybxhref"] != nil {
+		save["jyhref"] = tmp["jybxhref"]
+	} else {
+		save["jyhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", id))
+	}
+	for _, s := range []string{"spidercode", "buyerclass", "toptype", "subtype", "s_topscopeclass", "s_subscopeclass", "area", "city", "multipackage"} {
+		if tmp[s] != nil {
+			save[s] = tmp[s]
+		}
+	}
+
+	p := util.Int64All(tmp["publishtime"])
+	save["publishtime"] = p
+	save["s_publishtime"] = util.FormatDateByInt64(&p, util.Date_Short_Layout)
+	if tmp["updatetime"] != nil {
+		u := util.Int64All(tmp["updatetime"])
+		save["updatetime"] = u
+		save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout)
+	} else {
+		u := util.Int64All(tmp["createtime"])
+		save["updatetime"] = u
+		save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout)
+	}
+	if util.ObjToString(tmp["purchasing"]) != "" {
+		save["is_purchasing"] = 1
+	} else {
+		save["is_purchasing"] = 0
+	}
+
+	tagM := make(map[string]interface{}) // 标注字段
+	check := make(map[string]interface{})
+	tag := 0 // 标注字段
+	tagC := 0
+	tlog, _ := MongoE.FindById(config.Conf.DB.MongoE.Coll, id, bson.M{"field_source": 0, "kvtext": 0})
+	blog, _ := Mongo.FindById("bidding", id, bson.M{"purchasinglist": 1, "procurementlist": 1})
+	for _, f := range config.Conf.Serve.Fields {
+		if tmp[fmt.Sprintf("ck_%s", f)] != nil {
+			tag++
+			tagM[f] = util.IntAll(tmp[fmt.Sprintf("ck_%s", f)])
+		}
+		if tmp[f] == nil {
+			continue
+		}
+		m := make(map[string]interface{})
+		if f == "package" {
+			if tagM[f] != nil {
+				tagC++
+				m["clean_flag"] = 1
+				m["is_mark"] = 2
+				execObj(f, save, tmp, *tlog)
+			} else {
+				execObj1(f, save, tmp, *tlog)
+				m["clean_flag"] = 0
+			}
+		} else if f == "purchasinglist" || f == "procurementlist" {
+			if tagM[f] != nil {
+				tagC++
+				m["clean_flag"] = 1
+				m["is_mark"] = 2
+				execObj(f, save, tmp, *blog)
+			} else {
+				execObj1(f, save, tmp, *blog)
+				m["clean_flag"] = 0
+			}
+		} else {
+			if tagM[f] != nil {
+				m["clean_flag"] = 1
+				t := util.IntAll(tagM[f])
+				if t == 1 {
+					m["is_mark"] = t
+				} else {
+					m["is_mark"] = 2
+				}
+				if t != 1 {
+					tagC++
+					if (*tlog)[f] != nil {
+						m["pre_value"] = (*tlog)[f]
+					}
+					m["aft_value"] = tmp[f]
+				} else {
+					m["pre_value"] = tmp[f]
+				}
+			} else {
+				m["pre_value"] = tmp[f]
+				m["clean_flag"] = 0
+			}
+		}
+		check[f] = m
+	}
+	save["need_field_num"] = tag
+	save["v_fieldcheck"] = check
+	save["clean_field_num"] = tagC
+	if len(save) > 0 {
+		savePool <- save
+	}
+}
+
+func taskInfo(tmp map[string]interface{}) {
+	if info, ok := tmp["v_baseinfo"].(map[string]interface{}); ok {
+		id := util.ObjToString(info["id"])
+		if id == "" {
+			id = util.ObjToString(tmp["id"])
+		}
+		save := make(map[string]interface{})
+		save["infoid"] = id
+		save["data_tag"] = 1
+		site := util.ObjToString(info["site"])
+		save["site"] = site
+		if SiteMap[site] != nil {
+			s1 := strings.Split(util.ObjToString(SiteMap[site]), "-")
+			if s1[0] != "" {
+				save["site_toptype"] = s1[0]
+			}
+			if s1[1] != "" {
+				save["site_subtype"] = s1[1]
+			}
+		} else {
+			//log.Error("siteMap not find", zap.String("site", site))
+		}
+		if CodeMap[util.ObjToString(info["spidercode"])] != nil {
+			save["collection_platform"] = CodeMap[util.ObjToString(info["spidercode"])]
+		} else {
+			//log.Error("codeMap not find", zap.String("spidercode", util.ObjToString(info["spidercode"])))
+		}
+		save["href"] = info["href"]
+		if tmp["jybxhref"] != nil {
+			save["jyhref"] = tmp["jybxhref"]
+		} else {
+			save["jyhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", id))
+		}
+		for _, s := range []string{"spidercode", "buyerclass", "toptype", "subtype", "s_topscopeclass", "s_subscopeclass", "area", "city", "multipackage"} {
+			if info[s] != nil {
+				save[s] = info[s]
+			}
+		}
+		p := util.Int64All(info["publishtime"])
+		save["publishtime"] = p
+		save["s_publishtime"] = util.FormatDateByInt64(&p, util.Date_Short_Layout)
+		if tmp["i_updatetime"] != nil {
+			u := util.Int64All(tmp["i_updatetime"])
+			save["updatetime"] = u
+			save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout)
+		} else {
+			u := util.Int64All(tmp["i_createtime"])
+			save["updatetime"] = u
+			save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout)
+		}
+		if util.ObjToString(info["purchasing"]) != "" {
+			save["is_purchasing"] = 1
+		} else {
+			save["is_purchasing"] = 0
+		}
+		tag := tmp["v_taginfo"].(map[string]interface{})
+		save["need_field_num"] = len(tag)
+		check := make(map[string]interface{})
+		tagC := 0
+		tlog, _ := MongoE.FindById(config.Conf.DB.MongoE.Coll, id, bson.M{"field_source": 0, "kvtext": 0})
+		blog, _ := Mongo.FindById("bidding", id, bson.M{"purchasinglist": 1, "procurementlist": 1, "toptype": 1, "subtype": 1})
+		save["toptype"] = (*blog)["toptype"]
+		save["subtype"] = (*blog)["subtype"]
+		for _, f := range config.Conf.Serve.Fields {
+			if info[f] == nil {
+				continue
+			}
+			m := make(map[string]interface{})
+			if f == "package" {
+				if tag[f] != nil {
+					tagC++
+					m["clean_flag"] = 1
+					m["is_mark"] = 2
+					execObj(f, save, info, *tlog)
+				} else {
+					execObj1(f, save, info, *tlog)
+					m["clean_flag"] = 0
+				}
+			} else if f == "purchasinglist" || f == "procurementlist" {
+				if tag[f] != nil {
+					tagC++
+					m["clean_flag"] = 1
+					m["is_mark"] = 2
+					execObj(f, save, info, *blog)
+				} else {
+					execObj1(f, save, tmp, *blog)
+					m["clean_flag"] = 0
+				}
+			} else {
+				if tag[f] != nil {
+					m["clean_flag"] = 1
+					t := util.IntAll(tag[f])
+					m["is_mark"] = t
+					if t != 1 {
+						tagC++
+						if (*tlog)[f] != nil {
+							m["pre_value"] = (*tlog)[f]
+						}
+						m["aft_value"] = info[f]
+					} else {
+						m["pre_value"] = info[f]
+					}
+				} else {
+					m["pre_value"] = info[f]
+					m["clean_flag"] = 0
+				}
+			}
+			check[f] = m
+		}
+		save["v_fieldcheck"] = check
+		save["clean_field_num"] = tagC
+		if len(save) > 0 {
+			savePool <- save
+		}
+	}
+}
+
+func execObj(f string, save, info, tinfo map[string]interface{}) {
+	if info[f] == nil {
+		return
+	}
+	m := make(map[string]interface{})
+	if f == "package" {
+		obj := info[f].(map[string]interface{})
+		m["aft_num"] = len(obj)
+		m["modify_num"] = len(obj)
+		m["aft_value"] = obj
+		if preObj, ok := tinfo[f].(map[string]interface{}); ok {
+			m["pre_num"] = len(preObj)
+			m["pre_value"] = preObj
+		} else {
+			m["pre_num"] = 0
+		}
+
+		if save["multipackage"] == nil {
+			if package1 := m["pre_value"]; package1 != nil {
+				packageM, _ := package1.(map[string]interface{})
+				for _, p := range packageM {
+					pm, _ := p.(map[string]interface{})
+					if util.ObjToString(pm["winner"]) != "" || util.Float64All(pm["budget"]) > 0 ||
+						util.Float64All(pm["bidamount"]) > 0 {
+						save["multipackage"] = 1
+						break
+					}
+				}
+			} else {
+				save["multipackage"] = 0
+			}
+		}
+	} else {
+		obj := info[f].([]interface{})
+		m["aft_num"] = len(obj)
+		m["modify_num"] = len(obj)
+		m["aft_value"] = obj
+		if preObj, ok := tinfo[f].([]interface{}); ok {
+			m["pre_num"] = len(preObj)
+			m["pre_value"] = preObj
+		} else {
+			m["pre_num"] = 0
+		}
+	}
+	save[f] = m
+}
+
+func execObj1(f string, save, info, tinfo map[string]interface{}) {
+	if info[f] == nil {
+		return
+	}
+	m := make(map[string]interface{})
+	if f == "package" {
+		if preObj, ok := tinfo[f].(map[string]interface{}); ok {
+			m["pre_num"] = len(preObj)
+			m["pre_value"] = preObj
+		} else {
+			m["pre_num"] = 0
+		}
+	} else {
+		if preObj, ok := tinfo[f].([]interface{}); ok {
+			m["pre_num"] = len(preObj)
+			m["pre_value"] = preObj
+		} else {
+			m["pre_num"] = 0
+		}
+	}
+	save[f] = m
+}

field_purchasing/go.mod

@@ -3,7 +3,6 @@ module field_purchasing
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d // indirect
-	app.yhyue.com/data_processing/field_info_tag v1.0.2 // indirect
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
 	github.com/BurntSushi/toml v1.2.1 // indirect
 )

field_purchasing/init.go

@@ -1,11 +1,11 @@
 package main
 
 import (
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/mysqldb"
 	"field_purchasing/config"
 	"fmt"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
 	"os"
 )
 

fieldproject_com/go.mod

@@ -3,7 +3,7 @@ module fieldproject_common
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
 	github.com/BurntSushi/toml v1.2.0
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/satori/go.uuid v1.2.0
@@ -11,5 +11,5 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.mongodb.org/mongo-driver v1.10.1
 	go.uber.org/zap v1.22.0
-	gopkg.in/olivere/elastic.v2 v2.0.61
+	github.com/olivere/elastic/v7 v7.0.32
 )

fieldproject_com/init.go

@@ -1,14 +1,14 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/elastic"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/mysqldb"
 	"fieldproject_common/config"
 	"fmt"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
 	"os"
 )
 

fieldproject_com/main.go

@@ -1,16 +1,16 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/elastic"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/mysqldb"
 	"fieldproject_common/config"
 	"fmt"
 	"github.com/spf13/cobra"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
 	"sync"
 	"time"
 )

fieldproject_com/task.go

@@ -1,15 +1,15 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
-	"app.yhyue.com/data_processing/common_utils/mongodb"
-	"app.yhyue.com/data_processing/common_utils/redis"
 	"fieldproject_common/config"
 	"fmt"
 	uuid "github.com/satori/go.uuid"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
 	"strings"
 	"sync"
 	"time"
@@ -953,7 +953,7 @@ func SaveEs(i, t string) {
 					defer func() {
 						<-SP
 					}()
-					Es.BulkSave(i, t, &arru, false)
+					Es.BulkSave(i, arru)
 				}(arru)
 				arru = make([]map[string]interface{}, 500)
 				indexu = 0
@@ -965,7 +965,7 @@ func SaveEs(i, t string) {
 					defer func() {
 						<-SP
 					}()
-					Es.BulkSave(i, t, &arru, false)
+					Es.BulkSave(i, arru)
 				}(arru[:indexu])
 				arru = make([]map[string]interface{}, 500)
 				indexu = 0
@@ -988,7 +988,7 @@ func updateEsMethod() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk("bidding", "bidding", arru...)
+					Es.UpdateBulk("bidding", arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0
@@ -1000,7 +1000,7 @@ func updateEsMethod() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk("bidding", "bidding", arru...)
+					Es.UpdateBulk("bidding", arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0

fieldproject_com/task1.go

@@ -1,12 +1,13 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
+	"context"
 	"encoding/json"
 	"fieldproject_common/config"
 	"fmt"
-	es "gopkg.in/olivere/elastic.v2"
+	es "github.com/olivere/elastic/v7"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"strconv"
 	"sync"
 	"time"
@@ -303,14 +304,14 @@ func taskBiddingData() {
 	esquery := `{"query":{"bool":{"must":[],"must_not":[{"constant_score":{"filter":{"missing":{"field":"bid_field"}}}}],"should":[{"match_all":{}}]}},"from":0,"size":10,"sort":[],"facets":{}}`
 
 	q := es.NewRawStringQuery(esquery)
-	util.Debug(Es.Count("bidding", "bidding", q))
+	util.Debug(Es.Count("bidding", q))
 	countDocs := 0
-	res, err := client.Scroll().Index("bidding").Type("bidding").Query(q).Size(200).Do() //查询一条获取游标
+	res, err := client.Scroll().Index("bidding").Query(q).Size(200).Do(context.TODO()) //查询一条获取游标
 	if err == nil {
 		taskInfoA(res, wg, &countDocs)
 		scrollId := res.ScrollId
 		for {
-			searchResult, err := client.Scroll("1m").ScrollId(scrollId).Size(200).Do() //查询
+			searchResult, err := client.Scroll("1m").ScrollId(scrollId).Size(200).Do(context.TODO()) //查询
 			if err != nil {
 				util.Debug("Es Search Data Error：", err)
 				break
@@ -320,7 +321,7 @@ func taskBiddingData() {
 		}
 		wg.Wait()
 		util.Debug("over---", countDocs)
-		_, _ = client.ClearScroll().ScrollId(scrollId).Do() //清理游标
+		_, _ = client.ClearScroll().ScrollId(scrollId).Do(context.TODO()) //清理游标
 	} else {
 		util.Debug(err)
 	}
@@ -340,7 +341,7 @@ func taskInfoA(searchResult *es.SearchResult, wg *sync.WaitGroup, countDocs *int
 				wg.Done()
 			}()
 			tmp := make(map[string]interface{})
-			if json.Unmarshal(*tmpHit.Source, &tmp) == nil {
+			if json.Unmarshal(tmpHit.Source, &tmp) == nil {
 				id := util.ObjToString(tmp["_id"])
 				if tmp["entidlist"] != nil && tmp["purchasinglist"] != nil {
 					for _, eid := range util.ObjArrToStringArr(tmp["entidlist"].([]interface{})) {

fieldproject_com/taskEs.go

@@ -1,11 +1,11 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
-	"app.yhyue.com/data_processing/common_utils/log"
 	"fmt"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"sync"
 )
 

fieldproject_inc_data/field_inc_data

@@ -1 +0,0 @@
-ELF

proposed_project/go.mod

@@ -3,11 +3,11 @@ module proposed_project
 go 1.16
 
 require (
-	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
 	github.com/BurntSushi/toml v1.2.0
 	github.com/go-ego/gse v0.80.2
 	github.com/robfig/cron v1.2.0
 	github.com/spf13/cobra v1.6.1
 	go.mongodb.org/mongo-driver v1.10.3
 	go.uber.org/zap v1.23.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
 )

proposed_project/go.sum

@@ -1,9 +1,3 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20230308013947-0ef15d576c68 h1:560oa2np5fPKQOGvsOYMV9vNQ8Y6y9O+WlUoNbnZ0UU=
-app.yhyue.com/data_processing/common_utils v0.0.0-20230308013947-0ef15d576c68/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
-app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061 h1:UaE1gor8fIhSBAXcZsNOQDMI5sD9LxGECXFh7Ik4rk4=
-app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
-app.yhyue.com/data_processing/common_utils v0.0.0-20230519053026-75421af90e41 h1:MRE2bNcC4rzlzkMozqeAqyMazU/t8EHwH0vp/6W49JE=
-app.yhyue.com/data_processing/common_utils v0.0.0-20230519053026-75421af90e41/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
@@ -230,3 +224,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1 h1:MbtiS3BwDnB7uRjHFMCBe7/oW+aLaGMeQNiQa3qvd4c=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

proposed_project/main.go

@@ -186,13 +186,13 @@ func tidbSave() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			InitMysql()
 			InitArea()
-			go SaveFunc("dwd_f_nzj_baseinfo_new", BaseField)
-			go SaveRFunc("dwd_f_nzj_follw_record_new", RecordField)
-			go SaveCFunc("dwd_f_nzj_contact_new", ContactField)
-			go SaveCyFunc("dwd_f_nzj_category_tags_new", CategoryField)
-			go SaveEntFunc("dwd_f_nzj_ent_new", EntField)
+			go SaveFunc("dwd_f_nzj_baseinfo", BaseField)
+			go SaveRFunc("dwd_f_nzj_follw_record", RecordField)
+			go SaveCFunc("dwd_f_nzj_contact", ContactField)
+			go SaveCyFunc("dwd_f_nzj_category_tags", CategoryField)
+			go SaveEntFunc("dwd_f_nzj_ent", EntField)
 
-			redis.InitRedis1("ent_id=172.17.4.189:8379", 6) // name_id
+			//redis.InitRedis1("ent_id=172.17.4.189:8379", 6) // name_id
 
 			taskTidb(nil)
 			c := make(chan bool, 1)
@@ -371,14 +371,14 @@ func projectCombAdd() *cobra.Command {
 			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Proposed, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbd)
 			go SaveEntFunc1("dwd_f_nzj_ent", EntField)
 
-			go taskAA()
-			go taskBB()
+			//go taskAA()
+			//go taskBB()
 
 			crn := cron.New()
 			cronstr := "0 0 22 * * ?" // 每天10点执行一次
 			_ = crn.AddFunc(cronstr, func() {
 				if TaskSingle {
-					go taskAA()
+					//go taskAA()
 					go taskBB()
 				} else {
 					log.Info("上次任务未执行完成")

proposed_project/projectTask.go

@@ -676,6 +676,7 @@ func taskBB() {
 	wg := &sync.WaitGroup{}
 
 	q := bson.M{"pici": bson.M{"$gte": config.Conf.Serve.Pici}}
+	log.Info(fmt.Sprint(q), zap.Int("count", MgoPro.Count(config.Conf.DB.MongoP.ProposedColl, q)))
 	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(q).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {

proposed_project/xx_test.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	util "app.yhyue.com/data_processing/common_utils"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"testing"
 )
 

tieta_data/api_task.go

@@ -0,0 +1,143 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"net/http"
+	"tieta_data/config"
+)
+
+type Req struct {
+	Ids []string `json:"ids"`
+}
+
+type Resp struct {
+	Status  int    `json:"status"`
+	Message string `json:"message"`
+	Data    string `json:"data"`
+}
+
+func MergeProject(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	var res Req
+	if err := json.NewDecoder(req.Body).Decode(&res); err != nil {
+		_ = req.Body.Close()
+	}
+	log.Info("MergeProject", zap.Any("req", res))
+	//var result Resp
+	var result Resp
+	if len(res.Ids) > 1 {
+		// 重新捏合字段
+		pid := MergeMethod(res.Ids)
+		if pid != "" {
+			result = Resp{Status: 1, Message: "重新合并成功", Data: pid}
+		} else {
+			result = Resp{Status: -1, Message: "重新合并失败", Data: pid}
+		}
+	} else {
+		result = Resp{Status: -1, Message: "参数有误"}
+	}
+	if err := json.NewEncoder(w).Encode(result); err != nil {
+		log.Error("MergeProject", zap.Error(err))
+	}
+}
+
+func MergeMethod(ids []string) string {
+	newId := ""
+	for i, id := range ids {
+		log.Debug("项目id,", zap.String("id: ", id))
+		// 1、删除旧有项目
+		AllIdsMapLock.Lock()
+		delete(AllIdsMap, id)
+		AllIdsMapLock.Unlock()
+		b := MysqlTool.Delete(config.Conf.DB.Mysql.Pcoll, bson.M{"projectId": id})
+		if !b {
+			return ""
+		}
+		binfo := MysqlTool.Find(config.Conf.DB.Mysql.Coll, bson.M{"projectId": id}, "", "", -1, -1)
+		// 2、id标讯数据重新合并
+		for j, m := range *binfo {
+			info := ParseInfo(m)
+			if i == 0 && j == 0 {
+				var p1 *Project
+				newId, p1 = NewProject(m, info)
+				AllIdsMapLock.Lock()
+				AllIdsMap[newId] = &ID{Id: newId, P: p1}
+				AllIdsMapLock.Unlock()
+			} else {
+				AllIdsMapLock.Lock()
+				res := AllIdsMap[newId]
+				AllIdsMapLock.Unlock()
+				if res != nil {
+					comparePro := res.P
+					UpdateProject(m, info, comparePro, -1, "AAAAAAAAAA")
+				}
+			}
+		}
+	}
+	log.Debug(fmt.Sprintf("newId: %s", newId))
+	return newId
+}
+
+type UpReq struct {
+	Nid  string `json:"nid"`
+	Type int    `json:"type"` // 0：修改；1：删除
+}
+
+type UpResp struct {
+	Status  int    `json:"status"`
+	Message string `json:"message"`
+}
+
+func UpdateInfo(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	var res UpReq
+	if err := json.NewDecoder(req.Body).Decode(&res); err != nil {
+		_ = req.Body.Close()
+	}
+	log.Info("UpdateInfo", zap.Any("req", res))
+	var result UpResp
+	if res.Nid != "" {
+		if res.Type == 1 {
+			MysqlTool.Delete(config.Conf.DB.Mysql.Pcoll, bson.M{"projectId": res.Nid})
+			result = UpResp{Status: 1, Message: "删除成功"}
+		} else {
+			if info := MysqlTool.FindOne(config.Conf.DB.Mysql.Coll, bson.M{"id_new": res.Nid}, "", ""); info != nil && len(*info) > 0 {
+				if pid := util.ObjToString((*info)["projectId"]); pid != "" {
+					if pinfo := MysqlTool.Find(config.Conf.DB.Mysql.Pcoll, bson.M{"projectId": pid}, "", "", -1, -1); pinfo != nil && len(*pinfo) > 0 {
+						info1 := ParseInfo(*info)
+						if len(*pinfo) > 1 {
+							AllIdsMapLock.Lock()
+							res := AllIdsMap[pid]
+							AllIdsMapLock.Unlock()
+							if res != nil {
+								comparePro := res.P
+								UpdateProject(*info, info1, comparePro, -1, "AAAAAAAAAA")
+							}
+						} else {
+							// 删除旧有项目
+							AllIdsMapLock.Lock()
+							delete(AllIdsMap, pid)
+							AllIdsMapLock.Unlock()
+							MysqlTool.Delete(config.Conf.DB.Mysql.Pcoll, bson.M{"projectId": pid})
+							startProjectMerge(info1, *info)
+						}
+					}
+				}
+				result = UpResp{Status: 1, Message: "重新合并成功"}
+			} else {
+				result = UpResp{Status: -1, Message: "未查询到项目信息"}
+			}
+		}
+	} else {
+		result = UpResp{Status: -1, Message: "参数有误"}
+	}
+
+	if err := json.NewEncoder(w).Encode(result); err != nil {
+		log.Error("UpdateInfo", zap.Error(err))
+	}
+}

tieta_data/common.toml

@@ -4,14 +4,21 @@ jyhref = "https://www.jianyu360.cn/article/content/%s.html"
 
 [db]
 
-[db.mongo]
-addr = "192.168.3.166:27082"
-dbname = "zhaoxiuzhen"
-coll = "20230802Tt_pro"
-pcoll = "20230802Tt_project"
+[db.mysql]
+addr = "192.168.3.14:4000"
+dbname = "bi_service"
+coll = "customer_data_ttzl_dev"
+pcoll = "customer_data_ttzl_project_dev"
 size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+
+[db.es]
+addr = "http://192.168.3.241:9205"
 user = ""
 password = ""
+size = 5
+index = "ttbid"
 
 # 日志
 [log]

tieta_data/config/conf.go

@@ -46,6 +46,8 @@ type log struct {
 
 type db struct {
 	Mongo mgo
+	Mysql mysql
+	Es    es
 }
 
 type mgo struct {
@@ -58,11 +60,22 @@ type mgo struct {
 	Password string
 }
 
+type mysql struct {
+	Addr     string
+	Dbname   string
+	Coll     string
+	Pcoll    string
+	Size     int
+	User     string
+	Password string
+}
+
 type es struct {
-	Addr   string
-	Size   int
-	IndexS string
-	TypeS  string
+	Addr     string
+	User     string
+	Password string
+	Size     int
+	Index    string
 }
 
 type duration struct {

tieta_data/go.mod

@@ -4,26 +4,35 @@ go 1.20
 
 require (
 	github.com/BurntSushi/toml v1.2.0
+	github.com/google/uuid v1.1.2
+	github.com/robfig/cron v1.2.0
 	github.com/spf13/cobra v1.7.0
 	github.com/tealeg/xlsx v1.0.5
-	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061
+	go.mongodb.org/mongo-driver v1.10.1
+	go.uber.org/zap v1.22.0
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c
+)
+
+require (
 	github.com/PuerkitoBio/goquery v1.8.0 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.mongodb.org/mongo-driver v1.10.1 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.22.0 // indirect
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect

tieta_data/go.sum

@@ -22,9 +22,11 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -51,11 +53,13 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -64,10 +68,12 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -76,6 +82,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
+github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
@@ -208,5 +216,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061 h1:xfmoVsDEqVv1XzAVxgGEDi+W9ojHBJc6OmTbg3b1tP0=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230620072956-7ec055be2061/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c h1:fgH3lMOi2jG3KCejzC3d0otkJB6mJ9eaq1HecMoHG4c=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230915054514-628d4fe7544c/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

tieta_data/init.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"fmt"
-	"go.mongodb.org/mongo-driver/bson/primitive"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
 	"os"
 	"sync"
 	"tieta_data/config"
@@ -12,6 +14,8 @@ import (
 
 var (
 	MongoTool *mongodb.MongodbSim
+	MysqlTool *mysqldb.Mysql
+	Es        *elastic.Elastic
 
 	findLock            sync.Mutex
 	mapPb, mapPn, mapPc map[string]*Key
@@ -246,15 +250,26 @@ func init() {
 	}
 }
 
-func InitMgo() {
-	MongoTool = &mongodb.MongodbSim{
-		MongodbAddr: config.Conf.DB.Mongo.Addr,
-		Size:        config.Conf.DB.Mongo.Size,
-		DbName:      config.Conf.DB.Mongo.Dbname,
-		UserName:    config.Conf.DB.Mongo.User,
-		Password:    config.Conf.DB.Mongo.Password,
+func InitEs() {
+	util.Debug(config.Conf.DB.Es.Addr, config.Conf.DB.Es.Index)
+	Es = &elastic.Elastic{
+		S_esurl:  config.Conf.DB.Es.Addr,
+		I_size:   config.Conf.DB.Es.Size,
+		Username: config.Conf.DB.Es.User,
+		Password: config.Conf.DB.Es.Password,
 	}
-	MongoTool.InitPool()
+	Es.InitElasticSize()
+}
+
+func InitMysql() {
+	dbcfg := config.Conf.DB.Mysql
+	MysqlTool = &mysqldb.Mysql{
+		Address:  dbcfg.Addr,
+		DBName:   dbcfg.Dbname,
+		UserName: dbcfg.User,
+		PassWord: dbcfg.Password,
+	}
+	MysqlTool.Init()
 }
 
 func InitLog() {
@@ -276,77 +291,92 @@ func InitLog() {
 }
 
 type Project struct {
-	Id            primitive.ObjectID `json:"_id"`
-	Ids           []string           `json:"ids,omitempty"`
-	FirstTime     int64              `json:"firsttime,omitempty"`   //项目的最早时间
-	LastTime      int64              `json:"lasttime,omitempty"`    //项目的最后时间
-	ProjectName   string             `json:"projectname,omitempty"` //项目名称
-	ProjectCode   string             `json:"projectcode,omitempty"` //项目代码
-	Buyer         string             `json:"buyer,omitempty"`       //采购单位
-	Agency        string             `json:"agency"`                //代理机构
-	Area          string             `json:"area"`                  //地区
-	City          string             `json:"city"`                  //地市
-	District      string             `json:"district"`              //区县
-	Bidamount     float64            `json:"bidamount,omitempty"`   //中标金额
-	Budget        float64            `json:"budget,omitempty"`      //预算
-	score         int
-	comStr        string
-	MPN           []string `json:"mpn,omitempty"` //合并后多余的项目名称
-	MPC           []string `json:"mpc,omitempty"` //合并后多余的项目编号
-	resVal, pjVal int
-
-	Topscopeclass []string `json:"topscopeclass,omitempty"`
-	Subscopeclass []string `json:"subscopeclass,omitempty"` //子行业分类
-	Winners       string   `json:"s_winner,omitempty"`      //中标人
-	Buyerperson   string   `json:"buyerperson"`             //采购联系人
-	Buyertel      string   `json:"buyertel"`                //采购联系人电话
-	AgencyPerson  string   `json:"agencyperson"`
-	AgencyTel     string   `json:"agencytel"`
-	WinnerPerson  string   `json:"winnerperson"`
-	WinnerTel     string   `json:"winnertel"`
-	Buyerclass    string   `json:"buyerclass"`            //采购单位分类
-	Bidopentime   int64    `json:"bidopentime,omitempty"` //开标时间
-	Jgtime        int64    `json:"jgtime"`                //结果中标时间
-	Zbtime        int64    `json:"zbtime"`                //招标时间
+	MsgId          string  `json:"msg_Id"`
+	ProjectId      string  `json:"projectId"`
+	FirstTime      int64   `json:"firsttime,omitempty"`   //项目的最早时间
+	LastTime       int64   `json:"lasttime,omitempty"`    //项目的最后时间
+	ProjectName    string  `json:"projectname,omitempty"` //项目名称
+	ProjectCode    string  `json:"projectcode,omitempty"` //项目代码
+	Buyer          string  `json:"buyer,omitempty"`       //采购单位
+	Agency         string  `json:"agency"`                //代理机构
+	Winners        string  `json:"s_winner,omitempty"`    //中标人
+	Area           string  `json:"area"`                  //地区
+	City           string  `json:"city"`                  //地市
+	District       string  `json:"district"`              //区县
+	Bidamount      float64 `json:"bidamount,omitempty"`   //中标金额
+	Budget         float64 `json:"budget,omitempty"`      //预算
+	score          int
+	comStr         string
+	resVal, pjVal  int
+	Topscopeclass  []string `json:"topscopeclass,omitempty"`
+	Subscopeclass  []string `json:"subscopeclass,omitempty"` //子行业分类
+	Buyerperson    string   `json:"buyerperson"`             //采购联系人
+	Buyertel       string   `json:"buyertel"`                //采购联系人电话
+	AgencyPerson   string   `json:"agencyperson"`
+	AgencyTel      string   `json:"agencytel"`
+	WinnerPerson   string   `json:"winnerperson"`
+	WinnerTel      string   `json:"winnertel"`
+	Buyerclass     string   `json:"buyerclass"`            //采购单位分类
+	Bidopentime    int64    `json:"bidopentime,omitempty"` //开标时间
+	Jgtime         int64    `json:"jgtime"`                //结果中标时间
+	Zbtime         int64    `json:"zbtime"`                //招标时间
+	MultiPackage   int      `json:"multipackage"`          // 多包标记
+	MultiPackageId string   `json:"multipackageid"`
 
-	SceneTag string `json:"scenetag"`  //业务场景
-	BuyerTag string `json:"buyertag"`  //招标单位行业归类
-	TagName  string `json:"tagname"`   //招标单位一级标签
-	TagName2 string `json:"tagname_2"` //招标单位二级行业
-	TagName3 string `json:"tagname_3"` //中标单位行业归类
+	ZbInfoId     string `json:"zb_infoid"`     //招标公告唯一标识
+	ZbUpdateTime int64  `json:"zb_updatetime"` //招标一级公告类型
+	ZbTopType    string `json:"zb_toptype"`    //招标二级公告类型
+	ZbHref       string `json:"zb_href"`
+	ZbJybxHref   string `json:"zb_jybxhref"`
+	InfoId       string `json:"infoid"`     //中标公告唯一标识
+	UpdateTime   string `json:"updatetime"` //中标公告发布时间
+	TopType      string `json:"toptype"`    //中标一级公告类型
+	SubType      string `json:"subtype"`    //中标二级公告类型
+	Href         string `json:"href"`
+	JybxHref     string `json:"jybxhref"`
+	SceneTag     string `json:"scenetag"` //业务场景
+	BuyerTag     string `json:"buyertag"` //招标单位行业归类
+	TagName      string `json:"tagname"`  //招标单位一级标签
+	TagName2     string `json:"tagname2"` //招标单位二级行业
+	TagName3     string `json:"tagname3"` //中标单位行业归类
 }
 
 type Info struct {
-	Id            string   `json:"id"`
-	Href          string   `json:"href"` //源地址
-	Publishtime   int64    `json:"publishtime"`
-	Comeintime    int64    `json:"comeintime"`
-	Title         string   `json:"title"`
-	TopType       string   `json:"toptype"`
-	SubType       string   `json:"subtype"`
-	ProjectName   string   `json:"projectname"`
-	ProjectCode   string   `json:"projectcode"`
-	Buyer         string   `json:"buyer"`
-	Buyerperson   string   `json:"buyerperson"`
-	Buyertel      string   `json:"buyertel"`
-	Agency        string   `json:"agency"`
-	Area          string   `json:"area"`
-	City          string   `json:"city"`
-	District      string   `json:"district"`
-	Topscopeclass []string `json:"topscopeclass"`
-	Subscopeclass []string `json:"subscopeclass"`
-	Buyerclass    string   `json:"buyerclass"`
-	Bidopentime   int64    `json:"bidopentime"`
-	Budget        float64  `json:"budget"`
-	Bidamount     float64  `json:"bidamount"`
-	Winners       []string
-	PTC           string //从标题中抽的项目编号
-	pnbval        int    //项目名称、编号、采购单位存在的个数
-	LenPC         int    //项目编号长度
-	LenPN         int    //项目名称长度
-	LenPTC        int    //标题抽的项目编号长度
-	//以下三个元素做对比，计算包含时候使用
-	PNBH  int //0初始，+包含，-被包含
-	PCBH  int
-	PTCBH int
+	MsgId          int64   `json:"msg_id"`
+	Id             string  `json:"id"`
+	RuleName       string  `json:"rulename"`
+	TagName        string  `json:"tagname"`
+	TagName2       string  `json:"tagname2"`
+	TagName3       string  `json:"tagname3"`
+	Href           string  `json:"href"` //源地址
+	JybxHref       string  `json:"jybxhref"`
+	Publishtime    int64   `json:"publishtime"`
+	Comeintime     int64   `json:"comeintime"`
+	Title          string  `json:"title"`
+	TopType        string  `json:"toptype"`
+	SubType        string  `json:"subtype"`
+	ProjectName    string  `json:"projectname"`
+	ProjectCode    string  `json:"projectcode"`
+	Buyer          string  `json:"buyer"`
+	Buyerperson    string  `json:"buyerperson"`
+	Buyertel       string  `json:"buyertel"`
+	WinnerTel      string  `json:"winnertel"`
+	WinnerPerson   string  `json:"winnerperson"`
+	Agency         string  `json:"agency"`
+	AgencyPerson   string  `json:"agencyperson"`
+	AgencyTel      string  `json:"agencytel"`
+	Area           string  `json:"area"`
+	City           string  `json:"city"`
+	District       string  `json:"district"`
+	Buyerclass     string  `json:"buyerclass"`
+	Budget         float64 `json:"budget"`
+	Bidamount      float64 `json:"bidamount"`
+	MultiPackage   int     `json:"multipackage"`
+	MultiPackageId string  `json:"multipackage_id"`
+	Nid            string  `json:"id_new"`
+	Winners        []string
+	pnbval         int //项目名称、编号、采购单位存在的个数
+	LenPC          int //项目编号长度
+	LenPN          int //项目名称长度
+	LenPTC         int //标题抽的项目编号长度
 }

tieta_data/main.go

@@ -1,37 +1,24 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
+	"github.com/robfig/cron"
 	"github.com/spf13/cobra"
 	"github.com/tealeg/xlsx"
 	"go.uber.org/zap"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
-	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
-	"net"
+	"net/http"
 	"tieta_data/config"
-	"time"
-)
-
-var (
-	udpClient udp.UdpClient
-
-	updatePool chan []map[string]interface{}
-	updateSp   chan bool
-	saveSize   int
 )
 
 func init() {
 	config.Init("./common.toml")
 	InitLog()
-
-	updatePool = make(chan []map[string]interface{}, 5000)
-	updateSp = make(chan bool, 1)
-	saveSize = 200
 }
 
 func main() {
+
 	rootCmd := &cobra.Command{Use: "my cmd"}
 	rootCmd.AddCommand(exportB())
 	rootCmd.AddCommand(exportP())
@@ -39,6 +26,7 @@ func main() {
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println("rootCmd.Execute failed", err.Error())
 	}
+	rootCmd.InitDefaultCompletionCmd()
 }
 
 func exportB() *cobra.Command {
@@ -46,7 +34,7 @@ func exportB() *cobra.Command {
 		Use:   "export-bidding",
 		Short: "Start export task",
 		Run: func(cmd *cobra.Command, args []string) {
-			InitMgo()
+			//InitMgo()
 
 			sess := MongoTool.GetMgoConn()
 			defer MongoTool.DestoryMongoConn(sess)
@@ -88,7 +76,7 @@ func exportP() *cobra.Command {
 		Use:   "export-project",
 		Short: "Start export task",
 		Run: func(cmd *cobra.Command, args []string) {
-			InitMgo()
+			//InitMgo()
 
 			sess := MongoTool.GetMgoConn()
 			defer MongoTool.DestoryMongoConn(sess)
@@ -130,13 +118,19 @@ func project() *cobra.Command {
 		Use:   "project",
 		Short: "Start project task",
 		Run: func(cmd *cobra.Command, args []string) {
-			InitMgo()
+			InitMysql()
+			InitEs()
 
-			go updateAllQueue()
+			go ApiMethod()
 			loadData()
-			udpClient = udp.UdpClient{Local: config.Conf.Serve.LocPort, BufSize: 1024}
-			udpClient.Listen(processUdpMsg)
-			log.Info("Udp服务监听", zap.String("port:", config.Conf.Serve.LocPort))
+
+			taskProject()
+			crn := cron.New()
+			cronstr := "0 0 10 * * ?"
+			_ = crn.AddFunc(cronstr, func() {
+				taskProject()
+			})
+			crn.Start()
 
 			ch := make(chan bool, 1)
 			<-ch
@@ -146,61 +140,11 @@ func project() *cobra.Command {
 	return cmdClient
 }
 
-func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
-	switch act {
-	case udp.OP_TYPE_DATA: //上个节点的数据
-		var mapInfo map[string]interface{}
-		err := json.Unmarshal(data, &mapInfo)
-		log.Info("err:", zap.Error(err), zap.Any("mapInfo:", mapInfo))
-		if err != nil {
-			_ = udpClient.WriteUdp([]byte("err:"+err.Error()), udp.OP_NOOP, ra)
-		} else if mapInfo != nil {
-			key, _ := mapInfo["key"].(string)
-			if key == "" {
-				key = "udpok"
-			}
-			go udpClient.WriteUdp([]byte(key), udp.OP_NOOP, ra)
-			taskProject(mapInfo)
-		}
-	case udp.OP_NOOP: //下个节点回应
-		ok := string(data)
-		if ok != "" {
-			log.Info("re", zap.String("ok:", ok))
-		}
-	}
-}
-
-func updateAllQueue() {
-	arru := make([][]map[string]interface{}, saveSize)
-	indexu := 0
-	for {
-		select {
-		case v := <-updatePool:
-			arru[indexu] = v
-			indexu++
-			if indexu == saveSize {
-				updateSp <- true
-				go func(arru [][]map[string]interface{}) {
-					defer func() {
-						<-updateSp
-					}()
-					MongoTool.UpSertBulk(config.Conf.DB.Mongo.Pcoll, arru...)
-				}(arru)
-				arru = make([][]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		case <-time.After(1 * time.Second):
-			if indexu > 0 {
-				updateSp <- true
-				go func(arru [][]map[string]interface{}) {
-					defer func() {
-						<-updateSp
-					}()
-					MongoTool.UpSertBulk(config.Conf.DB.Mongo.Pcoll, arru...)
-				}(arru[:indexu])
-				arru = make([][]map[string]interface{}, saveSize)
-				indexu = 0
-			}
-		}
+func ApiMethod() {
+	http.HandleFunc("/mergeProject", MergeProject)
+	http.HandleFunc("/updateProject", UpdateInfo)
+	err := http.ListenAndServe(config.Conf.Serve.LocPort, nil)
+	if err != nil {
+		log.Fatal("ListenAndServe: ", zap.Error(err))
 	}
 }

tieta_data/merge_comparepnc.go

@@ -12,9 +12,6 @@ func comparePNC(info *Info, compareProject *Project) (compareStr string, score i
 		if compareProject.ProjectName != "" {
 			pns = append(pns, compareProject.ProjectName)
 		}
-		if len(compareProject.MPN) > 0 {
-			pns = append(pns, compareProject.MPN...)
-		}
 		ifind := 0
 		templen := 0
 		for _, v := range pns {
@@ -55,7 +52,6 @@ func comparePNC(info *Info, compareProject *Project) (compareStr string, score i
 			if templen > info.LenPN {
 				templen = info.LenPN
 			}
-			info.PNBH = templen
 			if templen > 12 {
 				score += 1
 			}
@@ -71,15 +67,12 @@ func comparePNC(info *Info, compareProject *Project) (compareStr string, score i
 				init_text = ["号","(重)","(第二次)","（重）"]
 		all_clean_mark = ["[","(","【","（","〖","]",")","】","）","〗","-","〔","〕","《","［","］","｛","｝","{","—"," ","－","﹝","﹞","–"]
 	*/
-	for index, pc := range []string{info.ProjectCode, info.PTC} {
+	for index, pc := range []string{info.ProjectCode, ""} {
 		if pc != "" {
 			pcs := []string{}
 			if compareProject.ProjectCode != "" {
 				pcs = append(pcs, compareProject.ProjectCode)
 			}
-			if len(compareProject.MPC) > 0 {
-				pcs = append(pcs, compareProject.MPC...)
-			}
 			ifind := 0
 			templen := 0
 			for _, v := range pcs {
@@ -126,7 +119,6 @@ func comparePNC(info *Info, compareProject *Project) (compareStr string, score i
 					if templen > info.LenPC {
 						templen = info.LenPC
 					}
-					info.PCBH = templen
 					if templen > 12 {
 						score += 1
 					}
@@ -135,7 +127,6 @@ func comparePNC(info *Info, compareProject *Project) (compareStr string, score i
 					if templen > info.LenPTC {
 						templen = info.LenPTC
 					}
-					info.PTCBH = templen
 					if templen > 12 {
 						score += 1
 					}

tieta_data/merge_select.go

@@ -18,49 +18,35 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 			res = 3
 			pj = 3
 		case "ABB":
-			if info.LenPN > 10 || info.PCBH > 8 || info.PTCBH > 8 {
+			if info.LenPN > 10 {
 				res = 3
 			} else {
 				res = 2
 			}
 			pj = 3
 		case "BAA":
-			if info.PNBH > 10 || info.LenPC > 8 || info.LenPTC > 8 {
+			if info.LenPC > 8 || info.LenPTC > 8 {
 				res = 3
 			} else {
 				res = 2
 			}
 			pj = 3
 		case "BAB":
-			if info.PNBH > 10 || info.LenPTC > 8 || info.PTCBH > 8 {
+			if info.LenPTC > 8 {
 				res = 3
 			} else {
 				res = 2
 			}
 			pj = 3
 		case "BBA":
-			if info.PNBH > 10 || info.PCBH > 8 || info.LenPC > 8 {
+			if info.LenPC > 8 {
 				res = 3
 			} else {
 				res = 2
 			}
 			pj = 3
 		case "BBB":
-			v := 0
-			if info.PNBH > 10 {
-				v++
-			}
-			if info.PCBH > 8 {
-				v++
-			}
-			if info.PTCBH > 8 {
-				v++
-			}
-			if v > 1 {
-				res = 3
-			} else {
-				res = 2
-			}
+			res = 2
 			pj = 2
 		case "AAD":
 			if info.LenPC > 8 || info.LenPN > 12 {
@@ -70,10 +56,10 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 			}
 			pj = 3
 		case "ABD":
-			if info.LenPN > 10 && info.PCBH > 8 {
+			if info.LenPN > 10 {
 				res = 3
 				pj = 2
-			} else if info.LenPN > 10 || info.PCBH > 8 {
+			} else if info.LenPN > 10 {
 				res = 2
 				pj = 3
 			} else {
@@ -81,10 +67,10 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "BAD":
-			if info.LenPC > 13 || (info.PNBH > 10 && info.LenPC > 8) {
+			if info.LenPC > 13 || info.LenPC > 8 {
 				res = 3
 				pj = 3
-			} else if info.PNBH > 10 || info.LenPC > 8 {
+			} else if info.LenPC > 8 {
 				res = 2
 				pj = 3
 			} else {
@@ -92,28 +78,20 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "BBD":
-			if info.PNBH > 12 && info.PCBH > 10 {
-				res = 3
-				pj = 1
-			} else if info.PNBH > 10 && info.PCBH > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
+			res = 1
+			pj = 3
 		case "ADA":
-			if info.LenPN > 12 || (info.LenPTC > 8 && !StrOrNum2.MatchString(info.PTC)) {
+			if info.LenPN > 12 || info.LenPTC > 8 {
 				res = 3
 			} else {
 				res = 2
 			}
 			pj = 2
 		case "ADB":
-			if info.LenPN > 10 && info.PTCBH > 8 && !StrOrNum2.MatchString(info.PTC) {
+			if info.LenPN > 10 {
 				res = 3
 				pj = 2
-			} else if info.LenPN > 10 || info.PTCBH > 8 {
+			} else if info.LenPN > 10 {
 				res = 2
 				pj = 3
 			} else {
@@ -121,10 +99,10 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "BDA":
-			if info.PNBH > 10 && info.LenPTC > 8 && !StrOrNum2.MatchString(info.PTC) {
+			if info.LenPTC > 8 {
 				res = 3
 				pj = 2
-			} else if info.PNBH > 10 || info.LenPTC > 8 {
+			} else if info.LenPTC > 8 {
 				res = 2
 				pj = 3
 			} else {
@@ -132,13 +110,7 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "BDB":
-			if info.PNBH > 12 && info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-			} else if info.PNBH > 10 && info.PTCBH > 8 {
-				res = 2
-			} else {
-				res = 1
-			}
+			res = 1
 			pj = 2
 		case "ADD":
 			if info.LenPN > 18 {
@@ -153,11 +125,7 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 			}
 		case "BDD":
 			if CheckContain(info.ProjectName, compareInfo.ProjectName) != 3 {
-				if info.PNBH > 10 {
-					res = 2
-				} else {
-					res = 1
-				}
+				res = 1
 				pj = 1
 			}
 		case "DAA":
@@ -169,16 +137,8 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "DAB":
-			if info.LenPC > 8 && info.PTCBH > 8 {
-				res = 3
-				pj = 2
-			} else if info.LenPC > 8 || info.PTCBH > 8 {
-				res = 2
-				pj = 3
-			} else {
-				res = 1
-				pj = 3
-			}
+			res = 1
+			pj = 3
 		case "DAD":
 			if info.LenPC > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
 				res = 3
@@ -191,10 +151,7 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 2
 			}
 		case "DBA":
-			if info.PCBH > 8 && info.LenPC > 8 && !StrOrNum2.MatchString(info.PTC) {
-				res = 3
-				pj = 2
-			} else if info.PCBH > 8 || info.LenPC > 8 {
+			if info.LenPC > 8 {
 				res = 2
 				pj = 2
 			} else {
@@ -202,18 +159,15 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "DBB":
-			if info.PCBH > 10 && info.PTCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+			if !StrOrNum2.MatchString(info.ProjectCode) {
 				res = 3
 				pj = 1
-			} else if info.PCBH > 8 && info.PTCBH > 8 {
-				res = 2
-				pj = 2
 			} else {
 				res = 1
 				pj = 3
 			}
 		case "DBD":
-			if info.PCBH > 12 && !StrOrNum2.MatchString(info.ProjectCode) {
+			if !StrOrNum2.MatchString(info.ProjectCode) {
 				res = 2
 				pj = 1
 			} else {
@@ -221,7 +175,7 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 1
 			}
 		case "DDA":
-			if info.LenPTC > 14 && !StrOrNum2.MatchString(info.PTC) {
+			if info.LenPTC > 14 {
 				res = 3
 				pj = 1
 			} else if info.LenPTC > 8 {
@@ -232,14 +186,10 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 2
 			}
 		case "DDB":
-			if info.PTCBH > 12 && !StrOrNum2.MatchString(info.PTC) {
-				res = 2
-			} else {
-				res = 1
-			}
+			res = 1
 			pj = 1
 		case "ACA":
-			if info.LenPN > 10 && info.LenPTC > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+			if info.LenPN > 10 && info.LenPTC > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) {
 				res = 3
 				pj = 2
 			} else if info.LenPN > 10 || info.LenPTC > 8 {
@@ -250,10 +200,10 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "ACB":
-			if info.LenPN > 10 && info.PTCBH > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+			if info.LenPN > 10 && info.LenPC != len([]rune(compareInfo.ProjectCode)) {
 				res = 3
 				pj = 2
-			} else if info.LenPN > 10 || info.PTCBH > 8 {
+			} else if info.LenPN > 10 {
 				res = 2
 				pj = 2
 			} else {
@@ -261,10 +211,10 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "BCA":
-			if (info.PNBH > 10 && info.LenPTC > 8) || info.LenPTC > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+			if info.LenPTC > 8 || info.LenPTC > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) {
 				res = 3
 				pj = 2
-			} else if info.PNBH > 10 || info.LenPTC > 8 {
+			} else if info.LenPTC > 8 {
 				res = 2
 				pj = 2
 			} else {
@@ -272,12 +222,9 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "BCB":
-			if info.PNBH > 12 && info.PTCBH > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
 				res = 3
 				pj = 1
-			} else if info.PNBH > 10 || info.PTCBH > 8 {
-				res = 2
-				pj = 2
 			} else {
 				res = 1
 				pj = 2
@@ -291,10 +238,10 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "ABC":
-			if info.LenPN > 14 && info.PCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+			if info.LenPN > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
 				res = 3
 				pj = 2
-			} else if info.LenPN > 10 || info.PCBH > 8 {
+			} else if info.LenPN > 10 {
 				res = 2
 				pj = 3
 			} else {
@@ -302,10 +249,10 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "BAC":
-			if info.PNBH > 14 && info.LenPC > 8 && !StrOrNum2.MatchString(info.ProjectCode) {
+			if info.LenPC > 8 && !StrOrNum2.MatchString(info.ProjectCode) {
 				res = 3
 				pj = 2
-			} else if info.PNBH > 10 || info.LenPC > 8 {
+			} else if info.LenPC > 8 {
 				res = 2
 				pj = 3
 			} else {
@@ -313,12 +260,9 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "BBC":
-			if info.PNBH > 14 && info.PCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+			if !StrOrNum2.MatchString(info.ProjectCode) {
 				res = 3
 				pj = 1
-			} else if info.PNBH > 10 || info.PCBH > 8 {
-				res = 2
-				pj = 2
 			} else {
 				res = 1
 				pj = 2
@@ -356,23 +300,17 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 		case "BCC":
 			//项目编号不一致
 			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.PNBH > 12 {
-					res = 1
-				}
+				res = 1
 				pj = 1
 			}
 		case "BCD":
 			//项目编号不一致
 			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.PNBH > 8 {
-					res = 1
-				}
+				res = 1
 				pj = 1
 			}
 		case "BDC":
-			if info.PNBH > 7 {
-				res = 1
-			}
+			res = 1
 			pj = 1
 		case "CAA":
 			if info.LenPC > 12 || info.LenPTC > 12 {
@@ -386,10 +324,10 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 3
 			}
 		case "CAB":
-			if info.LenPC > 12 && info.PTCBH > 8 {
+			if info.LenPC > 12 {
 				res = 3
 				pj = 2
-			} else if info.LenPC > 12 || info.PTCBH > 8 {
+			} else if info.LenPC > 12 {
 				res = 2
 				pj = 3
 			} else {
@@ -412,10 +350,10 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 			}
 			pj = 1
 		case "CBA":
-			if info.LenPTC > 14 && info.PCBH > 12 {
+			if info.LenPTC > 14 {
 				res = 3
 				pj = 2
-			} else if info.LenPTC > 12 || info.PCBH > 10 {
+			} else if info.LenPTC > 12 {
 				res = 2
 				pj = 2
 			} else {
@@ -423,33 +361,25 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 				pj = 2
 			}
 		case "CBB":
-			if info.PCBH > 13 && info.PTCBH > 13 {
-				res = 3
-				pj = 1
-			} else if info.PCBH > 9 || info.PTCBH > 9 {
-				res = 2
-				pj = 2
-			} else {
-				res = 1
-				pj = 2
-			}
+			res = 1
+			pj = 2
 		case "CBC":
-			if info.PCBH > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+			if !StrOrNum2.MatchString(info.ProjectCode) {
 				res = 2
-			} else if info.PCBH > 5 {
+			} else {
 				res = 1
 			}
 			pj = 1
 		case "CBD":
-			if info.PCBH > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+			if !StrOrNum2.MatchString(info.ProjectCode) {
 				res = 2
-			} else if info.PCBH > 5 {
+			} else {
 				res = 1
 			}
 			pj = 1
 		case "CCA":
 			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.LenPTC > 12 && !StrOrNum2.MatchString(info.PTC) {
+				if info.LenPTC > 12 {
 					res = 2
 				} else if info.LenPTC > 5 {
 					res = 1
@@ -458,23 +388,18 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 			}
 		case "CCB":
 			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
-					res = 1
-				}
+				res = 1
 				pj = 1
 			}
 		case "CDA":
-			if info.LenPTC > 12 && !StrOrNum2.MatchString(info.PTC) {
+			if info.LenPTC > 12 {
 				res = 2
 			} else {
 				res = 1
 			}
 			pj = 1
 		case "CDB":
-			if info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
-				res = 1
-				pj = 1
-			}
+
 		case "DAC":
 			if info.LenPC > 13 && !StrOrNum2.MatchString(info.ProjectCode) {
 				res = 3
@@ -485,9 +410,6 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 			}
 			pj = 1
 		case "DBC":
-			if info.PCBH > 8 {
-				res = 1
-			}
 			pj = 1
 		case "DCA":
 			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
@@ -500,9 +422,6 @@ func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
 			}
 		case "DCB":
 			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
-				if info.PTCBH > 8 && !StrOrNum2.MatchString(info.PTC) {
-					res = 1
-				}
 				pj = 1
 			}
 		}

tieta_data/project.go

@@ -1,19 +1,15 @@
 package main
 
 import (
-	"fmt"
+	"context"
+	"github.com/google/uuid"
 	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"go.uber.org/zap"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
-	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
-	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"math"
 	"sort"
 	"strings"
 	"sync"
 	"tieta_data/config"
-	"time"
 )
 
 var (
@@ -21,14 +17,14 @@ var (
 )
 
 // 从对应map中获取对比的项目id
-func getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
+func getCompareIds(pn, pc, pb string) (bpn, bpc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
 	//	p.ConCurrentLock(n1, n2, n3, n4)
 	//	defer p.ConCurrentUnLock(n1, n2, n3, n4)
 	wg.Add(1)
 	//查找到id数组
 	res = []*Key{}
 	//是否查找到，并标识位置。-1代表值为空或已经存在。
-	bpn, bpc, bptc, bpb = -1, -1, -1, -1
+	bpn, bpc, bpb = -1, -1, -1
 	if pn != "" {
 		ids := mapPn[pn]
 		if ids == nil {
@@ -51,17 +47,6 @@ func getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key,
 		res = append(res, ids)
 	}
 
-	if ptc != "" {
-		ids := mapPc[ptc]
-		if ids == nil {
-			ids = &Key{Arr: []string{}}
-			mapPc[ptc] = ids
-			bptc = len(res)
-		}
-		ids.Lock.Lock()
-		res = append(res, ids)
-	}
-
 	if pb != "" {
 		ids := mapPb[pb]
 		if ids == nil {
@@ -100,7 +85,7 @@ func startProjectMerge(info *Info, tmp map[string]interface{}) {
 	//bpn, bpc, bptc, bpb 是否查找到，并标识位置。-1代表未查找到。
 	//pids 是项目id数组集合
 	//IDArr，是单个项目ID对象集合
-	bpn, bpc, bptc, bpb, pids, _, IDArr := getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer) //, info.LenPN, info.LenPC, info.LenPTC, len([]rune(info.Buyer)))
+	bpn, bpc, bpb, pids, _, IDArr := getCompareIds(info.ProjectName, info.ProjectCode, info.Buyer) //, info.LenPN, info.LenPC, info.LenPTC, len([]rune(info.Buyer)))
 	defer wg.Done()
 	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock
 
@@ -122,7 +107,6 @@ func startProjectMerge(info *Info, tmp map[string]interface{}) {
 		comStr := ""
 		compareProject := v.P
 		compareProject.score = 0
-		//问题出地LastTime!!!!!
 		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
 		//代理机构完全不相同，直接新建项目
 		if CheckContain(compareProject.Agency, info.Agency) == 3 {
@@ -132,10 +116,11 @@ func startProjectMerge(info *Info, tmp map[string]interface{}) {
 		if ComparePlace(compareProject, info) {
 			continue
 		}
+		// 多包 处理
+		if ComparePkg(compareProject, info) {
+			continue
+		}
 
-		info.PNBH = 0
-		info.PCBH = 0
-		info.PTCBH = 0
 		compareStr, score := comparePNC(info, compareProject)
 
 		resVal, pjVal := Select(compareStr, info, compareProject)
@@ -176,30 +161,18 @@ func startProjectMerge(info *Info, tmp map[string]interface{}) {
 			}
 			if len(resArr) > 0 {
 				bFindProject = true
-				findPid = resArr[0].Id.Hex()
+				findPid = resArr[0].ProjectId
 				UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr)
-				for k2, bv := range []int{bpn, bpc, bptc, bpb} {
+				for k2, bv := range []int{bpn, bpc, bpb} {
 					if bv > -1 {
 						pids[bv].Arr = append(pids[bv].Arr, findPid)
 						if k2 == 0 {
 							if resArr[0].ProjectName == "" {
 								resArr[0].ProjectName = info.ProjectName
-							} else {
-								if resArr[0].MPN == nil {
-									resArr[0].MPN = []string{info.ProjectName}
-								} else {
-									resArr[0].MPN = append(resArr[0].MPN, info.ProjectName)
-								}
 							}
 						} else if k2 < 3 {
 							if resArr[0].ProjectCode == "" {
-								resArr[0].ProjectCode = util.If(k2 == 1, info.ProjectCode, info.PTC).(string)
-							} else {
-								if resArr[0].MPC == nil {
-									resArr[0].MPC = []string{util.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
-								} else {
-									resArr[0].MPC = append(resArr[0].MPC, util.If(k2 == 1, info.ProjectCode, info.PTC).(string))
-								}
+								resArr[0].ProjectCode = info.ProjectCode
 							}
 						} else {
 							if resArr[0].Buyer == "" {
@@ -416,74 +389,65 @@ func ComparePlace(project *Project, info *Info) bool {
 	return true
 }
 
-func NewCachePinfo(id primitive.ObjectID, thisinfo *Info) Project {
+func ComparePkg(project *Project, info *Info) bool {
+	if project.MultiPackage == 1 && project.MultiPackageId == info.MultiPackageId {
+		return false
+	}
+	return true
+}
+
+func NewCachePinfo(thisinfo *Info, pid string) Project {
 	p1 := Project{
-		Id:          id,
-		Ids:         []string{thisinfo.Id},
-		ProjectName: thisinfo.ProjectName,
-		ProjectCode: thisinfo.ProjectCode,
-		Buyer:       thisinfo.Buyer,
-		Agency:      thisinfo.Agency,
-		Area:        thisinfo.Area,
-		City:        thisinfo.City,
-		District:    thisinfo.District,
-		MPN:         []string{},
-		MPC:         []string{},
-		FirstTime:   thisinfo.Publishtime,
-		LastTime:    thisinfo.Publishtime,
-		Budget:      thisinfo.Budget,
-		Bidamount:   thisinfo.Bidamount,
-	}
-	if thisinfo.LenPTC > 5 {
-		p1.MPC = append(p1.MPC, thisinfo.PTC)
+		ProjectId:      pid,
+		ProjectName:    thisinfo.ProjectName,
+		ProjectCode:    thisinfo.ProjectCode,
+		Buyer:          thisinfo.Buyer,
+		Agency:         thisinfo.Agency,
+		Area:           thisinfo.Area,
+		City:           thisinfo.City,
+		District:       thisinfo.District,
+		FirstTime:      thisinfo.Publishtime,
+		LastTime:       thisinfo.Publishtime,
+		Budget:         thisinfo.Budget,
+		Bidamount:      thisinfo.Bidamount,
+		MultiPackage:   thisinfo.MultiPackage,
+		MultiPackageId: thisinfo.MultiPackageId,
 	}
 	return p1
 }
 
 func NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *Project) {
-	pId := mongodb.StringTOBsonId(thisinfo.Id)
+	pId := uuid.New().String()
+	pId = strings.ReplaceAll(pId, "-", "")
 	set := map[string]interface{}{}
-	set["_id"] = pId
+	set["projectId"] = pId
 	set["sourceinfoid"] = thisinfo.Id
 	for _, f := range FIELDS {
 		if tmp[f] != nil && tmp[f] != "" {
 			set[f] = tmp[f]
 		}
 	}
-	bidopentime := util.Int64All(tmp["bidopentime"])
-	if bidopentime > 0 {
-		set["bidopentime"] = bidopentime
+	if util.ObjToString(tmp["projectname"]) == "" {
+		set["projectname"] = thisinfo.ProjectName
 	}
-	p1 := NewCachePinfo(pId, thisinfo)
-
-	now := time.Now().Unix()
-	set["createtime"] = now
-	set["firsttime"] = tmp["publishtime"]
-	set["lasttime"] = tmp["publishtime"]
-
-	set["ids"] = []string{thisinfo.Id}
+	p1 := NewCachePinfo(thisinfo, pId)
 	if thisinfo.TopType == "招标" {
-		//if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
-		set["zbtime"] = tmp["publishtime"]
 		p1.Zbtime = util.Int64All(tmp["publishtime"])
-		set["zb_href"] = tmp["href"]
-		set["zb_jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", thisinfo.Id))
-		set["zb_updatetime"] = util.Int64All(tmp["publishtime"])
+		set["zb_href"] = thisinfo.Href
+		set["zb_jybxhref"] = thisinfo.JybxHref
+		set["zb_updatetime"] = tmp["publishtime"]
 		set["zb_infoid"] = thisinfo.Id
-		set["bidtype"] = thisinfo.TopType
-		//}
+		set["zb_toptype"] = thisinfo.TopType
+		set["zb_subtype"] = thisinfo.SubType
 	} else if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "合同" || thisinfo.SubType == "验收" ||
 		thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
-		set["jgtime"] = tmp["publishtime"]
 		p1.Jgtime = thisinfo.Publishtime
-		set["href"] = tmp["href"]
-		set["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", thisinfo.Id))
-		set["updatetime"] = util.Int64All(tmp["publishtime"])
+		set["href"] = thisinfo.Href
+		set["jybxhref"] = thisinfo.JybxHref
+		set["updatetime"] = tmp["publishtime"]
 		set["infoid"] = thisinfo.Id
-		set["bidstatus"] = thisinfo.SubType
-	}
-	if len(thisinfo.Subscopeclass) > 0 {
-		set["s_subscopeclass"] = strings.Join(thisinfo.Subscopeclass, ",")
+		set["toptype"] = thisinfo.TopType
+		set["subtype"] = thisinfo.SubType
 	}
 	if len(thisinfo.Winners) > 0 {
 		set["s_winner"] = strings.Join(thisinfo.Winners, ",")
@@ -496,85 +460,38 @@ func NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *Project) {
 			set["winnerperson"] = tmp["winnerperson"]
 		}
 	}
-
-	if util.ObjToString(tmp["scenetag"]) != "" {
-		p1.SceneTag = util.ObjToString(tmp["scenetag"])
-		set["scenetag"] = util.ObjToString(tmp["scenetag"])
-	}
-	if util.ObjToString(tmp["buyertag"]) != "" {
-		p1.BuyerTag = util.ObjToString(tmp["buyertag"])
-		set["buyertag"] = util.ObjToString(tmp["buyertag"])
-	}
 	if util.ObjToString(tmp["tagname"]) != "" {
 		p1.TagName = util.ObjToString(tmp["tagname"])
 		set["tagname"] = util.ObjToString(tmp["tagname"])
 	}
-	if util.ObjToString(tmp["tagname_2"]) != "" {
-		p1.TagName2 = util.ObjToString(tmp["tagname_2"])
-		set["tagname_2"] = util.ObjToString(tmp["tagname_2"])
+	if util.ObjToString(tmp["tagname2"]) != "" {
+		p1.TagName2 = util.ObjToString(tmp["tagname2"])
+		set["tagname2"] = util.ObjToString(tmp["tagname2"])
 	}
-	if util.ObjToString(tmp["tagname_3"]) != "" {
-		p1.TagName3 = util.ObjToString(tmp["tagname_3"])
-		set["tagname_3"] = util.ObjToString(tmp["tagname_3"])
+	if util.ObjToString(tmp["tagname3"]) != "" {
+		p1.TagName3 = util.ObjToString(tmp["tagname3"])
+		set["tagname3"] = util.ObjToString(tmp["tagname3"])
 	}
-
-	if tmp["budget"] != nil {
-		set["budget"] = thisinfo.Budget
-		p1.Budget = thisinfo.Budget
-	}
-	if tmp["bidamount"] != nil {
-		set["bidamount"] = thisinfo.Bidamount
-		p1.Bidamount = thisinfo.Bidamount
-	}
-	if p1.Bidamount > 0 {
-		set["sortprice"] = p1.Bidamount
+	if thisinfo.MultiPackage == 0 {
+		set["multipackage"] = 0
 	} else {
-		if p1.Budget > 0 {
-			set["sortprice"] = p1.Budget
-		}
-	}
-	push := PushListInfo(tmp, thisinfo.Id)
-	push["s_winner"] = strings.Join(thisinfo.Winners, ",")
-	set["list"] = []bson.M{
-		push,
+		set["multipackage"] = 1
+		set["multipackageid"] = thisinfo.MultiPackageId
 	}
-	updatePool <- []map[string]interface{}{
-		{
-			"_id": pId,
-		},
-		{
-			"$set": set,
-		},
-	}
-	return pId.Hex(), &p1
-}
 
-// 项目中list的信息
-func PushListInfo(tmp map[string]interface{}, infoid string) bson.M {
-	res := bson.M{
-		"infoid": infoid,
-	}
-	for _, k := range INFOFIELDS {
-		if tmp[k] != nil {
-			res[k] = tmp[k]
-		}
+	id := MysqlTool.Insert(config.Conf.DB.Mysql.Pcoll, set)
+	if id > 0 {
+		MysqlTool.Update(config.Conf.DB.Mysql.Coll, bson.M{"msg_id": thisinfo.MsgId}, bson.M{"projectId": pId})
+		client := Es.GetEsConn()
+		defer Es.DestoryEsConn(client)
+		client.Update().Index(config.Conf.DB.Es.Index).Id(thisinfo.Nid).
+			Doc(map[string]interface{}{"projectId": pId}).Refresh("true").Do(context.TODO())
 	}
-	return res
+	return pId, &p1
 }
 
 func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *Project, weight int, comStr string) {
-	if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
-		log.Info("repeat", zap.String("info", thisinfo.Id), zap.Any("pid", pInfo.Id))
-		return
-	}
-
 	set := map[string]interface{}{}
-	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
-	if thisinfo.Bidopentime > pInfo.Bidopentime {
-		pInfo.Bidopentime = thisinfo.Bidopentime
-		set["bidopentime"] = pInfo.Bidopentime
-	}
-
 	//相同城市的公告才会合并到一起(全国列外)
 	if thisinfo.Area != "全国" {
 		pInfo.Area = thisinfo.Area
@@ -589,13 +506,10 @@ func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *Project, w
 		}
 	}
 	//项目名称
-	//if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
 	if pInfo.ProjectName == "" && thisinfo.ProjectName != "" {
 		pInfo.ProjectName = thisinfo.ProjectName
 		set["projectname"] = thisinfo.ProjectName
 	}
-	//7--项目编号
-	//if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
 	if pInfo.ProjectCode == "" && thisinfo.ProjectCode != "" {
 		pInfo.ProjectCode = thisinfo.ProjectCode
 		set["projectcode"] = thisinfo.ProjectCode
@@ -608,10 +522,6 @@ func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *Project, w
 		pInfo.Buyerclass = thisinfo.Buyerclass
 		set["buyerclass"] = thisinfo.Buyerclass
 	}
-	if pInfo.Buyer == "" {
-		pInfo.Buyerclass = ""
-		set["buyerclass"] = ""
-	}
 	//采购单位联系人
 	if thisinfo.Buyerperson != "" {
 		pInfo.Buyerperson = thisinfo.Buyerperson
@@ -634,34 +544,9 @@ func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *Project, w
 	if util.ObjToString(tmp["agencytel"]) != "" {
 		set["agencytel"] = tmp["agencytel"]
 	}
-
-	if len(thisinfo.Topscopeclass) > 0 {
-		sort.Strings(pInfo.Topscopeclass)
-		for _, k := range thisinfo.Topscopeclass {
-			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
-				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
-				sort.Strings(pInfo.Topscopeclass)
-			}
-		}
-		set["topscopeclass"] = pInfo.Topscopeclass
-	}
-
-	if len(thisinfo.Subscopeclass) > 0 {
-		sort.Strings(pInfo.Subscopeclass)
-		for _, k := range thisinfo.Subscopeclass {
-			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
-				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
-				sort.Strings(pInfo.Subscopeclass)
-			}
-		}
-		set["subscopeclass"] = pInfo.Subscopeclass
-		set["s_subscopeclass"] = strings.Join(pInfo.Subscopeclass, ",")
-	}
-
 	if len(thisinfo.Winners) > 0 {
 		var winners []string
 		if pInfo.Winners == "" {
-			set["winner"] = util.ObjToString(tmp["winner"])
 			for _, k := range thisinfo.Winners {
 				if k == "" {
 					continue
@@ -695,41 +580,34 @@ func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *Project, w
 			set["winnerperson"] = tmp["winnerperson"]
 		}
 	}
-
 	if thisinfo.TopType == "招标" {
-		set["zb_href"] = tmp["href"]
-		set["zb_jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", thisinfo.Id))
+		set["zb_href"] = thisinfo.Href
+		set["zb_jybxhref"] = thisinfo.JybxHref
 		set["zb_infoid"] = thisinfo.Id
-		set["zb_updatetime"] = util.Int64All(tmp["publishtime"])
-		set["bidtype"] = thisinfo.TopType
+		set["zb_updatetime"] = tmp["publishtime"]
+		set["zb_toptype"] = thisinfo.TopType
+		set["zb_subtype"] = thisinfo.SubType
 	} else if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "合同" || thisinfo.SubType == "验收" ||
 		thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
-		set["href"] = tmp["href"]
-		set["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", thisinfo.Id))
-		set["updatetime"] = util.Int64All(tmp["publishtime"])
+		set["href"] = thisinfo.Href
+		set["jybxhref"] = thisinfo.JybxHref
+		set["updatetime"] = tmp["publishtime"]
 		set["infoid"] = thisinfo.Id
-		set["bidstatus"] = thisinfo.SubType
+		set["toptype"] = thisinfo.TopType
+		set["subtype"] = thisinfo.SubType
 	}
 
-	if util.ObjToString(tmp["scenetag"]) != "" && pInfo.SceneTag != "" {
-		pInfo.SceneTag = util.ObjToString(tmp["scenetag"])
-		set["scenetag"] = util.ObjToString(tmp["scenetag"])
-	}
-	if util.ObjToString(tmp["buyertag"]) != "" && pInfo.BuyerTag != "" {
-		pInfo.BuyerTag = util.ObjToString(tmp["buyertag"])
-		set["buyertag"] = util.ObjToString(tmp["buyertag"])
-	}
 	if util.ObjToString(tmp["tagname"]) != "" && pInfo.TagName != "" {
 		pInfo.TagName = util.ObjToString(tmp["tagname"])
 		set["tagname"] = util.ObjToString(tmp["tagname"])
 	}
-	if util.ObjToString(tmp["tagname_2"]) != "" && pInfo.TagName2 != "" {
-		pInfo.TagName2 = util.ObjToString(tmp["tagname_2"])
-		set["tagname_2"] = util.ObjToString(tmp["tagname_2"])
+	if util.ObjToString(tmp["tagname2"]) != "" && pInfo.TagName2 != "" {
+		pInfo.TagName2 = util.ObjToString(tmp["tagname2"])
+		set["tagname2"] = util.ObjToString(tmp["tagname2"])
 	}
-	if util.ObjToString(tmp["tagname_3"]) != "" && pInfo.TagName3 != "" {
-		pInfo.TagName3 = util.ObjToString(tmp["tagname_3"])
-		set["tagname_3"] = util.ObjToString(tmp["tagname_3"])
+	if util.ObjToString(tmp["tagname3"]) != "" && pInfo.TagName3 != "" {
+		pInfo.TagName3 = util.ObjToString(tmp["tagname3"])
+		set["tagname3"] = util.ObjToString(tmp["tagname3"])
 	}
 
 	if thisinfo.Budget > 0 {
@@ -740,35 +618,13 @@ func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *Project, w
 		set["bidamount"] = thisinfo.Bidamount
 		pInfo.Bidamount = thisinfo.Bidamount
 	}
-	if pInfo.Bidamount > 0 {
-		set["sortprice"] = pInfo.Bidamount
-	} else {
-		if pInfo.Budget > 0 {
-			set["sortprice"] = pInfo.Budget
-		}
-	}
 
-	update := map[string]interface{}{}
-	if len(set) > 0 {
-		update["$set"] = set
-	}
-	push := PushListInfo(tmp, thisinfo.Id)
-	push["s_winner"] = strings.Join(thisinfo.Winners, ",")
-	push["compareStr"] = comStr
-	push["resVal"] = pInfo.resVal
-	push["pjVal"] = pInfo.pjVal
-	update["$push"] = map[string]interface{}{
-		"list": push,
-		"ids":  thisinfo.Id,
-	}
-
-	if len(update) > 0 {
-		updateInfo := []map[string]interface{}{
-			{
-				"_id": pInfo.Id,
-			},
-			update,
-		}
-		updatePool <- updateInfo
+	b := MysqlTool.Update(config.Conf.DB.Mysql.Pcoll, bson.M{"projectId": pInfo.ProjectId}, set)
+	if b {
+		MysqlTool.Update(config.Conf.DB.Mysql.Coll, bson.M{"msg_id": thisinfo.MsgId}, bson.M{"projectId": pInfo.ProjectId})
+		client := Es.GetEsConn()
+		defer Es.DestoryEsConn(client)
+		client.Update().Index(config.Conf.DB.Es.Index).Id(thisinfo.Nid).
+			Doc(map[string]interface{}{"projectId": pInfo.ProjectId}).Refresh("true").Do(context.TODO())
 	}
 }

tieta_data/task.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"database/sql"
 	"encoding/json"
+	"fmt"
 	"github.com/tealeg/xlsx"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
@@ -10,8 +12,8 @@ import (
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"regexp"
 	"strings"
+	"sync"
 	"tieta_data/config"
-	"time"
 )
 
 var (
@@ -97,23 +99,62 @@ func taskExcelB(tmp map[string]interface{}, row *xlsx.Row) {
 }
 
 func loadData() {
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
-
-	it := sess.DB(config.Conf.DB.Mongo.Dbname).C(config.Conf.DB.Mongo.Pcoll).Find(nil).Select(nil).Iter()
-	n := 0
-	result := make(map[string]interface{})
-	if it.Next(&result) {
-		n++
-		bys, _ := json.Marshal(result)
-		var pro *Project
-		_ = json.Unmarshal(bys, &pro)
+	finalId := 0
+	lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT msg_id FROM %s ORDER BY msg_id DESC LIMIT 1", config.Conf.DB.Mysql.Pcoll))
+	if len(*lastInfo) > 0 {
+		finalId = util.IntAll((*lastInfo)[0]["msg_id"])
+	}
+	util.Debug("loadData---", "finally id", finalId)
+	lastid, count := 0, 0
+	for {
+		util.Debug("重新查询，lastid---", lastid)
+		q := fmt.Sprintf("SELECT * FROM %s WHERE msg_id > %d ORDER BY msg_id ASC limit 100000", config.Conf.DB.Mysql.Pcoll, lastid)
+		rows, err := MysqlTool.DB.Query(q)
+		if err != nil {
+			util.Debug("loadData---", err)
+		}
+		columns, err := rows.Columns()
+		if finalId == lastid {
+			util.Debug("---loadData-finish----", count)
+			break
+		}
+		for rows.Next() {
+			scanArgs := make([]interface{}, len(columns))
+			values := make([]interface{}, len(columns))
+			ret := make(map[string]interface{})
+			for k := range values {
+				scanArgs[k] = &values[k]
+			}
+			err = rows.Scan(scanArgs...)
+			if err != nil {
+				util.Debug("loadData---", err)
+				break
+			}
+			for i, col := range values {
+				if v, ok := col.([]uint8); ok {
+					ret[columns[i]] = string(v)
+				} else {
+					ret[columns[i]] = col
+				}
+			}
+			lastid = util.IntAll(ret["msg_id"])
+			count++
+			if count%20000 == 0 {
+				util.Debug("loadData current-------", count, lastid)
+			}
+			pool <- true
+			wg.Add(1)
+			func(result map[string]interface{}) {
+				defer func() {
+					<-pool
+					wg.Done()
+				}()
+				bys, _ := json.Marshal(result)
+				var pro *Project
+				_ = json.Unmarshal(bys, &pro)
 
-		id := pro.Id.Hex()
-		for _, v := range append([]string{pro.ProjectName}, pro.MPN...) {
-			if v != "" {
-				//v = pcReplace.ReplaceAllString(v, "")
-				if v != "" {
+				id := pro.ProjectId
+				if v := pro.ProjectName; v != "" {
 					k := mapPn[v]
 					if k == nil {
 						k = &Key{Arr: []string{id}}
@@ -122,11 +163,7 @@ func loadData() {
 						k.Arr = append(k.Arr, id)
 					}
 				}
-			}
-		}
-		for _, v := range append([]string{pro.ProjectCode}, pro.MPC...) {
-			if v != "" {
-				if v != "" {
+				if v := pro.ProjectCode; v != "" {
 					k := mapPc[v]
 					if k == nil {
 						k = &Key{Arr: []string{id}}
@@ -135,101 +172,97 @@ func loadData() {
 						k.Arr = append(k.Arr, id)
 					}
 				}
-			}
-		}
-		if pro.Buyer != "" && len([]rune(pro.Buyer)) > 2 {
-			k := mapPb[pro.Buyer]
-			if k == nil {
-				k = &Key{Arr: []string{id}}
-				mapPb[pro.Buyer] = k
-			} else {
-				k.Arr = append(k.Arr, id)
-			}
+				if pro.Buyer != "" && len([]rune(pro.Buyer)) > 2 {
+					k := mapPb[pro.Buyer]
+					if k == nil {
+						k = &Key{Arr: []string{id}}
+						mapPb[pro.Buyer] = k
+					} else {
+						k.Arr = append(k.Arr, id)
+					}
+				}
+				AllIdsMap[id] = &ID{Id: id, P: pro}
+			}(ret)
+			ret = make(map[string]interface{})
 		}
-		AllIdsMap[id] = &ID{Id: id, P: pro}
+		_ = rows.Close()
+		wg.Wait()
 	}
-
-	time.Sleep(2 * time.Second)
-	log.Info("load project over..", zap.Int("n", n))
+	log.Info("load project over..", zap.Int("n", count))
 }
 
-func taskProject(mapinfo map[string]interface{}) {
-	q := make(map[string]interface{})
-	gtid := util.ObjToString(mapinfo["gtid"])
-	lteid := util.ObjToString(mapinfo["lteid"])
-	q["_id"] = map[string]interface{}{
-		"$lte": mongodb.StringTOBsonId(lteid),
-		"$gte": mongodb.StringTOBsonId(gtid),
-	}
-	//生成查询语句执行
-	log.Info("查询语句：", zap.Any("q", q))
-	count := 0
+func taskProject() {
+	pool := make(chan bool, 2) //控制线程数
+	wg := &sync.WaitGroup{}
 
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
-	infoPool := make(chan map[string]interface{}, 2000)
-	over := make(chan bool)
-	go func() {
-	L:
-		for {
-			select {
-			case tmp := <-infoPool:
-				pool <- true
-				go func(tmp map[string]interface{}) {
-					defer func() {
-						<-pool
-					}()
-					info := ParseInfo(tmp)
-					startProjectMerge(info, tmp)
-				}(tmp)
-			default:
-				select {
-				case tmp := <-infoPool:
-					pool <- true
-					go func(tmp map[string]interface{}) {
-						defer func() {
-							<-pool
-						}()
-						info := ParseInfo(tmp)
-						startProjectMerge(info, tmp)
-					}(tmp)
-				case <-over:
-					break L
-				}
-			}
-		}
-	}()
-	fields := map[string]interface{}{"kvtext": 0, "repeat_reason": 0, "field_source": 0, "detail": 0, "contenthtml": 0, "jsondata": 0}
-	ms := sess.DB(config.Conf.DB.Mongo.Dbname).C(config.Conf.DB.Mongo.Coll).Find(q).Select(fields)
-	query := ms.Iter()
-	var lastid interface{}
-L:
+	finalId := 0
+	lastInfo := MysqlTool.SelectBySql(fmt.Sprintf("SELECT msg_id FROM %s where projectId is null ORDER BY msg_id DESC LIMIT 1", config.Conf.DB.Mysql.Coll))
+	if len(*lastInfo) > 0 {
+		finalId = util.IntAll((*lastInfo)[0]["msg_id"])
+	}
+	util.Debug("taskProject---", "finally id", finalId)
+	lastid, count := 0, 0
 	for {
-		select {
-		case <-queryClose:
-			log.Error("receive interrupt sign")
-			queryCloseOver <- true
-			break L
-		default:
-			tmp := make(map[string]interface{})
-			if query.Next(&tmp) {
-				lastid = tmp["_id"]
-				if count%2000 == 0 {
-					log.Info("current", zap.Int("count", count), zap.Any("lastid", lastid))
+		util.Debug("重新查询，lastid---", lastid)
+		q := fmt.Sprintf("SELECT * FROM %s WHERE msg_id > %d And projectId is null ORDER BY msg_id ASC limit 100000", config.Conf.DB.Mysql.Coll, lastid)
+		var stmtOut *sql.Stmt
+		var tx *sql.Tx
+		var err error
+		if tx == nil {
+			stmtOut, err = MysqlTool.DB.Prepare(q)
+		} else {
+			stmtOut, err = tx.Prepare(q)
+		}
+		rows, err := stmtOut.Query()
+		if err != nil {
+			util.Debug("taskProject---", err)
+		}
+		columns, err := rows.Columns()
+		if finalId == lastid {
+			util.Debug("----finish----------", count)
+			break
+		}
+		for rows.Next() {
+			scanArgs := make([]interface{}, len(columns))
+			values := make([]interface{}, len(columns))
+			ret := make(map[string]interface{})
+			for k := range values {
+				scanArgs[k] = &values[k]
+			}
+			err = rows.Scan(scanArgs...)
+			if err != nil {
+				util.Debug("taskProject---", err)
+				break
+			}
+			for i, col := range values {
+				if v, ok := col.([]uint8); ok {
+					ret[columns[i]] = string(v)
+				} else {
+					ret[columns[i]] = col
 				}
-				infoPool <- tmp
-				count++
-			} else {
-				break L
 			}
+			lastid = util.IntAll(ret["msg_id"])
+			count++
+			if count%2000 == 0 {
+				util.Debug("current-------", count, lastid)
+			}
+			pool <- true
+			wg.Add(1)
+			go func(tmp map[string]interface{}) {
+				defer func() {
+					<-pool
+					wg.Done()
+				}()
+				info := ParseInfo(tmp)
+				startProjectMerge(info, tmp)
+			}(ret)
+			ret = make(map[string]interface{})
 		}
+		_ = rows.Close()
+		stmtOut.Close()
+		wg.Wait()
 	}
-	time.Sleep(5 * time.Second)
-	over <- true
-	////阻塞
-	for n := 0; n < 1; n++ {
-		pool <- true
-	}
+
 	log.Info("所有线程执行完成...", zap.Int("count:", count))
 }
 
@@ -240,59 +273,35 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	if thisinfo == nil {
 		return nil
 	}
-
-	if len(thisinfo.Topscopeclass) == 0 {
-		thisinfo.Topscopeclass = []string{}
-	}
-	if len(thisinfo.Subscopeclass) == 0 {
-		thisinfo.Subscopeclass = []string{}
-	}
-	if thisinfo.SubType == "" {
-		thisinfo.SubType = util.ObjToString(tmp["bidstatus"])
+	if thisinfo.ProjectName == "" {
+		thisinfo.ProjectName = thisinfo.Title
 	}
-
 	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
-		//thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
-		//if thisinfo.ProjectName != "" {
 		thisinfo.pnbval++
-		//}
 	}
-
-	if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
+	if thisinfo.ProjectCode != "" {
 		if thisinfo.ProjectCode != "" {
-			//thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
 			if thisinfo.pnbval == 0 && len([]rune(thisinfo.ProjectCode)) < 5 {
 				thisinfo.ProjectCode = StrOrNum.ReplaceAllString(thisinfo.ProjectCode, "")
 			}
-		} else {
-			//thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
-			if thisinfo.pnbval == 0 && len([]rune(thisinfo.PTC)) < 5 {
-				thisinfo.PTC = StrOrNum.ReplaceAllString(thisinfo.PTC, "")
-			}
 		}
 		thisinfo.pnbval++
 	}
-	if thisinfo.ProjectCode == thisinfo.PTC || strings.Index(thisinfo.ProjectCode, thisinfo.PTC) > -1 {
-		thisinfo.PTC = ""
-	}
-
 	if thisinfo.Buyer != "" && len([]rune(thisinfo.Buyer)) > 2 {
 		thisinfo.pnbval++
 	} else {
 		thisinfo.Buyer = ""
 	}
-	//winners整理、清理
-	winner := util.ObjToString(tmp["winner"])
-	m1 := map[string]bool{}
-	winners := []string{}
-	if winner != "" {
-		m1[winner] = true
-		winners = append(winners, winner)
+	if tmp["multipackage"] == nil || util.ObjToString(tmp["multipackage"]) == "否" {
+		thisinfo.MultiPackage = 0
+	} else {
+		thisinfo.MultiPackage = 1
 	}
-	thisinfo.Winners = winners
+	//winners整理、清理
+	winner := util.ObjToString(tmp["s_winner"])
+	thisinfo.Winners = strings.Split(winner, ",")
 
 	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
-	thisinfo.LenPTC = len([]rune(thisinfo.PTC))
 	thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
 	return thisinfo
 }

tieta_data/tool.go

@@ -26,11 +26,8 @@ var FIELDS = []string{
 	"agency",
 	"agencyperson",
 	"agencytel",
-	"topscopeclass",
-	"subscopeclass",
-	"package",
-	"scenetag",
-	"buyertag",
+	"budget",
+	"bidamount",
 	"tagname",
 	"tagname_2",
 	"tagname_3",