xg

README.md

@@ -54,3 +54,6 @@
 
 ## data_project 项目合并
 +  依赖bidding数据进行项目合并
+
+## data_project_information 项目数据物业字段处理
+

data_project/main.go

@@ -79,7 +79,7 @@ func DealSign() {
 	}
 }
 
-func main() {
+func mainT() {
 
 	P_QL.loadSpiderCode()
 	P_QL.loadSite()
@@ -167,9 +167,9 @@ func main() {
 	}
 }
 
-func mainT() {
+func main() {
 	sid = "64f5d956513e65bb3f0e4d4f"
-	eid = "64f5d956513e65bb3f0e4d50"
+	eid = "65e1a06866cf0db42adf5a87"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()

data_project/task.go

@@ -346,7 +346,7 @@ func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
 	//data := Es.Get(config.Conf.DB.Es.Index, esquery)
 	data, _ := MgoP.FindOne(config.Conf.DB.MongoP.Coll, bson.M{"ids": infoid})
 	if len(*data) > 0 {
-		pid := util.ObjToString((*data)["_id"])
+		pid := mongodb.BsonIdToSId((*data)["_id"])
 		p.updateJudge(*infoMap, pid)
 	} else {
 		log.Info("not find project---," + infoid)
@@ -738,7 +738,7 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 }
 
 func (p *ProjectTask) updateJudge(infoMap map[string]interface{}, pid string) {
-	tmpPro, _ := MgoB.FindById(ProjectColl, pid, nil)
+	tmpPro, _ := MgoP.FindById(ProjectColl, pid, nil)
 	modifyProMap := make(map[string]interface{}) // 修改项目的字段
 	if modifyMap, ok := infoMap["modifyinfo"].(map[string]interface{}); ok {
 		for k := range modifyMap {

data_project_information/clickhouse.go

@@ -0,0 +1,201 @@
+package main
+
+import (
+	"bytes"
+	"database/sql"
+	"fmt"
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"log"
+	"reflect"
+	"strings"
+	"time"
+)
+
+type Ck struct {
+	Address      string  //数据库地址:端口
+	UserName     string  //用户名
+	PassWord     string  //密码
+	DBName       string  //数据库名
+	DB           *sql.DB //数据库连接池对象
+	MaxOpenConns int     //用于设置最大打开的连接数，默认值为0表示不限制。
+	MaxIdleConns int     //用于设置闲置的连接数。
+}
+
+func (c *Ck) Init() {
+	if c.MaxOpenConns <= 0 {
+		c.MaxOpenConns = 20
+	}
+	if c.MaxIdleConns <= 0 {
+		c.MaxIdleConns = 20
+	}
+	c.DB = clickhouse.OpenDB(&clickhouse.Options{
+		Addr: strings.Split(c.Address, ","),
+		Auth: clickhouse.Auth{
+			Database: c.DBName,
+			Username: c.UserName,
+			Password: c.PassWord,
+		},
+		//TLS: &tls.Config{
+		//	InsecureSkipVerify: true,
+		//},
+		//Settings: clickhouse.Settings{
+		//	"max_execution_time": 60,
+		//},
+		DialTimeout: time.Second * 30,
+		//Compression: &clickhouse.Compression{
+		//	Method: clickhouse.CompressionLZ4,
+		//},
+		//Debug:                true,
+		BlockBufferSize:      10,
+		MaxCompressionBuffer: 10240,
+		//ClientInfo: clickhouse.ClientInfo{
+		//	Products: []struct {
+		//		Name    string
+		//		Version string
+		//	}{
+		//		{Name: "my-app", Version: "0.1"},
+		//	},
+		//},
+	})
+	c.DB.SetMaxIdleConns(c.MaxIdleConns)
+	c.DB.SetMaxOpenConns(c.MaxOpenConns)
+	c.DB.SetConnMaxLifetime(time.Hour)
+
+}
+
+func (m *Ck) FindOne(tableName string, query map[string]interface{}, fields, order string) *map[string]interface{} {
+	list := m.Find(tableName, query, fields, order, 0, 1)
+	if list != nil && len(*list) == 1 {
+		temp := (*list)[0]
+		return &temp
+	}
+	return nil
+}
+
+func (m *Ck) Find(tableName string, query map[string]interface{}, fields, order string, start, pageSize int) *[]map[string]interface{} {
+	fs := []string{}
+	vs := []interface{}{}
+	for k, v := range query {
+		rt := reflect.TypeOf(v)
+		rv := reflect.ValueOf(v)
+		if rt.Kind() == reflect.Map {
+			for _, rv_k := range rv.MapKeys() {
+				if rv_k.String() == "ne" {
+					fs = append(fs, fmt.Sprintf("%s!=?", k))
+					vs = append(vs, rv.MapIndex(rv_k).Interface())
+				}
+				if rv_k.String() == "notin" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						for _, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							fs = append(fs, fmt.Sprintf("%s!=?", k))
+							vs = append(vs, v)
+						}
+					}
+				}
+				if rv_k.String() == "in" {
+					if len(rv.MapIndex(rv_k).Interface().([]interface{})) > 0 {
+						_fs := fmt.Sprintf("%s in (?", k)
+						for k, v := range rv.MapIndex(rv_k).Interface().([]interface{}) {
+							if k > 0 {
+								_fs += ",?"
+							}
+							vs = append(vs, v)
+						}
+						_fs += ")"
+						fs = append(fs, _fs)
+					}
+				}
+			}
+		} else {
+			if v == "$isNull" {
+				fs = append(fs, fmt.Sprintf("%s is null", k))
+			} else if v == "$isNotNull" {
+				fs = append(fs, fmt.Sprintf("%s is not null", k))
+			} else {
+				fs = append(fs, fmt.Sprintf("%s=?", k))
+				vs = append(vs, v)
+			}
+		}
+	}
+	var buffer bytes.Buffer
+	buffer.WriteString("select ")
+	if fields == "" {
+		buffer.WriteString("*")
+	} else {
+		buffer.WriteString(fields)
+	}
+	buffer.WriteString(" from ")
+	buffer.WriteString(tableName)
+	if len(fs) > 0 {
+		buffer.WriteString(" where ")
+		buffer.WriteString(strings.Join(fs, " and "))
+	}
+	if order != "" {
+		buffer.WriteString(" order by ")
+		buffer.WriteString(order)
+	}
+	if start > -1 && pageSize > 0 {
+		buffer.WriteString(" limit ")
+		buffer.WriteString(fmt.Sprint(start))
+		buffer.WriteString(",")
+		buffer.WriteString(fmt.Sprint(pageSize))
+	}
+	q := buffer.String()
+	//log.Println(q, vs)
+	return m.SelectBySql(q, vs...)
+}
+
+func (m *Ck) SelectBySql(q string, args ...interface{}) *[]map[string]interface{} {
+	return m.SelectBySqlByTx(nil, q, args...)
+}
+func (m *Ck) SelectBySqlByTx(tx *sql.Tx, q string, args ...interface{}) *[]map[string]interface{} {
+	return m.Select(0, nil, tx, q, args...)
+}
+
+func (m *Ck) Select(bath int, f func(l *[]map[string]interface{}), tx *sql.Tx, q string, args ...interface{}) *[]map[string]interface{} {
+	var err error
+	rows, err := m.DB.Query(q, args...)
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	if rows != nil {
+		defer rows.Close()
+	}
+	columns, err := rows.Columns()
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	list := []map[string]interface{}{}
+	for rows.Next() {
+		scanArgs := make([]interface{}, len(columns))
+		values := make([]interface{}, len(columns))
+		ret := make(map[string]interface{})
+		for k, _ := range values {
+			scanArgs[k] = &values[k]
+		}
+		err = rows.Scan(scanArgs...)
+		if err != nil {
+			log.Println(err)
+			break
+		}
+		for i, col := range values {
+			if v, ok := col.([]uint8); ok {
+				ret[columns[i]] = string(v)
+			} else {
+				ret[columns[i]] = col
+			}
+		}
+		list = append(list, ret)
+		if bath > 0 && len(list) == bath {
+			f(&list)
+			list = []map[string]interface{}{}
+		}
+	}
+	if bath > 0 && len(list) > 0 {
+		f(&list)
+		list = []map[string]interface{}{}
+	}
+	return &list
+}

data_project_information/common.toml

@@ -0,0 +1,52 @@
+[serve]
+udp = ":1782"
+thread = 1
+
+[db]
+
+[db.mongo]
+addr = "127.0.0.1:29099"
+dbname = "wjh"
+coll = "projectset"
+size = 15
+user = ""
+password = ""
+[db.mysql]
+addr = "192.168.3.14:4000"
+dbname = "crm"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+[db.es]
+addr = "http://127.0.0.1:19805"
+index = "bidding"
+pool = 10
+user = "es_all"
+password = "TopJkO2E_d1x"
+[db.myck]
+addr = "192.168.3.207:19000"
+dbname = "information"
+user = "jytop"
+password = "pwdTopJy123"
+
+[mail]
+send = false
+to = "wangjianghan@topnet.net.cn"
+api = "http://172.17.145.179:19281/_send/_mail"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"

data_project_information/config/conf.go

@@ -0,0 +1,106 @@
+package config
+
+import (
+	"fmt"
+	"github.com/BurntSushi/toml"
+	"os"
+)
+
+var (
+	// Conf crocodile conf
+	Conf *conf
+)
+
+// Init Config
+func Init(conf string) {
+	_, err := toml.DecodeFile(conf, &Conf)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+}
+
+type conf struct {
+	Serve serve
+	DB    db
+	Mail  mail
+	Log   log
+}
+
+type serve struct {
+	Udp        string
+	LoadStart  int64
+	Thread     int
+	SiteColl   string
+	ValidDays  int
+	StatusDays int
+	BackupFlag bool
+}
+
+type node struct {
+	Addr string
+	Port int
+	Memo string
+}
+
+type mail struct {
+	Send bool
+	To   string
+	Api  string
+}
+
+// Log Config
+type log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type db struct {
+	Mongo mgo
+	Mysql mysql
+	Myck  clickhouse
+	Es    es
+}
+
+type mgo struct {
+	Addr     string
+	Dbname   string
+	Coll     string
+	Size     int
+	User     string
+	Password string
+}
+
+type mysql struct {
+	Addr     string
+	Dbname   string
+	Size     int
+	User     string
+	Password string
+}
+
+type clickhouse struct {
+	Addr     string
+	Dbname   string
+	User     string
+	Password string
+}
+
+type redis struct {
+	Addr   string
+	AddrQb string
+	DbQb   int
+}
+
+type es struct {
+	Addr     string
+	Size     int
+	Index    string
+	User     string
+	Password string
+}

data_project_information/config/conf_test.go

@@ -0,0 +1,33 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var confs = `
+
+[serve]
+
+[db]
+
+[udp]
+
+[[udpNode]]
+addr = "192.168.20.104"
+port = 1783
+memo = "创建项目索引"
+[[udpNode]]
+addr = "127.0.0.1"
+port = 17833
+memo = "修改项目创建"
+`
+
+func TestInit(t *testing.T) {
+	testfile := "/tmp/crocodile.toml"
+	ioutil.WriteFile(testfile, []byte(confs), 0644)
+	Init(testfile)
+	t.Logf("%+v", Conf.UdpNode)
+	os.Remove(testfile)
+}

data_project_information/go.mod

@@ -0,0 +1,52 @@
+module data_project_information
+
+go 1.20
+
+require (
+	github.com/BurntSushi/toml v1.3.2
+	github.com/ClickHouse/clickhouse-go/v2 v2.20.0
+	github.com/olivere/elastic/v7 v7.0.32
+	github.com/spf13/cobra v1.8.0
+	go.mongodb.org/mongo-driver v1.11.4
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67
+)
+
+require (
+	github.com/ClickHouse/ch-go v0.61.3 // indirect
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/paulmach/orb v0.11.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

data_project_information/go.sum

@@ -0,0 +1,265 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/ClickHouse/ch-go v0.61.3 h1:MmBwUhXrAOBZK7n/sWBzq6FdIQ01cuF2SaaO8KlDRzI=
+github.com/ClickHouse/ch-go v0.61.3/go.mod h1:1PqXjMz/7S1ZUaKvwPA3i35W2bz2mAMFeCi6DIXgGwQ=
+github.com/ClickHouse/clickhouse-go/v2 v2.20.0 h1:bvlLQ31XJfl7MxIqAq2l1G6JhHYzqEXdvfpMeU6bkKc=
+github.com/ClickHouse/clickhouse-go/v2 v2.20.0/go.mod h1:VQfyA+tCwCRw2G7ogfY8V0fq/r0yJWzy8UDrjiP/Lbs=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
+github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
+github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
+go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
+go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67 h1:5rabE3IQRjI/NXfgJyquRTMdzSy7SfufFhmREixLZXg=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230627091444-ee2add33ba67/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

data_project_information/init.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"data_project_information/config"
+	"fmt"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
+	"os"
+)
+
+var (
+	Mgo       *mongodb.MongodbSim
+	Es        *elastic.Elastic
+	MysqlTool *mysqldb.Mysql
+	CkhTool   *Ck
+
+	saveSize   int
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+	savePool   chan map[string]interface{}
+	saveSp     chan bool
+
+	BaseField = []string{"source_id", "project_name", "industry_label", "activities_label", "business_label", "bidstatus", "create_time", "update_time", "result_time",
+		"publish_time", "next_bid_time", "contract_cycle_start_time", "contract_cycle_end_time", "contract_cycle", "area", "city", "district", "bidamount", "budget", "first_url", "winner"}
+)
+
+func InitMgo() {
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.Mongo.Addr,
+		DbName:      config.Conf.DB.Mongo.Dbname,
+		Size:        config.Conf.DB.Mongo.Size,
+		UserName:    config.Conf.DB.Mongo.User,
+		Password:    config.Conf.DB.Mongo.Password,
+	}
+	Mgo.InitPool()
+}
+
+func InitEs() {
+	Es = &elastic.Elastic{
+		S_esurl:  config.Conf.DB.Es.Addr,
+		I_size:   config.Conf.DB.Es.Size,
+		Username: config.Conf.DB.Es.User,
+		Password: config.Conf.DB.Es.Password,
+	}
+	Es.InitElasticSize()
+}
+
+func InitMysql() {
+	dbcfg := config.Conf.DB.Mysql
+	MysqlTool = &mysqldb.Mysql{
+		Address:  dbcfg.Addr,
+		DBName:   dbcfg.Dbname,
+		UserName: dbcfg.User,
+		PassWord: dbcfg.Password,
+	}
+	MysqlTool.Init()
+
+	CkhTool = &Ck{
+		Address:  config.Conf.DB.Myck.Addr,
+		UserName: config.Conf.DB.Myck.User,
+		PassWord: config.Conf.DB.Myck.Password,
+		DBName:   config.Conf.DB.Myck.Dbname,
+	}
+	CkhTool.Init()
+}
+
+func InitLog() {
+	logcfg := config.Conf.Log
+
+	err := log.InitLog(
+		log.Path(logcfg.LogPath),
+		log.Level(logcfg.LogLevel),
+		log.Compress(logcfg.Compress),
+		log.MaxSize(logcfg.MaxSize),
+		log.MaxBackups(logcfg.MaxBackups),
+		log.MaxAge(logcfg.MaxAge),
+		log.Format(logcfg.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+}

data_project_information/main.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"data_project_information/config"
+	"fmt"
+	"github.com/spf13/cobra"
+	"time"
+)
+
+func init() {
+	config.Init("./common.toml")
+	InitLog()
+
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 5)
+
+	saveSize = 200
+	savePool = make(chan map[string]interface{}, 5000)
+	saveSp = make(chan bool, 3)
+}
+
+func main() {
+
+	go SaveFunc()
+
+	rootCmd := &cobra.Command{Use: "my cmd"}
+	rootCmd.AddCommand(project())
+	rootCmd.AddCommand(tidb())
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println("rootCmd.Execute failed", err.Error())
+	}
+
+	c := make(chan bool, 1)
+	<-c
+}
+
+func project() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "project",
+		Short: "Start processing project data",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitMgo()
+			InitEs()
+
+			go updateFuc()
+			task()
+		},
+	}
+	//cmdClient.Flags().StringVarP(&cfg, "conf", "c", "", "server config [toml]")
+	return cmdClient
+}
+
+func tidb() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "tidb",
+		Short: "Start processing tidb data",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitMgo()
+			InitMysql()
+
+			go SaveFunc()
+			taskT()
+		},
+	}
+	//cmdClient.Flags().StringVarP(&cfg, "conf", "c", "", "server config [toml]")
+	return cmdClient
+}
+
+func updateFuc() {
+	arru := make([][]map[string]interface{}, 500)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 500 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					Mgo.UpdateBulk(config.Conf.DB.Mongo.Coll, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 500)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					Mgo.UpdateBulk(config.Conf.DB.Mongo.Coll, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 500)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveFunc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MysqlTool.InsertBulk("property_project", BaseField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MysqlTool.InsertBulk("property_project", BaseField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

data_project_information/task.go

@@ -0,0 +1,341 @@
+package main
+
+import (
+	"context"
+	"data_project_information/config"
+	"encoding/json"
+	"fmt"
+	es "github.com/olivere/elastic/v7"
+	"go.mongodb.org/mongo-driver/bson"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"strings"
+	"sync"
+	"time"
+)
+
+func task() {
+
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+
+	wg := &sync.WaitGroup{}
+
+	tf := []int64{1577808000, 1609430400, 1640966400, 1672502400, 1712851200}
+	fsc := es.NewFetchSourceContext(true).Include("tag_topinformation", "tag_subinformation", "tag_set") // 查询字段
+	for i, tm := range tf {
+		query := es.NewBoolQuery().
+			Must(es.NewMatchQuery("tag_topinformation", "情报_物业"))
+		if i == 0 {
+			query.Must(es.NewRangeQuery("comeintime").Lte(tm))
+		} else if i == 1 {
+			query.Must(es.NewRangeQuery("comeintime").Gte(tf[0]).Lte(tm))
+		} else if i == 2 {
+			query.Must(es.NewRangeQuery("comeintime").Gte(tf[1]).Lte(tm))
+		} else if i == 3 {
+			query.Must(es.NewRangeQuery("comeintime").Gte(tf[2]).Lte(tm))
+		} else {
+			query.Must(es.NewRangeQuery("comeintime").Lte(tf[3]))
+		}
+
+		util.Debug(fmt.Sprintf("第%d次查询，数据量为：%d", i+1, Es.Count("bidding", query)))
+		countDocs := 0
+		res, err := client.Scroll().Index("bidding").Query(query).FetchSourceContext(fsc).Scroll("5m").Size(2000).Do(context.Background()) //查询一条获取游标
+		if err == nil {
+			taskInfoA(res, wg, &countDocs)
+			scrollId := res.ScrollId
+			for {
+				searchResult, err := client.Scroll("1m").Index("bidding").ScrollId(scrollId).Size(2000).Do(context.TODO()) //查询
+				if err != nil {
+					util.Debug("Es Search Data Error：", err.Error())
+					break
+				}
+				taskInfoA(searchResult, wg, &countDocs)
+				scrollId = searchResult.ScrollId
+			}
+			wg.Wait()
+			util.Debug(fmt.Sprintf("第%d次处理结束，处理文档%d条", i+1, countDocs))
+			_, _ = client.ClearScroll().ScrollId(scrollId).Do(context.Background()) //清理游标
+		} else {
+			util.Debug(err)
+		}
+	}
+}
+
+func taskInfoA(searchResult *es.SearchResult, wg *sync.WaitGroup, countDocs *int) {
+	chd := make(chan bool, 5)
+	for _, hit := range searchResult.Hits.Hits {
+		//开始处理数据
+		wg.Add(1)
+		chd <- true
+		go func(tmpHit *es.SearchHit) {
+			defer func() {
+				<-chd
+				wg.Done()
+			}()
+			tmp := make(map[string]interface{})
+			if json.Unmarshal(tmpHit.Source, &tmp) == nil {
+				update := make(map[string]interface{})
+				update["tag_topinformation"] = tmp["tag_topinformation"]
+				update["tag_subinformation"] = tmp["tag_subinformation"]
+				if tmp["tag_set"] != nil {
+					update["tag_set"] = tmp["tag_set"]
+				}
+				//updatePool <- []map[string]interface{}{
+				//	{"ids": tmpHit.Id},
+				//	{"$set": update, "$push": bson.M{"tag_information_ids": tmpHit.Id}},
+				//}
+				updatePool <- []map[string]interface{}{
+					{"ids": tmpHit.Id},
+					{"$set": update},
+				}
+
+			}
+		}(hit)
+		*countDocs += 1
+
+		if *countDocs%10000 == 0 {
+			util.Debug("Current：", *countDocs)
+		}
+	}
+}
+
+func taskT() {
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	ch := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+
+	q := bson.M{"firsttime": bson.M{"$gte": 1577808000}}
+	query := sess.DB(config.Conf.DB.Mongo.Dbname).C(config.Conf.DB.Mongo.Coll).Find(q).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%20000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			if tmp["tag_topinformation"] != nil && tmp["tag_subinformation"] != nil {
+				taskinfotA(tmp)
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count))
+}
+
+var YeTai = map[string]string{
+	"21": "住宅",
+	"22": "政府办公楼",
+	"23": "学校",
+	"24": "医院",
+	"25": "产业园区",
+	"26": "旅游景区",
+	"27": "交通运输",
+	"28": "商务办公楼",
+	"29": "酒店",
+}
+var ZhouQi = map[int]string{
+	11: "1年以下",
+	12: "1年",
+	13: "2年",
+	14: "3年",
+	15: "5年",
+	16: "其他",
+}
+
+func taskinfotA(tmp map[string]interface{}) {
+	save := make(map[string]interface{})
+	pid := mongodb.BsonIdToSId(tmp["_id"])
+	ids := util.ObjArrToStringArr(tmp["ids"].([]interface{}))
+	save["source_id"] = pid
+	save["project_name"] = util.ObjToString(tmp["projectname"])
+	save["area"] = tmp["area"]
+	save["city"] = tmp["city"]
+	save["district"] = tmp["district"]
+	save["budget"] = tmp["budget"]
+	save["bidamount"] = tmp["bidamount"]
+	save["first_url"] = fmt.Sprintf("https://www.jianyu360.cn/article/content/%s.html", util.CommonEncodeArticle("content", ids[0]))
+	save["business_label"] = getStr(util.ObjToString(tmp["buyerclass"]))
+
+	//bt := util.ObjToString(tmp["bidtype"])
+	bs := util.ObjToString(tmp["bidstatus"])
+	if bs == "中标" || bs == "成交" || bs == "合同" {
+		save["bidstatus"] = 1
+	} else if bs == "废标" || bs == "流标" {
+		save["bidstatus"] = 0
+	} else {
+		save["bidstatus"] = 2
+	}
+	if tmp["firsttime"] != nil && util.IntAll(tmp["firsttime"]) > 0 {
+		t := util.Int64All(tmp["firsttime"])
+		save["publish_time"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+	}
+	if util.IntAll(save["bidstatus"]) == 1 && util.IntAll(tmp["jgtime"]) > 0 {
+		t := util.Int64All(tmp["jgtime"])
+		save["result_time"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+	}
+	// 补充结果时间
+	//if util.Int64All(save["result_time"]) <= 0 && util.IntAll(save["bidstatus"]) == 1 {
+	//	t := util.Int64All(tmp["lasttime"])
+	//	if t > 0 {
+	//		save["result_time"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+	//	}
+	//}
+
+	save["create_time"] = time.Now().Format(util.Date_Full_Layout)
+	save["update_time"] = time.Now().Format(util.Date_Full_Layout)
+
+	for _, id := range ids {
+		if info := CkhTool.FindOne("information", bson.M{"datajson_id": id}, "starttime,endtime,datajson_expiredate", ""); info != nil && len(*info) > 0 {
+			if (*info)["starttime"] != nil && util.IntAll((*info)["starttime"]) > 0 {
+				t := util.Int64All((*info)["starttime"])
+				save["contract_cycle_start_time"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+			}
+			if (*info)["datajson_expiredate"] != nil && util.IntAll((*info)["datajson_expiredate"]) > 0 {
+				t := util.Int64All((*info)["datajson_expiredate"])
+				save["contract_cycle_end_time"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+			}
+			if (*info)["endtime"] != nil && util.IntAll((*info)["endtime"]) > 0 && save["contract_cycle_end_time"] == nil {
+				t := util.Int64All((*info)["endtime"])
+				save["contract_cycle_end_time"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+			}
+			break
+		}
+	}
+
+	// 行业标签
+	v1 := util.ObjArrToStringArr(tmp["tag_subinformation"].([]interface{}))
+	for _, sub := range v1 {
+		MysqlTool.Insert("property_project_tag", bson.M{"source_id": pid, "value": sub, "stype": 1})
+	}
+	save["industry_label"] = strings.Join(v1, ",")
+	// 物业业态
+	if tagset, ok := tmp["tag_set"].(map[string]interface{}); ok {
+		if wuye, o1 := tagset["wuye"].(map[string]interface{}); o1 {
+			if yt := util.ObjToString(wuye["property_form"]); yt != "" {
+				var v2 []string
+				for _, s := range strings.Split(yt, ",") {
+					if s1 := YeTai[s]; s1 != "" {
+						v2 = append(v2, s1)
+						MysqlTool.Insert("property_project_tag", bson.M{"source_id": pid, "value": s1, "stype": 2})
+					}
+				}
+				save["activities_label"] = strings.Join(v2, ",")
+			}
+			if p := util.IntAll(wuye["period"]); p != 0 && ZhouQi[p] != "" {
+				save["contract_cycle"] = ZhouQi[p]
+				if util.IntAll(save["bidstatus"]) == 1 {
+					if util.IntAll(tmp["jgtime"]) > 0 {
+						if et := getDate(p, util.Int64All(tmp["jgtime"])); et > 0 {
+							save["next_bid_time"] = util.FormatDateByInt64(&et, util.Date_Full_Layout)
+						}
+					} else if t := getDate(p, util.Int64All(tmp["lasttime"])); t > 0 {
+						if t > 0 {
+							save["next_bid_time"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+						}
+					}
+				}
+			}
+		}
+	}
+	// 2023年之后的数据
+	if util.IntAll(save["bidstatus"]) == 1 {
+		if save["next_bid_time"] == nil && util.Int64All(tmp["jgtime"]) > 1672502400 {
+			st, _ := time.Parse(time.DateTime, util.ObjToString(save["contract_cycle_start_time"]))
+			et, _ := time.Parse(time.DateTime, util.ObjToString(save["contract_cycle_end_time"]))
+			if t := et.Unix() - st.Unix(); t < 90*24*60*60 {
+				if tmp["jgtime"] != nil && util.IntAll(tmp["jgtime"]) > 0 {
+					if et1 := getDate(12, util.Int64All(tmp["jgtime"])); et1 > 0 {
+						save["next_bid_time"] = util.FormatDateByInt64(&et1, util.Date_Full_Layout)
+					}
+				} else {
+					if t1 := util.Int64All(tmp["lasttime"]); t1 > 0 {
+						if et1 := getDate(12, util.Int64All(tmp["lasttime"])); et1 > 0 {
+							save["next_bid_time"] = util.FormatDateByInt64(&et1, util.Date_Full_Layout)
+						}
+					}
+				}
+			} else {
+				if t1 := util.Int64All(tmp["jgtime"]); t1 > 0 {
+					et1 := t1 + t
+					save["next_bid_time"] = util.FormatDateByInt64(&et1, util.Date_Full_Layout)
+				} else {
+					if t2 := util.Int64All(tmp["lasttime"]); t2 > 0 {
+						et1 := t2 + t
+						save["next_bid_time"] = util.FormatDateByInt64(&et1, util.Date_Full_Layout)
+					}
+				}
+			}
+		}
+		// 2023年之后的数据，无合约周期默认 1年
+		if save["next_bid_time"] == nil && util.Int64All(save["jgtime"]) > 1672502400 {
+			if et := getDate(12, util.Int64All(tmp["jgtime"])); et > 0 {
+				save["next_bid_time"] = util.FormatDateByInt64(&et, util.Date_Full_Layout)
+			}
+			//else {
+			//	if t := util.Int64All(tmp["lasttime"]); t > 0 {
+			//		if et := getDate(12, util.Int64All(tmp["lasttime"])); et > 0 {
+			//			save["next_bid_time"] = util.FormatDateByInt64(&et, util.Date_Full_Layout)
+			//		}
+			//	}
+			//}
+		}
+	}
+
+	// 中标单位
+	for _, w := range strings.Split(util.ObjToString(tmp["s_winner"]), ",") {
+		if w != "" {
+			MysqlTool.Insert("property_project_tag", bson.M{"source_id": pid, "value": w, "stype": 3})
+		}
+	}
+	save["winner"] = util.ObjToString(tmp["s_winner"])
+
+	savePool <- save
+}
+
+func getStr(b string) string {
+	if b == "" {
+		return ""
+	}
+	a1 := "(交通|运输物流|工信|农业|住建|城管|市政|出版广电|检察院|科技|民政|生态环境|市场监管|水利|应急管理|自然\n资源|财政|档案|党委办|组织|发改|宣传|政府办|政务中心|人大|政协|法院|公安|国资委|海关|机关事务|纪委|军队|人社|商务|审计税务|司法|体育|统计|统战|文旅|民宗|银保监|证监|气象|社会团体|公共资源交易)"
+	a2 := "(卫健委|医疗)"
+	a3 := "(教育|学校)"
+	a4 := "(人行l金融业)"
+	a5 := "(信息技术|电信行业|农林牧渔|建筑业|传媒|制造业|住宿餐饮|采矿业|能源化工|批发零售)"
+	if strings.Contains(a1, b) {
+		return "政府机构"
+	} else if strings.Contains(a2, b) {
+		return "医疗单位"
+	} else if strings.Contains(a3, b) {
+		return "教育单位"
+	} else if strings.Contains(a4, b) {
+		return "金融企业"
+	} else if strings.Contains(a5, b) {
+		return "商业公司"
+	}
+	return ""
+}
+func getDate(p int, stime int64) int64 {
+	if p == 11 {
+	} else if p == 12 {
+		return stime + (365 * 24 * 60 * 60)
+	} else if p == 13 {
+		return stime + (365 * 24 * 60 * 60 * 2)
+	} else if p == 14 {
+		return stime + (365 * 24 * 60 * 60 * 3)
+	} else if p == 15 {
+		return stime + (365 * 24 * 60 * 60 * 5)
+	}
+
+	return 0
+}

field_sync/task.go

@@ -812,7 +812,6 @@ func getTagSet(tmp, compare map[string]interface{}) map[string]map[string]interf
 			changehand := fmt.Sprintf("%.2f", float64(count)/float64(len(*data)))
 			changehands, _ := strconv.ParseFloat(changehand, 64)
 			wuye["changehand"] = changehands
-			log.Println("changehands", count, len(*data), changehands)
 			if changehands > 0.3 {
 				wuye["changehandindex"] = 61
 			}