cm

README.md

@@ -0,0 +1,7 @@
+## 数据处理流程-python服务处理字段
+
+### 采购意向 procurementlist、
++ rpc serve-name goods_service
+
+### 评标专家 review_experts
++ rpc serve-name extract_expert_service

common.toml

@@ -0,0 +1,54 @@
+
+[serve]
+grpcAddr = "192.168.3.12:10021"
+
+[db]
+
+[db.mongo]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+size = 5
+user = ""
+password = ""
+
+[db.mongo1]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+size = 5
+user = ""
+password = ""
+
+[db.es]
+addr = "http://192.168.3.206:9800"
+size = 5
+indexs = "bidding"
+types = "bidding"
+
+[udp]
+locport = ":1786"
+[udp.next]
+addr = "127.0.0.1"
+port = 1783
+stype = "extract"
+
+[mail]
+send = false
+to = "wangjianghan@topnet.net.cn"
+api = "http://172.17.145.179:19281/_send/_mail"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"

config/conf.go

@@ -0,0 +1,95 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	// Conf crocodile conf
+	Conf *conf
+)
+
+// Init Config
+func Init(conf string) {
+	_, err := toml.DecodeFile(conf, &Conf)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+}
+
+type conf struct {
+	Serve serve
+	DB    db
+	Udp   udp
+	Mail  mail
+	Log   log
+}
+
+type serve struct {
+	GrpcAddr string
+}
+
+type udp struct {
+	LocPort string
+	Next    udpNext
+}
+
+type udpNext struct {
+	Addr  string
+	Port  int
+	Stype string
+}
+
+type mail struct {
+	Send bool
+	To   string
+	Api  string
+}
+
+// Log Config
+type log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type db struct {
+	Mongo  mgo
+	Mongo1 mgo
+	Es     es
+}
+
+type mgo struct {
+	Addr     string
+	Dbname   string
+	Size     int
+	User     string
+	Password string
+}
+
+type es struct {
+	Addr   string
+	Size   int
+	IndexS string
+	TypeS  string
+}
+
+type duration struct {
+	time.Duration
+}
+
+// UnmarshalText parse 10s to time.Time
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}

config/conf_test.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var confs = `# log
+[log]
+logpath = ""
+maxsize = 10
+compress = true
+maxage =  7
+maxbackups = 10
+loglevel  = "info"
+format = "text"
+
+[serve]
+grpcAddr = "192.168.3.12:10021"
+udpPort = "1782"
+
+[db]
+[db.mongo]
+addr = "192.168.3.207:27092"
+dbname = "qfw"
+size = 10
+user = ""
+password = ""
+[db.mongo1]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+size = 5
+user = ""
+password = ""
+
+[db.es]
+addr = "http://192.168.3.206:9800"
+size = 5
+indexm = "medical_institution_v1"
+typem = "medical_institution"
+indexs = "supplier_product_v1"
+types = "supplier_product"
+
+`
+
+func TestInit(t *testing.T) {
+	testfile := "/tmp/crocodile.toml"
+	ioutil.WriteFile(testfile, []byte(confs), 0644)
+	Init(testfile)
+	t.Logf("%+v", Conf.Serve.GrpcAddr)
+	os.Remove(testfile)
+}

go.mod

@@ -0,0 +1,12 @@
+module field-dispose
+
+go 1.16
+
+require (
+	app.yhyue.com/BP/servicerd v0.0.0-20201203055056-87643512f867
+	app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f
+	github.com/BurntSushi/toml v1.2.0
+	go.uber.org/zap v1.22.0
+	google.golang.org/grpc v1.49.0
+	google.golang.org/protobuf v1.27.1
+)

go.sum

@@ -0,0 +1,237 @@
+app.yhyue.com/BP/servicerd v0.0.0-20201203055056-87643512f867 h1:NBRydlfT1w5LLKCBAf1yfhDNlaBWLam4xQBp1Nn667U=
+app.yhyue.com/BP/servicerd v0.0.0-20201203055056-87643512f867/go.mod h1:vDomUktO0sHs06GW9XSrNjYA8oOkrVDB+TvFKeKSsUU=
+app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f h1:5fUbVRwPM3oBsZgvG76Bia3I4SdwdBB6PvJ6B28Qkyc=
+app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-memdb v1.2.1/go.mod h1:OSvLJ662Jim8hMM+gWGyhktyWk2xPCnWMc7DWIqtkGA=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mattn/go-sqlite3 v1.14.2/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/shirou/gopsutil v2.20.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw=
+google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/olivere/elastic.v2 v2.0.61 h1:7cpl3MW8ysa4GYFBXklpo5mspe4NK0rpZTdyZ+QcD4U=
+gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

init.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"app.yhyue.com/data_processing/common_utils/log"
+	"field-dispose/config"
+	"fmt"
+	"os"
+)
+
+// InitLog @Description
+// @Author J 2022/7/26 15:30
+func InitLog() {
+	logcfg := config.Conf.Log
+
+	err := log.InitLog(
+		log.Path(logcfg.LogPath),
+		log.Level(logcfg.LogLevel),
+		log.Compress(logcfg.Compress),
+		log.MaxSize(logcfg.MaxSize),
+		log.MaxBackups(logcfg.MaxBackups),
+		log.MaxAge(logcfg.MaxAge),
+		log.Format(logcfg.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+}

main.go

@@ -0,0 +1,228 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/elastic"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/udp"
+	"encoding/json"
+	"field-dispose/config"
+	"fmt"
+	"go.uber.org/zap"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+)
+
+var (
+	MgoB *mongodb.MongodbSim
+	Es   *elastic.Elastic
+
+	UdpClient  udp.UdpClient
+	UdpTaskMap = &sync.Map{}
+
+	updatePool   chan []map[string]interface{}
+	updateSp     chan bool
+	updateEsPool chan []map[string]interface{}
+	updateEsSp   chan bool
+)
+
+func init() {
+	config.Init("./common.toml")
+	InitLog()
+	InitMgo()
+
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 5)
+	updateEsPool = make(chan []map[string]interface{}, 5000)
+	updateEsSp = make(chan bool, 2)
+
+	UdpClient = udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
+	UdpClient.Listen(processUdpMsg)
+	log.Info("Udp服务监听", zap.String("port:", config.Conf.Udp.LocPort))
+}
+
+type UdpNode struct {
+	data      []byte
+	addr      *net.UDPAddr
+	timestamp int64
+	retry     int
+}
+
+func main() {
+	go SaveErrorInfo() //保存异常信息
+	//go CheckErrorNum()
+	go updateEsMethod()
+	go checkMapJob()
+	go updateMethod()
+
+	ch := make(chan bool, 1)
+	<-ch
+}
+func InitMgo() {
+	MgoB = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.Mongo.Addr,
+		DbName:      config.Conf.DB.Mongo.Dbname,
+		Size:        config.Conf.DB.Mongo.Size,
+		UserName:    config.Conf.DB.Mongo.User,
+		Password:    config.Conf.DB.Mongo.Password,
+	}
+	MgoB.InitPool()
+
+	Es = &elastic.Elastic{
+		S_esurl: config.Conf.DB.Es.Addr,
+		I_size:  config.Conf.DB.Es.Size,
+	}
+	Es.InitElasticSize()
+}
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	defer util.Catch()
+	switch act {
+	case udp.OP_TYPE_DATA: //上个节点的数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Info("processUdpMsg", zap.Any("mapInfo:", mapInfo))
+		gtid, _ := mapInfo["gtid"].(string)
+		lteid, _ := mapInfo["lteid"].(string)
+		if err != nil || gtid == "" || lteid == "" {
+			UdpClient.WriteUdp([]byte("cgyx udp error"), udp.OP_NOOP, ra) //udp失败回写
+		} else {
+			//udp成功回写
+			if k := util.ObjToString(mapInfo["key"]); k != "" {
+				UdpClient.WriteUdp([]byte(k), udp.OP_NOOP, ra)
+			} else {
+				k = fmt.Sprintf("%s-%s-%s", gtid, lteid, util.ObjToString(mapInfo["stype"]))
+				UdpClient.WriteUdp([]byte(k), udp.OP_NOOP, ra)
+			}
+			log.Info("start dispose ...")
+			getIntention(gtid, lteid, mapInfo)
+		}
+	case udp.OP_NOOP: //下个节点回应
+		ok := string(data)
+		if ok != "" {
+			log.Info("udp re", zap.String("data:", ok))
+			UdpTaskMap.Delete(ok)
+		}
+	}
+}
+
+func NextNode(mapInfo map[string]interface{}) {
+	var next = &net.UDPAddr{
+		IP:   net.ParseIP(config.Conf.Udp.Next.Addr),
+		Port: util.IntAll(config.Conf.Udp.Next.Port),
+	}
+	mapInfo["stype"] = config.Conf.Udp.Next.Stype
+	key := fmt.Sprintf("%s-%s-%s", util.ObjToString(mapInfo["gtid"]), util.ObjToString(mapInfo["lteid"]), config.Conf.Udp.Next.Stype)
+	mapInfo["key"] = key
+	log.Info("udp next node", zap.Any("mapinfo:", mapInfo))
+	datas, _ := json.Marshal(mapInfo)
+	node := &UdpNode{datas, next, time.Now().Unix(), 0}
+	UdpTaskMap.Store(key, node)
+	_ = UdpClient.WriteUdp(datas, udp.OP_TYPE_DATA, next)
+}
+
+func checkMapJob() {
+	if config.Conf.Mail.Send {
+		log.Info("checkMapJob", zap.String("to:", config.Conf.Mail.To))
+		for {
+			UdpTaskMap.Range(func(k, v interface{}) bool {
+				now := time.Now().Unix()
+				node, _ := v.(*UdpNode)
+				if now-node.timestamp > 120 {
+					node.retry++
+					if node.retry > 5 {
+						UdpTaskMap.Delete(k)
+						res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", config.Conf.Mail.Api, config.Conf.Mail.To, "project-send-fail", k.(string)))
+						if err == nil {
+							defer res.Body.Close()
+							read, err := ioutil.ReadAll(res.Body)
+							log.Info("send mail ...", zap.String("r:", string(read)), zap.Any("err:", err))
+						}
+					} else {
+						log.Info("udp重发", zap.Any("k:", k))
+						UdpClient.WriteUdp(node.data, udp.OP_TYPE_DATA, node.addr)
+					}
+				} else if now-node.timestamp > 10 {
+					log.Info("udp任务超时中..", zap.Any("k:", k))
+				}
+				return true
+			})
+			time.Sleep(60 * time.Second)
+		}
+	}
+}
+
+func updateMethod() {
+	log.Info("updateMethod 保存...")
+	arru := make([][]map[string]interface{}, 500)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 500 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MgoB.UpdateBulk("bidding", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 500)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MgoB.UpdateBulk("bidding", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 500)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func updateEsMethod() {
+	arru := make([][]map[string]interface{}, 200)
+	indexu := 0
+	for {
+		select {
+		case v := <-updateEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 200 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk(config.Conf.DB.Es.IndexS, config.Conf.DB.Es.TypeS, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk(config.Conf.DB.Es.IndexS, config.Conf.DB.Es.TypeS, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 200)
+				indexu = 0
+			}
+		}
+	}
+}

proto/exists_expert.pb.go

@@ -0,0 +1,298 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.27.1
+// 	protoc        v3.12.4
+// source: proto/exists_expert.proto
+
+// 标的物抽取
+
+package proto
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type ContentRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Contents string `protobuf:"bytes,1,opt,name=contents,proto3" json:"contents,omitempty"` //json{"文档ID"：detail}  如果公告为附件 {"文档ID"：附件}
+}
+
+func (x *ContentRequest) Reset() {
+	*x = ContentRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_exists_expert_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ContentRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ContentRequest) ProtoMessage() {}
+
+func (x *ContentRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_exists_expert_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ContentRequest.ProtoReflect.Descriptor instead.
+func (*ContentRequest) Descriptor() ([]byte, []int) {
+	return file_proto_exists_expert_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *ContentRequest) GetContents() string {
+	if x != nil {
+		return x.Contents
+	}
+	return ""
+}
+
+type ExpertResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Results string `protobuf:"bytes,1,opt,name=results,proto3" json:"results,omitempty"` //json {'result': {文档id: [], 文档id: ['杜春节', '王颖', '陈永富', '左崴东', '赵红', '万荣祥', '顾炳中']}, 'status': 200}
+}
+
+func (x *ExpertResponse) Reset() {
+	*x = ExpertResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_exists_expert_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ExpertResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ExpertResponse) ProtoMessage() {}
+
+func (x *ExpertResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_exists_expert_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ExpertResponse.ProtoReflect.Descriptor instead.
+func (*ExpertResponse) Descriptor() ([]byte, []int) {
+	return file_proto_exists_expert_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *ExpertResponse) GetResults() string {
+	if x != nil {
+		return x.Results
+	}
+	return ""
+}
+
+var File_proto_exists_expert_proto protoreflect.FileDescriptor
+
+var file_proto_exists_expert_proto_rawDesc = []byte{
+	0x0a, 0x19, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x65, 0x78, 0x69, 0x73, 0x74, 0x73, 0x5f, 0x65,
+	0x78, 0x70, 0x65, 0x72, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x22, 0x2c, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73,
+	0x22, 0x2a, 0x0a, 0x0e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x32, 0x47, 0x0a, 0x0c,
+	0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x45, 0x78, 0x70, 0x65, 0x72, 0x74, 0x12, 0x37, 0x0a, 0x07,
+	0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
+	0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
+	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_proto_exists_expert_proto_rawDescOnce sync.Once
+	file_proto_exists_expert_proto_rawDescData = file_proto_exists_expert_proto_rawDesc
+)
+
+func file_proto_exists_expert_proto_rawDescGZIP() []byte {
+	file_proto_exists_expert_proto_rawDescOnce.Do(func() {
+		file_proto_exists_expert_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_exists_expert_proto_rawDescData)
+	})
+	return file_proto_exists_expert_proto_rawDescData
+}
+
+var file_proto_exists_expert_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_proto_exists_expert_proto_goTypes = []interface{}{
+	(*ContentRequest)(nil), // 0: proto.ContentRequest
+	(*ExpertResponse)(nil), // 1: proto.ExpertResponse
+}
+var file_proto_exists_expert_proto_depIdxs = []int32{
+	0, // 0: proto.ExistsExpert.extract:input_type -> proto.ContentRequest
+	1, // 1: proto.ExistsExpert.extract:output_type -> proto.ExpertResponse
+	1, // [1:2] is the sub-list for method output_type
+	0, // [0:1] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_proto_exists_expert_proto_init() }
+func file_proto_exists_expert_proto_init() {
+	if File_proto_exists_expert_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_proto_exists_expert_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ContentRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_exists_expert_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ExpertResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_proto_exists_expert_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_proto_exists_expert_proto_goTypes,
+		DependencyIndexes: file_proto_exists_expert_proto_depIdxs,
+		MessageInfos:      file_proto_exists_expert_proto_msgTypes,
+	}.Build()
+	File_proto_exists_expert_proto = out.File
+	file_proto_exists_expert_proto_rawDesc = nil
+	file_proto_exists_expert_proto_goTypes = nil
+	file_proto_exists_expert_proto_depIdxs = nil
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// ExistsExpertClient is the client API for ExistsExpert service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type ExistsExpertClient interface {
+	Extract(ctx context.Context, in *ContentRequest, opts ...grpc.CallOption) (*ExpertResponse, error)
+}
+
+type existsExpertClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewExistsExpertClient(cc grpc.ClientConnInterface) ExistsExpertClient {
+	return &existsExpertClient{cc}
+}
+
+func (c *existsExpertClient) Extract(ctx context.Context, in *ContentRequest, opts ...grpc.CallOption) (*ExpertResponse, error) {
+	out := new(ExpertResponse)
+	err := c.cc.Invoke(ctx, "/proto.ExistsExpert/extract", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// ExistsExpertServer is the server API for ExistsExpert service.
+type ExistsExpertServer interface {
+	Extract(context.Context, *ContentRequest) (*ExpertResponse, error)
+}
+
+// UnimplementedExistsExpertServer can be embedded to have forward compatible implementations.
+type UnimplementedExistsExpertServer struct {
+}
+
+func (*UnimplementedExistsExpertServer) Extract(context.Context, *ContentRequest) (*ExpertResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Extract not implemented")
+}
+
+func RegisterExistsExpertServer(s *grpc.Server, srv ExistsExpertServer) {
+	s.RegisterService(&_ExistsExpert_serviceDesc, srv)
+}
+
+func _ExistsExpert_Extract_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ContentRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ExistsExpertServer).Extract(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/proto.ExistsExpert/Extract",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ExistsExpertServer).Extract(ctx, req.(*ContentRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _ExistsExpert_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "proto.ExistsExpert",
+	HandlerType: (*ExistsExpertServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "extract",
+			Handler:    _ExistsExpert_Extract_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "proto/exists_expert.proto",
+}

proto/exists_expert.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+// 标的物抽取
+package proto;
+
+option  go_package = "/proto";
+
+message ContentRequest{
+  string contents = 1; //json{"文档ID"：detail}  如果公告为附件 {"文档ID"：附件}
+}
+message ExpertResponse{
+  string results = 1; //json {'result': {文档id: [], 文档id: ['杜春节', '王颖', '陈永富', '左崴东', '赵红', '万荣祥', '顾炳中']}, 'status': 200}
+}
+
+service ExistsExpert{
+  rpc extract(ContentRequest) returns (ExpertResponse);
+}

proto/getgoods.pb.go

@@ -0,0 +1,297 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.27.1
+// 	protoc        v3.12.4
+// source: proto/getgoods.proto
+
+// 标的物抽取
+
+package proto
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type GoodsRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Contents string `protobuf:"bytes,1,opt,name=contents,proto3" json:"contents,omitempty"`
+}
+
+func (x *GoodsRequest) Reset() {
+	*x = GoodsRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_getgoods_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *GoodsRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GoodsRequest) ProtoMessage() {}
+
+func (x *GoodsRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_getgoods_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use GoodsRequest.ProtoReflect.Descriptor instead.
+func (*GoodsRequest) Descriptor() ([]byte, []int) {
+	return file_proto_getgoods_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *GoodsRequest) GetContents() string {
+	if x != nil {
+		return x.Contents
+	}
+	return ""
+}
+
+type GoodsResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Goods string `protobuf:"bytes,1,opt,name=Goods,proto3" json:"Goods,omitempty"`
+}
+
+func (x *GoodsResponse) Reset() {
+	*x = GoodsResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_getgoods_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *GoodsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GoodsResponse) ProtoMessage() {}
+
+func (x *GoodsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_getgoods_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use GoodsResponse.ProtoReflect.Descriptor instead.
+func (*GoodsResponse) Descriptor() ([]byte, []int) {
+	return file_proto_getgoods_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *GoodsResponse) GetGoods() string {
+	if x != nil {
+		return x.Goods
+	}
+	return ""
+}
+
+var File_proto_getgoods_proto protoreflect.FileDescriptor
+
+var file_proto_getgoods_proto_rawDesc = []byte{
+	0x0a, 0x14, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x74, 0x67, 0x6f, 0x6f, 0x64, 0x73,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x2a, 0x0a,
+	0x0c, 0x47, 0x6f, 0x6f, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a,
+	0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x47, 0x6f, 0x6f,
+	0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x47, 0x6f,
+	0x6f, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x47, 0x6f, 0x6f, 0x64, 0x73,
+	0x32, 0x49, 0x0a, 0x0c, 0x47, 0x6f, 0x6f, 0x64, 0x73, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74,
+	0x12, 0x39, 0x0a, 0x0c, 0x47, 0x6f, 0x6f, 0x64, 0x73, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74,
+	0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x6f, 0x6f, 0x64, 0x73, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x6f,
+	0x6f, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x08, 0x5a, 0x06, 0x2f,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_proto_getgoods_proto_rawDescOnce sync.Once
+	file_proto_getgoods_proto_rawDescData = file_proto_getgoods_proto_rawDesc
+)
+
+func file_proto_getgoods_proto_rawDescGZIP() []byte {
+	file_proto_getgoods_proto_rawDescOnce.Do(func() {
+		file_proto_getgoods_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_getgoods_proto_rawDescData)
+	})
+	return file_proto_getgoods_proto_rawDescData
+}
+
+var file_proto_getgoods_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_proto_getgoods_proto_goTypes = []interface{}{
+	(*GoodsRequest)(nil),  // 0: proto.GoodsRequest
+	(*GoodsResponse)(nil), // 1: proto.GoodsResponse
+}
+var file_proto_getgoods_proto_depIdxs = []int32{
+	0, // 0: proto.GoodsExtract.GoodsExtract:input_type -> proto.GoodsRequest
+	1, // 1: proto.GoodsExtract.GoodsExtract:output_type -> proto.GoodsResponse
+	1, // [1:2] is the sub-list for method output_type
+	0, // [0:1] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_proto_getgoods_proto_init() }
+func file_proto_getgoods_proto_init() {
+	if File_proto_getgoods_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_proto_getgoods_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GoodsRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_getgoods_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GoodsResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_proto_getgoods_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_proto_getgoods_proto_goTypes,
+		DependencyIndexes: file_proto_getgoods_proto_depIdxs,
+		MessageInfos:      file_proto_getgoods_proto_msgTypes,
+	}.Build()
+	File_proto_getgoods_proto = out.File
+	file_proto_getgoods_proto_rawDesc = nil
+	file_proto_getgoods_proto_goTypes = nil
+	file_proto_getgoods_proto_depIdxs = nil
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConnInterface
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion6
+
+// GoodsExtractClient is the client API for GoodsExtract service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type GoodsExtractClient interface {
+	GoodsExtract(ctx context.Context, in *GoodsRequest, opts ...grpc.CallOption) (*GoodsResponse, error)
+}
+
+type goodsExtractClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewGoodsExtractClient(cc grpc.ClientConnInterface) GoodsExtractClient {
+	return &goodsExtractClient{cc}
+}
+
+func (c *goodsExtractClient) GoodsExtract(ctx context.Context, in *GoodsRequest, opts ...grpc.CallOption) (*GoodsResponse, error) {
+	out := new(GoodsResponse)
+	err := c.cc.Invoke(ctx, "/proto.GoodsExtract/GoodsExtract", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// GoodsExtractServer is the server API for GoodsExtract service.
+type GoodsExtractServer interface {
+	GoodsExtract(context.Context, *GoodsRequest) (*GoodsResponse, error)
+}
+
+// UnimplementedGoodsExtractServer can be embedded to have forward compatible implementations.
+type UnimplementedGoodsExtractServer struct {
+}
+
+func (*UnimplementedGoodsExtractServer) GoodsExtract(context.Context, *GoodsRequest) (*GoodsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GoodsExtract not implemented")
+}
+
+func RegisterGoodsExtractServer(s *grpc.Server, srv GoodsExtractServer) {
+	s.RegisterService(&_GoodsExtract_serviceDesc, srv)
+}
+
+func _GoodsExtract_GoodsExtract_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GoodsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(GoodsExtractServer).GoodsExtract(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/proto.GoodsExtract/GoodsExtract",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(GoodsExtractServer).GoodsExtract(ctx, req.(*GoodsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _GoodsExtract_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "proto.GoodsExtract",
+	HandlerType: (*GoodsExtractServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GoodsExtract",
+			Handler:    _GoodsExtract_GoodsExtract_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "proto/getgoods.proto",
+}

proto/getgoods.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+// 标的物抽取
+package proto;
+
+option  go_package = "/proto";
+
+message GoodsRequest{
+  string contents = 1;
+}
+message GoodsResponse{
+  string Goods = 1;
+}
+
+service GoodsExtract{
+  rpc GoodsExtract(GoodsRequest) returns (GoodsResponse);
+}

task.go

@@ -0,0 +1,359 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"field-dispose/config"
+	"field-dispose/proto"
+	"fmt"
+	"go.uber.org/zap"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	service "app.yhyue.com/BP/servicerd/proto"
+)
+
+var IpDialErrNum, IpGetErrNum, ExtractDialErrNum = int64(0), int64(0), int64(0)
+
+//@Description 处理字段procurementlist、review_experts
+// @Author J 2022/8/31 14:57
+func getIntention(gtid, lteid string, mapinfo map[string]interface{}) {
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+	ch := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+	query := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  mongodb.StringTOBsonId(gtid),
+			"$lte": mongodb.StringTOBsonId(lteid),
+		},
+	}
+	field := map[string]interface{}{
+		"toptype":     1,
+		"attach_text": 1,
+		"contenthtml": 1,
+		"site":        1,
+		"detail":      1,
+	}
+	log.Info(fmt.Sprintf("%d", MgoB.Count("bidding", query)))
+	it := sess.DB(config.Conf.DB.Mongo.Dbname).C("bidding").Find(&query).Select(&field).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(tmp); count++ {
+		if count%20000 == 0 {
+			log.Info("getIntention", zap.Int("current:", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			update := make(map[string]interface{})
+			result2 := taskB(tmp, gtid, lteid)
+			//result1 := taskA(tmp, gtid, lteid)
+			if r, ok := result2["result"].(map[string]interface{}); ok {
+				if r[id] != nil && len(r[id].([]interface{})) > 0 {
+					update["review_experts"] = strings.Join(util.ObjArrToStringArr(r[id].([]interface{})), ",")
+				}
+			}
+			//if result1 != nil && len(result1) > 0 {
+			//	update = result1
+			//}
+			if len(update) > 0 {
+				updatePool <- []map[string]interface{}{
+					{"_id": tmp["_id"]},
+					{"$set": update},
+				}
+				updateEsPool <- []map[string]interface{}{{
+					"_id": id,
+				},
+					update,
+				}
+			}
+		}(tmp)
+		tmp = map[string]interface{}{}
+	}
+	wg.Wait()
+	log.Info("dispose over...", zap.Int("count:", count), zap.String("gtid:", gtid), zap.String("lteid:", lteid))
+	//NextNode(mapinfo)
+}
+
+// @Description procurementlist
+// @Author J 2022/8/31 15:28
+func taskA(tmp map[string]interface{}, gtid, lteid string) map[string]interface{} {
+	toptype, _ := tmp["toptype"].(string)
+	if toptype != "采购意向" {
+		return nil
+	}
+	id := mongodb.BsonIdToSId(tmp["_id"])
+	delete(tmp, "_id")
+	delete(tmp, "detail")
+	reqStr, err := json.Marshal(tmp)
+	if err != nil {
+		ErrorInfoCache <- map[string]interface{}{
+			"err":        "Json Marshal Error",
+			"type":       "采购意向",
+			"id":         id,
+			"comeintime": time.Now().Unix(),
+			"ok":         false,
+			"gtid":       gtid,
+			"letid":      lteid,
+		}
+		return nil
+	}
+	//处理数据
+	result, err := rpcGetFieldP(string(reqStr))
+	if err != nil { //保存处理异常信息
+		ErrorInfoCache <- map[string]interface{}{
+			"err":        err.Error(),
+			"type":       "采购意向",
+			"id":         id,
+			"comeintime": time.Now().Unix(),
+			"ok":         false,
+			"gtid":       gtid,
+			"letid":      lteid,
+		}
+	}
+	return result
+}
+
+// @Description review_experts
+// @Author J 2022/8/31 15:28
+func taskB(tmp map[string]interface{}, gtid, lteid string) map[string]interface{} {
+	id := mongodb.BsonIdToSId(tmp["_id"])
+	reqStr, err := json.Marshal(map[string]interface{}{id: tmp["detail"]})
+	if err != nil {
+		ErrorInfoCache <- map[string]interface{}{
+			"err":        "Json Marshal Error",
+			"type":       "评标专家字段",
+			"id":         id,
+			"comeintime": time.Now().Unix(),
+			"ok":         false,
+			"gtid":       gtid,
+			"letid":      lteid,
+		}
+		return nil
+	}
+	//处理数据
+	result, err := rpcGetFieldR(string(reqStr))
+	if err != nil { //保存处理异常信息
+		ErrorInfoCache <- map[string]interface{}{
+			"err":        err.Error(),
+			"type":       "评标专家字段",
+			"id":         id,
+			"comeintime": time.Now().Unix(),
+			"ok":         false,
+			"gtid":       gtid,
+			"letid":      lteid,
+		}
+	}
+	return result
+}
+
+func rpcGetFieldP(reqStr string) (map[string]interface{}, error) {
+	defer util.Catch()
+	//获取ip、port服务
+	ipConn, ipErr := grpc.Dial(config.Conf.Serve.GrpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if ipErr != nil {
+		atomic.AddInt64(&IpDialErrNum, 1) //异常次数+1
+		return nil, errors.New("Ip Grpc Dial Error")
+	}
+	atomic.StoreInt64(&IpDialErrNum, 0) //异常次数重置
+	defer ipConn.Close()
+	ipClient := service.NewServiceClient(ipConn)
+	ip := ""
+	port := -1
+	//重试获取ip、port
+	for i := 1; i <= 3; i++ {
+		repl, err := ipClient.Apply(context.Background(), &service.ApplyReqData{Name: "goods_service", Balance: 0})
+		if err != nil {
+			continue
+		} else {
+			ip = repl.Ip
+			port = int(repl.Port)
+			break
+		}
+	}
+	if ip == "" || port == -1 { //重试三次，回去ip、port失败
+		atomic.AddInt64(&IpGetErrNum, 1) //异常次数+1
+		return nil, errors.New("Get Ip Error")
+	}
+	atomic.StoreInt64(&IpGetErrNum, 0) //异常次数重置
+	//处理数据
+	addr := ip + ":" + fmt.Sprint(port)
+	conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		atomic.AddInt64(&ExtractDialErrNum, 1) //异常次数+1
+		return nil, errors.New("Extract Grpc Dial Error,addr:" + addr)
+	}
+	atomic.StoreInt64(&ExtractDialErrNum, 0) //异常次数重置
+	defer conn.Close()
+	client := proto.NewGoodsExtractClient(conn)
+	req := &proto.GoodsRequest{
+		Contents: reqStr,
+	}
+	resp, err := client.GoodsExtract(context.Background(), req)
+	if err != nil {
+		return nil, errors.New("Deal Data Error")
+	}
+	result := map[string]interface{}{}
+	if json.Unmarshal([]byte(resp.Goods), &result) != nil {
+		return nil, errors.New("Json Unmarshal Error")
+	}
+	return result, nil
+}
+
+func rpcGetFieldR(reqStr string) (map[string]interface{}, error) {
+	defer util.Catch()
+	//获取ip、port服务
+	ipConn, ipErr := grpc.Dial(config.Conf.Serve.GrpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if ipErr != nil {
+		atomic.AddInt64(&IpDialErrNum, 1) //异常次数+1
+		return nil, errors.New("Ip Grpc Dial Error")
+	}
+	atomic.StoreInt64(&IpDialErrNum, 0) //异常次数重置
+	defer ipConn.Close()
+	ipClient := service.NewServiceClient(ipConn)
+	ip := ""
+	port := -1
+	//重试获取ip、port
+	for i := 1; i <= 3; i++ {
+		repl, err := ipClient.Apply(context.Background(), &service.ApplyReqData{Name: "extract_expert_service", Balance: 0})
+		if err != nil {
+			continue
+		} else {
+			ip = repl.Ip
+			port = int(repl.Port)
+			break
+		}
+	}
+	if ip == "" || port == -1 { //重试三次，回去ip、port失败
+		atomic.AddInt64(&IpGetErrNum, 1) //异常次数+1
+		return nil, errors.New("Get Ip Error")
+	}
+	atomic.StoreInt64(&IpGetErrNum, 0) //异常次数重置
+	//处理数据
+	addr := ip + ":" + fmt.Sprint(port)
+	conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		atomic.AddInt64(&ExtractDialErrNum, 1) //异常次数+1
+		return nil, errors.New("Extract Grpc Dial Error,addr:" + addr)
+	}
+	atomic.StoreInt64(&ExtractDialErrNum, 0) //异常次数重置
+	defer conn.Close()
+	client := proto.NewExistsExpertClient(conn)
+	req := &proto.ContentRequest{
+		Contents: reqStr,
+	}
+	resp, err := client.Extract(context.Background(), req)
+	if err != nil {
+		return nil, errors.New("Deal Data Error")
+	}
+	result := make(map[string]interface{})
+	if json.Unmarshal([]byte(resp.Results), &result) != nil {
+		return nil, errors.New("Json Unmarshal Error")
+	}
+	return result, nil
+}
+
+//定时检测异常数据量，发告警
+func CheckErrorNum() {
+	defer util.Catch()
+	for {
+		warnText := ""
+		//消息治理中心服务连接异常告警
+		if IpDialErrNum > 3 {
+			warnText += "消息治理中心连接失败次数：" + fmt.Sprint(IpDialErrNum) + ";\n"
+		}
+		//获取治理中心IP异常告警
+		if IpDialErrNum > 5 {
+			warnText += "获取IP异常次数：" + fmt.Sprint(IpDialErrNum) + ";\n"
+		}
+		//连接IP服务异常告警
+		if IpDialErrNum > 5 {
+			warnText += "IP连接异常次数：" + fmt.Sprint(IpDialErrNum) + ";\n"
+		}
+		if warnText != "" {
+			SendWarnInfo(warnText)
+		}
+		time.Sleep(30 * time.Minute)
+	}
+}
+
+var TextModel = `{
+    "msgtype": "text",
+    "text": {
+        "content": "%s",
+        "mentioned_mobile_list":[%s]
+    }
+}`
+
+//告警信息
+func SendWarnInfo(content string) {
+	defer util.Catch()
+	toUserMsg := fmt.Sprintf(TextModel, content, "13373929153,15090279371") //李俊亮、王江含
+	log.Info("SendWarnInfo", zap.String("toUserMsg", toUserMsg))
+	resp1, err := http.Post(
+		"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=45962efc-ca87-4996-9ffa-08bf6608ab7a",
+		"application/json",
+		bytes.NewBuffer([]byte(toUserMsg)),
+	)
+	if err != nil {
+		fmt.Println("request error:", err)
+		return
+	}
+	defer resp1.Body.Close()
+}
+
+var SP = make(chan bool, 5)
+var ErrorInfoCache = make(chan map[string]interface{}, 1000) //异常信息集合
+
+//批量保存异常信息
+func SaveErrorInfo() {
+	log.Info("SaveErrorInfo 异常信息...")
+	savearr := make([]map[string]interface{}, 200)
+	indexh := 0
+	for {
+		select {
+		case v := <-ErrorInfoCache:
+			savearr[indexh] = v
+			indexh++
+			if indexh == 200 {
+				SP <- true
+				go func(savearr []map[string]interface{}) {
+					defer func() {
+						<-SP
+					}()
+					MgoB.SaveBulk("bidding_warn", savearr...)
+				}(savearr)
+				savearr = make([]map[string]interface{}, 200)
+				indexh = 0
+			}
+		case <-time.After(1 * time.Minute):
+			if indexh > 0 {
+				SP <- true
+				go func(savearr []map[string]interface{}) {
+					defer func() {
+						<-SP
+					}()
+					MgoB.SaveBulk("bidding_warn", savearr...)
+				}(savearr[:indexh])
+				savearr = make([]map[string]interface{}, 200)
+				indexh = 0
+			}
+		}
+	}
+}