Merge branch 'master' of http://192.168.3.207:8080/data_processing/fieldproject

README.md

@@ -30,4 +30,26 @@
 
 + data_service (服务)
   + 1、标的物相关信息  
-  + 2、相关服务
+  + 2、相关服务
+
+###  proposed_project(拟在建数据项目合并)
+####  拟在建数据处理分为三个阶段
+1. 数据打标签，标签有：业主类型标签、申报方式（项目类型）、项目类别、项目性质、项目阶段，同时会有标签记录表
+2. 数据进行项目合并，
+3. tidb库数据迁移
+
+####  拟在建关联
+1. 建立关联关系，拟建项目与在建项目
+2. 关联数据更新到tidb库
+   + 基本信息表的跟进数量、项目阶段
+   + 跟进记录表的信息
+3. 关联数据增量处理
+   + 拟建项目增量处理
+   + 在建项目增量数据处理
+
+
+### medical_project(医疗领域数据合并)
+
+#### 处理逻辑
++ 
+

field_purchasing/common.toml

@@ -0,0 +1,43 @@
+[serve]
+jyhref = "https://www.jianyu360.cn/article/content/%s.html"
+[db]
+[db.mongo]
+addr = "127.0.0.1:27092"
+dbname = "qfw_data"
+coll = ""
+size = 5
+user = ""
+password = ""
+
+[db.mysql]
+# mysql: [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
+#        root:123456@tcp(localhost:3306)/crocodile?charset=utf8mb4&parseTime=True&loc=Local
+#drivename = "mysql"
+#dsn = "root:=PDT49#80Z!RVv52_z@tcp(192.168.3.217:4000)/medical_fileld_data?charset=utf8mb4&parseTime=True&loc=Local"
+addr = "127.0.0.1:4001"
+dbnameBasic = "global_common_data"
+dbnameMedical = "medical_field_data_dev"
+size = 5
+user = "root"
+password = "Tibi#20211222"
+
+maxidle = 10
+maxconn = 20
+maxquerytime = "10s"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"

field_purchasing/config/conf.go

@@ -0,0 +1,96 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	// Conf crocodile conf
+	Conf *conf
+)
+
+// Init Config
+func Init(conf string) {
+	_, err := toml.DecodeFile(conf, &Conf)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+}
+
+type conf struct {
+	Serve serve
+	DB    Db
+	Log   Log
+}
+
+type serve struct {
+	JyHref string
+}
+
+// Log Config
+type Log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type Db struct {
+	Mongo mgo
+	Mysql mysql
+	Es    es
+}
+
+type mgo struct {
+	Addr     string
+	Dbname   string
+	Coll     string
+	Size     int
+	User     string
+	Password string
+}
+
+type mysql struct {
+	Addr          string
+	DbnameBasic   string
+	DbnameMedical string
+	Size          int
+	User          string
+	Password      string
+
+	Drivename    string
+	Dsn          string
+	MaxIdle      int
+	MaxConn      int
+	MaxQueryTime duration
+}
+
+type es struct {
+	Addr   string
+	Size   int
+	IndexM string
+	TypeM  string
+	IndexS string
+	TypeS  string
+	FieldM map[string]interface{}
+	FieldS map[string]interface{}
+}
+
+type duration struct {
+	time.Duration
+}
+
+// UnmarshalText parse 10s to time.Time
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}

field_purchasing/config/conf_test.go

@@ -0,0 +1,66 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var confs = `# log
+[log]
+logpath = ""
+maxsize = 10
+compress = true
+maxage =  7
+maxbackups = 10
+loglevel  = "info"
+format = "text"
+
+[db]
+[db.mongo]
+addr = "192.168.3.207:27092"
+dbname = "qfw"
+size = 10
+user = ""
+password = ""
+[db.mongo1]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+size = 5
+user = ""
+password = ""
+[db.mysql]
+addr = "192.168.3.217:4000"
+dbnameBasic = "global_common_data"
+dbnameMedical = "medical_fileld_data"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+
+[db.es]
+addr = "http://192.168.3.206:9800"
+size = 5
+indexm = "medical_institution_v1"
+typem = "medical_institution"
+indexs = "supplier_product_v1"
+types = "supplier_product"
+
+[db.es.fieldarrm]
+id = "int"
+company_id = "string"
+mi_name = "string"
+alias = "string"
+area_code = "string"
+level_code = "string"
+mi_type_code = "string"
+business_type = "int"
+sdequipment = "string"
+`
+
+func TestInit(t *testing.T) {
+	testfile := "/tmp/crocodile.toml"
+	ioutil.WriteFile(testfile, []byte(confs), 0644)
+	Init(testfile)
+	t.Logf("%+v", Conf.DB.Mysql.DbnameBasic)
+	os.Remove(testfile)
+}

field_purchasing/go.mod

@@ -0,0 +1,9 @@
+module field_purchasing
+
+go 1.16
+
+require (
+	app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d // indirect
+	app.yhyue.com/data_processing/field_info_tag v1.0.2 // indirect
+	github.com/BurntSushi/toml v1.2.1 // indirect
+)

field_purchasing/go.sum

@@ -0,0 +1,112 @@
+app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
+app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+app.yhyue.com/data_processing/field_info_tag v1.0.2 h1:WpIawVHo4gkIMkvFkLTI2f8/Cxio2vDGqhITFJc1wsI=
+app.yhyue.com/data_processing/field_info_tag v1.0.2/go.mod h1:i9NKOqhWmcDl0Cl5qHBBBx46Da0Qzys7L0J+3my/Hbw=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/antlabs/strsim v0.0.3 h1:J9AHxnybJZHKBoxeup1VZNWt3ST8QD+ieDJsm/nEpRo=
+github.com/antlabs/strsim v0.0.3/go.mod h1:bIcymn+2jtt01korFun0bs8PsYZeQa82aHoYMi7cm30=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/donnie4w/go-logger v0.0.0-20170827050443-4740c51383f4 h1:T9PR91sjTtrA1HmZB4G+M7OLCelch0f6rIEY7Mm1T4U=
+github.com/donnie4w/go-logger v0.0.0-20170827050443-4740c51383f4/go.mod h1:L7S4x0R7vv3xoOhGuyAJyCO2MYzWOpccM4Isn8jIUgY=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-ego/gse v0.70.2 h1:y2UMOHJMtI+0b2GjxTtQfKON5DMmlyX1hOQHTo8UVVs=
+github.com/go-ego/gse v0.70.2/go.mod h1:kesekpZfcFQ/kwd9b27VZHUOH5dQUjaaQUZ4OGt4Hj4=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/vcaesar/cedar v0.20.1 h1:cDOmYWdprO7ZW8cngJrDi8Zivnscj9dA/y8Y+2SB1P0=
+github.com/vcaesar/cedar v0.20.1/go.mod h1:iMDweyuW76RvSrCkQeZeQk4iCbshiPzcCvcGCtpM7iI=
+github.com/vcaesar/tt v0.20.0/go.mod h1:GHPxQYhn+7OgKakRusH7KJ0M5MhywoeLb8Fcffs/Gtg=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.11.0 h1:FZKhBSTydeuffHj9CBjXlR8vQLee1cQyTWYPA6/tqiE=
+go.mongodb.org/mongo-driver v1.11.0/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

field_purchasing/init.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/mysqldb"
+	"field_purchasing/config"
+	"fmt"
+	"os"
+)
+
+var (
+	MongoTool *mongodb.MongodbSim
+	Mysql     *mysqldb.Mysql
+)
+
+// InitLog @Description
+// @Author J 2022/7/26 15:30
+func InitLog() {
+	logcfg := config.Conf.Log
+
+	err := log.InitLog(
+		log.Path(logcfg.LogPath),
+		log.Level(logcfg.LogLevel),
+		log.Compress(logcfg.Compress),
+		log.MaxSize(logcfg.MaxSize),
+		log.MaxBackups(logcfg.MaxBackups),
+		log.MaxAge(logcfg.MaxAge),
+		log.Format(logcfg.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func InitMgo() {
+	MongoTool = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.Mongo.Addr,
+		Size:        config.Conf.DB.Mongo.Size,
+		DbName:      config.Conf.DB.Mongo.Dbname,
+		UserName:    config.Conf.DB.Mongo.User,
+		Password:    config.Conf.DB.Mongo.Password,
+	}
+	MongoTool.InitPool()
+}
+
+func InitMysql() {
+	dbcfg := config.Conf.DB.Mysql
+	Mysql = &mysqldb.Mysql{
+		Address:  dbcfg.Addr,
+		DBName:   dbcfg.DbnameBasic,
+		UserName: dbcfg.User,
+		PassWord: dbcfg.Password,
+	}
+	Mysql.Init()
+
+}
+
+//
+//func InitField() {
+//	info := Mysql.Find("code_area", nil, "", "", -1, -1)
+//	for _, m := range *info {
+//		var key string
+//		for i, v := range []string{"area", "city", "district"} {
+//			if i == 0 && util.ObjToString(m[v]) != "" {
+//				key = util.ObjToString(m[v])
+//			} else if util.ObjToString(m[v]) != "" {
+//				key += "," + util.ObjToString(m[v])
+//			}
+//		}
+//		AreaCode[key] = util.ObjToString(m["code"])
+//	}
+//	log.Info("InitField", zap.Int("AreaCode", len(AreaCode)))
+//}
+//
+//func InitPoCode() {
+//	data_class := Mysql.Find("d_yl_productclass_code", nil, "", "", -1, -1)
+//
+//	for _, v := range *data_class {
+//		name := util.ObjToString(v["name"])
+//		code := util.ObjToString(v["code"])
+//		pcode := util.ObjToString(v["pcode"])
+//		level := util.IntAll(v["level"])
+//		info := map[string]string{}
+//		for key, value := range PclassCode[pcode] {
+//			info[key] = value
+//		}
+//		new_key := "class_" + fmt.Sprintf("%d", level)
+//		info[new_key] = name
+//		PclassCode[code] = info
+//
+//	}
+//
+//}

field_purchasing/main.go

@@ -0,0 +1,5 @@
+package main
+
+func main() {
+
+}

field_purchasing/task.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/field_info_tag/bidding"
+	"field_purchasing/config"
+	"fmt"
+	"sync"
+)
+
+func task() {
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+
+	ch := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+
+	query := sess.DB(config.Conf.DB.Mongo.Dbname).C(config.Conf.DB.Mongo.Coll).Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%500 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			b := bidding.ComparisonBiddingInfo(tmp)
+			util.Debug(b)
+
+			if b {
+				
+			}
+
+			//updatePool <- []map[string]interface{}{
+			//	{"_id": tmp["_id"]},
+			//	{"$set": update},
+			//}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count))
+}
+
+// @Description 医疗领域项目信息
+// @Author J 2022/11/17 11:01
+func taskA(tmp map[string]interface{}) {
+	p := make(map[string]interface{})
+	var plist, plist1 []map[string]interface{}
+	for _, m := range tmp["list"].([]interface{}) {
+		m1 := m.(map[string]interface{})
+		if ml, ok := m1["purchasinglist"].([]map[string]interface{}); ok && len(ml) > 0 {
+			if util.ObjToString(tmp["bidstatus"]) == "招标" {
+				plist = append(plist, ml...)
+			} else {
+				plist1 = append(plist1, ml...)
+			}
+		}
+	}
+
+}

fieldproject_com/common.toml

@@ -1,22 +1,22 @@
-
+[serve]
+jyhref = "https://www.jianyu360.cn/article/content/%s.html"
 [db]
-
 [db.mongo]
-addr = "192.168.3.207:27001"
+addr = "127.0.0.1:27092"
 dbname = "qfw_data"
 size = 5
 user = "root"
 password = "root"
 
 [db.mongo1]
-addr = "192.168.3.207:27092"
+addr = "127.0.0.1:27092"
 dbname = "wjh"
 size = 5
 user = ""
 password = ""
 
 [db.mongo2]
-addr = "192.168.3.207:27092"
+addr = "127.0.0.1:27092"
 dbname = "zhengkun"
 size = 5
 user = ""
@@ -27,19 +27,19 @@ password = ""
 #        root:123456@tcp(localhost:3306)/crocodile?charset=utf8mb4&parseTime=True&loc=Local
 #drivename = "mysql"
 #dsn = "root:=PDT49#80Z!RVv52_z@tcp(192.168.3.217:4000)/medical_fileld_data?charset=utf8mb4&parseTime=True&loc=Local"
-addr = "192.168.3.217:4000"
+addr = "127.0.0.1:4001"
 dbnameBasic = "global_common_data"
-dbnameMedical = "medical_field_data"
+dbnameMedical = "medical_field_data_dev"
 size = 5
 user = "root"
-password = "=PDT49#80Z!RVv52_z"
+password = "Tibi#20211222"
 
 maxidle = 10
 maxconn = 20
 maxquerytime = "10s"
 
 [db.es]
-addr = "http://192.168.3.206:9800"
+addr = "http://127.0.0.1:9800"
 size = 5
 indexm = "medical_institution_v1"
 typem = "medical_institution"

fieldproject_com/config/conf.go

@@ -23,8 +23,13 @@ func Init(conf string) {
 }
 
 type conf struct {
-	DB  Db
-	Log Log
+	Serve serve
+	DB    Db
+	Log   Log
+}
+
+type serve struct {
+	JyHref string
 }
 
 // Log Config

fieldproject_com/go.mod

@@ -3,14 +3,13 @@ module fieldproject_common
 go 1.16
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f
+	app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d
 	github.com/BurntSushi/toml v1.2.0
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/olivere/elastic v6.2.37+incompatible // indirect
+	github.com/satori/go.uuid v1.2.0
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.5 // indirect
+	go.mongodb.org/mongo-driver v1.10.1
 	go.uber.org/zap v1.22.0
-	gopkg.in/olivere/elastic.v1 v1.0.1
-	gopkg.in/olivere/elastic.v2 v2.0.61 // indirect
+	gopkg.in/olivere/elastic.v2 v2.0.61
 )

fieldproject_com/go.sum

@@ -1,5 +1,5 @@
-app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f h1:5fUbVRwPM3oBsZgvG76Bia3I4SdwdBB6PvJ6B28Qkyc=
-app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
+app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
@@ -19,13 +19,12 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
-github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -33,18 +32,16 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
-github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
-github.com/olivere/elastic v6.2.37+incompatible h1:UfSGJem5czY+x/LqxgeCBgjDn6St+z8OnsCuxwD3L0U=
-github.com/olivere/elastic v6.2.37+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -120,8 +117,6 @@ gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/olivere/elastic.v1 v1.0.1 h1:ZoJwTKCI0jJdVptoGB0QEFt/4bDUs6A5Pjrmn/Zb+5g=
-gopkg.in/olivere/elastic.v1 v1.0.1/go.mod h1:sMIrW2Y2hS8bEAqdTvdcrNN/KV21XXOfjdi4tHxwVnI=
 gopkg.in/olivere/elastic.v2 v2.0.61 h1:7cpl3MW8ysa4GYFBXklpo5mspe4NK0rpZTdyZ+QcD4U=
 gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

fieldproject_com/init.go

@@ -1,15 +1,49 @@
 package main
 
 import (
+	util "app.yhyue.com/data_processing/common_utils"
 	"app.yhyue.com/data_processing/common_utils/elastic"
 	"app.yhyue.com/data_processing/common_utils/log"
 	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"app.yhyue.com/data_processing/common_utils/mysqldb"
 	"fieldproject_common/config"
 	"fmt"
+	"go.uber.org/zap"
 	"os"
 )
 
+var (
+	RecordField = []string{"name", "name_id", "type", "createtime"}
+	// DealerField 经销商字段
+	DealerField = []string{"name_id", "company_id", "dealer_name", "area_code", "city_code", "district_code",
+		"business_model", "capital", "employee_no", "insurance_amount", "company_address", "website", "capital_code",
+		"employee_no_code", "insurance_amount_code", "exists_id", "createtime"}
+	EntField = []string{"company_id", "company_name", "company_code", "credit_no", "org_code", "tax_code", "establish_date",
+		"legal_person", "legal_person_caption", "company_status", "company_type", "authority", "issue_date", "operation_startdate",
+		"operation_enddate", "capital", "company_address", "business_scope", "cancel_date", "cancel_reason", "revoke_date",
+		"revoke_reason", "legal_person_type", "real_capital", "en_name", "list_code", "area_code", "employee_no", "company_phone",
+		"insurance_amount", "company_email", "website", "sourcetype", "createtime", "updatetime", "employees"}
+	// RegField 许可备案
+	RegField = []string{"dealer_id", "company_name", "regnum", "type", "address", "scope", "business_type", "approve_depart",
+		"validity_date", "approve_date", "type_address", "website_name", "website", "platform_name", "client_name",
+		"responsible_person", "legal_person", "residence", "createtime"}
+	ProductField = []string{"dealer_id", "company_id", "company_name", "regno", "company_address", "make_adress", "make_country",
+		"agent_name", "agent_address", "brand", "model", "main_part", "product_descrip", "use_scope", "storage_conditions",
+		"remark", "approve_depart", "approve_date", "validity_date", "changes", "regist_type", "source_channel", "manage_type",
+		"product_name", "medical_equipment_class1", "medical_equipment_class2", "medical_equipment_class3", "sdproduct_name",
+		"sdequipment_code", "isvalid", "createtime"}
+	ProjectField = []string{"projectid", "infoid", "jyhref", "itemname", "brandname", "specs", "model", "unitname", "number", "unitprice",
+		"totalprice", "medical_equipment_class1", "medical_equipment_class2", "medical_equipment_class3", "sdproduct_name",
+		"sdequipment_code", "area_code", "city_code", "district_code", "budget", "title", "projectname", "projectcode",
+		"purchasing", "buyer_id", "agency_id", "buyer", "mi_level_code", "mi_area_code", "agency", "bidstatus",
+		"bidtype", "bidopentime", "zbtime", "jgtime", "updatetime", "createtime", "month_tags", "year_tags", "quarter_tags"}
+	WinerField = []string{"projectid", "infoid", "winner", "winner_id", "contact_name", "contact_tel", "winner_area_code",
+		"winner_city_code", "bidamount", "jgtime", "is_winner", "createtime", "updatetime"}
+	AreaCode   = make(map[string]string, 5000)
+	LvCode     = make(map[string]string, 10000000)
+	PclassCode = make(map[string]map[string]string, 10000)
+)
+
 // InitLog @Description
 // @Author J 2022/7/26 15:30
 func InitLog() {
@@ -86,3 +120,49 @@ func InitMysql() {
 	MysqlM.Init()
 
 }
+
+func InitField() {
+	info := MysqlB.Find("code_area", nil, "", "", -1, -1)
+	for _, m := range *info {
+		var key string
+		for i, v := range []string{"area", "city", "district"} {
+			if i == 0 && util.ObjToString(m[v]) != "" {
+				key = util.ObjToString(m[v])
+			} else if util.ObjToString(m[v]) != "" {
+				key += "," + util.ObjToString(m[v])
+			}
+		}
+		AreaCode[key] = util.ObjToString(m["code"])
+	}
+	log.Info("InitField", zap.Int("AreaCode", len(AreaCode)))
+}
+
+func InitLvCode() {
+	info := MysqlM.Find("dwd_f_yl_organ_baseinfo", nil, "name_id, level_code", "", -1, -1)
+	for _, m := range *info {
+		name_id := util.ObjToString(m["name_id"])
+		code := util.ObjToString(m["level_code"])
+		LvCode[name_id] = util.ObjToString(code)
+	}
+	log.Info("InitLvCode", zap.Int("LvCode", len(LvCode)))
+}
+
+func InitPoCode() {
+	data_class := MysqlM.Find("d_yl_productclass_code", nil, "", "", -1, -1)
+
+	for _, v := range *data_class {
+		name := util.ObjToString(v["name"])
+		code := util.ObjToString(v["code"])
+		pcode := util.ObjToString(v["pcode"])
+		level := util.IntAll(v["level"])
+		info := map[string]string{}
+		for key, value := range PclassCode[pcode] {
+			info[key] = value
+		}
+		new_key := "class_" + fmt.Sprintf("%d", level)
+		info[new_key] = name
+		PclassCode[code] = info
+
+	}
+
+}

fieldproject_com/main.go

@@ -9,6 +9,7 @@ import (
 	"fieldproject_common/config"
 	"fmt"
 	"github.com/spf13/cobra"
+	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
 	"sync"
 	"time"
@@ -23,10 +24,19 @@ var (
 	saveSize     int
 	savePool     chan map[string]interface{}
 	saveSp       chan bool
+	updatePool   chan []map[string]interface{}
+	updateSp     chan bool
 	EsSaveCache  chan map[string]interface{}
 	SP           chan bool
 	updateEsPool chan []map[string]interface{}
 	updateEsSp   chan bool
+
+	saveBasePool  chan map[string]interface{}
+	saveBaseSp    chan bool
+	saveBasePool1 chan map[string]interface{}
+	saveBaseSp1   chan bool
+	saveRcPool    chan map[string]interface{}
+	saveRcSp      chan bool
 )
 
 func init() {
@@ -40,15 +50,26 @@ func init() {
 	saveSize = 200
 	savePool = make(chan map[string]interface{}, 5000)
 	saveSp = make(chan bool, 2)
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 2)
 	EsSaveCache = make(chan map[string]interface{}, 1000)
 	SP = make(chan bool, 5)
 	updateEsPool = make(chan []map[string]interface{}, 5000)
 	updateEsSp = make(chan bool, 1)
 
+	saveBasePool = make(chan map[string]interface{}, 5000)
+	saveBaseSp = make(chan bool, 1)
+	saveBasePool1 = make(chan map[string]interface{}, 5000)
+	saveBaseSp1 = make(chan bool, 1)
+	saveRcPool = make(chan map[string]interface{}, 5000)
+	saveRcSp = make(chan bool, 1)
+
+	InitField()
 	log.Info("init success")
 }
 
 func main() {
+	//go UpdateMethod()
 	//task()
 
 	//taskBiddingData()
@@ -57,8 +78,14 @@ func main() {
 
 	rootCmd := &cobra.Command{Use: "my cmd"}
 	rootCmd.AddCommand(institution())
-	rootCmd.AddCommand(product())
 	rootCmd.AddCommand(bidding())
+	rootCmd.AddCommand(dealer()) // 经销商
+	rootCmd.AddCommand(dealerEs())
+	rootCmd.AddCommand(ent())      // 法人
+	rootCmd.AddCommand(register()) // 许可备案
+	rootCmd.AddCommand(product())  // 产品信息
+	rootCmd.AddCommand(project())  // 标的物信息宽表、中标信息
+
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println("rootCmd.Execute failed", err.Error())
 	}
@@ -68,18 +95,18 @@ func main() {
 }
 
 func task() {
-	sess := MongoTool.GetMgoConn()
-	defer MongoTool.DestoryMongoConn(sess)
+	sess := MongoTool2.GetMgoConn()
+	defer MongoTool2.DestoryMongoConn(sess)
 
 	ch := make(chan bool, 2)
 	wg := &sync.WaitGroup{}
 
-	log.Info(fmt.Sprintf("%d", MongoTool.Count("qyxy_0824", nil)))
-	field := map[string]interface{}{"use_flag": 0, "province_short": 0, "create_time": 0, "update_time": 0}
-	query := sess.DB(config.Conf.DB.Mongo.Dbname).C("qyxy_0824").Find(nil).Select(field).Iter()
+	//log.Info(fmt.Sprintf("%d", MongoTool.Count("qyxy_0824", nil)))
+	field := map[string]interface{}{"company_name": 1, "company_id": 1}
+	query := sess.DB(config.Conf.DB.Mongo2.Dbname).C("zktest_mysql_company_info").Find(nil).Select(field).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
-		if count%2000 == 0 {
+		if count%20000 == 0 {
 			log.Info(fmt.Sprintf("current --- %d", count))
 		}
 
@@ -90,35 +117,23 @@ func task() {
 				<-ch
 				wg.Done()
 			}()
-			delete(tmp, "_id")
-			m := make(map[string]interface{})
-			if util.ObjToString(tmp["company_district"]) != "" {
-				m["district"] = tmp["company_district"]
-			} else if util.ObjToString(tmp["city"]) != "" {
-				m["city"] = tmp["company_city"]
+			update := make(map[string]interface{})
+			name := util.ObjToString(tmp["company_name"])
+			info := MysqlB.FindOne("dws_f_ent_baseinfo", bson.M{"name": name}, "name_id", "")
+			if info != nil && len(*info) > 0 {
+				update["name_id"] = (*info)["name_id"]
 			} else {
-				m["area"] = tmp["company_area"]
-			}
-			if len(m) > 0 {
-				info := MysqlB.FindOne("code_area", m, "", "")
+				info = MysqlM.FindOne("dws_d_name_id_record", bson.M{"name": name}, "name_id", "")
 				if info != nil && len(*info) > 0 {
-					tmp["area_code"] = (*info)["code"]
-				} else {
-					tmp["area_code"] = "000000"
+					update["name_id"] = (*info)["name_id"]
 				}
-			} else {
-				tmp["area_code"] = "000000"
-			}
-			delete(tmp, "company_area")
-			delete(tmp, "company_city")
-			delete(tmp, "company_district")
-			tmp["comeintime"] = time.Now()
-			tmp["updatetime"] = time.Now()
-			tmp["sourcetype"] = 1
-			MysqlB.Insert("company_business_model", map[string]interface{}{"company_id": tmp["company_id"],
-				"business_model": util.IntAll(tmp["business_type"]), "company_field_code": "0101", "comeintime": time.Now()})
-			delete(tmp, "business_type")
-			MysqlB.Insert("company_baseinfo", tmp)
+			}
+			if len(update) > 0 {
+				updatePool <- []map[string]interface{}{
+					{"_id": tmp["_id"]},
+					{"$set": update},
+				}
+			}
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
@@ -230,3 +245,143 @@ func SaveMethod() {
 		}
 	}
 }
+
+func SaveFunc(table string, arr []string) {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveBasePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveBaseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveBaseSp
+					}()
+					MysqlM.InsertBulk(table, arr, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveBaseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveBaseSp
+					}()
+					MysqlM.InsertBulk(table, arr, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveFunc1(table string, arr []string) {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveBasePool1:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveBaseSp1 <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveBaseSp1
+					}()
+					MysqlM.InsertBulk(table, arr, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveBaseSp1 <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveBaseSp1
+					}()
+					MysqlM.InsertBulk(table, arr, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveFuncRc() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveRcPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveRcSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveRcSp
+					}()
+					MysqlM.InsertBulk("dws_d_name_id_record", RecordField, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveRcSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveRcSp
+					}()
+					MysqlM.InsertBulk("dws_d_name_id_record", RecordField, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func UpdateMethod() {
+	arru := make([][]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoTool2.UpdateBulk("zktest_mysql_company_info", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoTool2.UpdateBulk("zktest_mysql_company_info", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

fieldproject_com/task1.go

@@ -0,0 +1,392 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"encoding/json"
+	"fieldproject_common/config"
+	"fmt"
+	es "gopkg.in/olivere/elastic.v2"
+	"strconv"
+	"sync"
+	"time"
+)
+
+var province_map = map[string]string{
+	"BJ": "北京", "TJ": "天津", "SH": "上海", "CQ": "重庆", "HB": "河北", "SX": "山西", "NMG": "内蒙古", "LN": "辽宁", "JL": "吉林",
+	"HLJ": "黑龙江", "JS": "江苏", "ZJ": "浙江", "AH": "安徽", "FJ": "福建", "JX": "江西", "SD": "山东", "HEN": "河南", "HUB": "湖北",
+	"HUN": "湖南", "GD": "广东", "GX": "广西", "HAIN": "海南", "SC": "四川", "GZ": "贵州", "YN": "云南", "XZ": "西藏", "SAX": "陕西",
+	"GS": "甘肃", "QH": "青海", "NX": "宁夏", "XJ": "新疆",
+}
+
+// @Description 标讯数据
+// @Author J 2022/9/7 11:42
+func taskBiddingData1() {
+	go SaveMethod()
+
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+
+	ch := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+
+	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("5a8d7f4840d2d9bbe8962002")}
+	query := sess.DB(config.Conf.DB.Mongo.Dbname).C("bidding").Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%20000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			if b := util.ObjToString(tmp["bid_field"]); b != "" {
+				taskA(tmp)
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count))
+}
+
+// @Description 企业数据
+// @Author J 2022/8/23 09:08
+func taskCompanyData() {
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+
+	ch := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+
+	//log.Info(fmt.Sprintf("%d", MongoTool2.Count("zktest_mysql_company_info", nil)))
+	var p1 []map[string]interface{}
+	p1 = append(p1, map[string]interface{}{"$group": map[string]interface{}{"_id": "$company_id"}})
+	query := sess.DB(config.Conf.DB.Mongo.Dbname).C("bidding_p_list_0907").Pipe(p1).Iter()
+	//q := map[string]interface{}{"_id": mongodb.StringTOBsonId("61efb24b70f4a1409599badb")}
+	//query := sess.DB(config.Conf.DB.Mongo2.Dbname).C("zktest_mysql_company_info").Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%5000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			//taskC(tmp)
+			taskB(util.ObjToString(tmp["_id"]))
+			//info, _ := MongoTool.FindOne("qyxy_std", map[string]interface{}{"_id": util.ObjToString(tmp["company_id"])})
+			//if len(*info) > 0 {
+			//	savePool <- *info
+			//}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count))
+}
+
+// @Description 标讯数据企业与产品
+// @Author J 2022/8/24 14:37
+func taskA(tmp map[string]interface{}) {
+	if tmp["entidlist"] != nil {
+		saveMap := make(map[string]interface{})
+		for _, eid := range tmp["entidlist"].([]interface{}) {
+			if util.ObjToString(eid) != "" && util.ObjToString(eid) != "-" {
+				saveMap["company_id"] = eid
+				if tmp["purchasinglist"] != nil {
+					for _, p1 := range tmp["purchasinglist"].([]interface{}) {
+						p2 := p1.(map[string]interface{})
+						if p2["itemname"] != nil {
+							saveMap["itemname"] = p2["itemname"]
+							if p2["brandname"] != nil {
+								saveMap["brand"] = p2["brandname"]
+							}
+							if p2["model"] != nil {
+								saveMap["model"] = p2["model"]
+							}
+							MongoTool.Save("bidding_p_list_0907", saveMap)
+							//savePool <- saveMap
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+var company_field = []string{"company_id", "company_name", "company_code", "credit_no", "org_code", "tax_code", "establish_date", "legal_person", "legal_person_caption",
+	"company_status", "company_type", "authority", "issue_date", "operation_startdate", "operation_enddate", "capital", "company_address", "business_scope", "cancel_date",
+	"cancel_reason", "revoke_date", "revoke_reason", "legal_person_type", "real_capital", "en_name", "list_code", "area_code", "employee_no", "company_phone", "company_email",
+	"website", "sourcetype"}
+
+// @Description 医疗企业
+// @Author J 2022/8/24 14:37
+func taskB(tid string) {
+	saveM := make(map[string]interface{})
+	info, _ := MongoTool1.FindOneByField("company_base", map[string]interface{}{"company_id": tid}, "")
+	if len(*info) == 0 {
+		info, _ = MongoTool1.FindOneByField("special_enterprise", map[string]interface{}{"company_id": tid}, "")
+	}
+	if len(*info) > 0 {
+		for _, v := range company_field {
+			if (*info)[v] != nil {
+				saveM[v] = (*info)[v]
+			}
+			if v == "area_code" {
+				std, _ := MongoTool.FindOneByField("qyxy_std", map[string]interface{}{"_id": tid}, map[string]interface{}{"company_area": 1, "company_city": 1, "company_district": 1, "website_url": 1, "capital": 1})
+				if len(*std) > 0 {
+					m := make(map[string]interface{})
+					for k, v := range map[string]string{"company_district": "district", "company_city": "city", "company_area": "area"} {
+						if v1 := util.ObjToString((*std)[k]); v1 != "" {
+							m = map[string]interface{}{v: v1}
+							info := MysqlB.FindOne("code_area", m, "", "")
+							if info != nil && len(*info) > 0 {
+								saveM["area_code"] = (*info)["code"]
+								break
+							}
+						}
+					}
+					if saveM["area_code"] == nil {
+						saveM["area_code"] = "000000"
+					}
+
+					if (*std)["website_url"] != nil && len(util.ObjToString((*std)["website_url"])) <= 255 {
+						saveM["website"] = (*std)["website_url"]
+					}
+					if (*std)["company_phone"] != nil {
+						saveM["company_phone"] = (*std)["company_phone"]
+					}
+					if (*std)["company_email"] != nil {
+						saveM["company_email"] = (*std)["company_email"]
+					}
+					if (*std)["capital"] != nil {
+						saveM["capital"] = util.ObjToString((*std)["capital"])
+					}
+				}
+			} else if v == "employee_no" {
+				rep, _ := MongoTool1.Find("annual_report_base", map[string]interface{}{"company_id": tid}, map[string]interface{}{"_id": -1}, "", false, -1, -1)
+				if len(*rep) > 0 && util.ObjToString((*rep)[0]["employee_no"]) != "" {
+					i, err := strconv.Atoi(util.ObjToString((*rep)[0]["employee_no"]))
+					if err != nil {
+						saveM[v] = i
+					}
+				}
+			}
+			//else if v == "website" {
+			//	info, _ := MongoTool.Find("annual_report_website", map[string]interface{}{"company_id": tid}, map[string]interface{}{"_id": -1}, "", false, -1, -1)
+			//	if len(*info) > 0 && util.ObjToString((*info)[0]["website_url"]) != "" {
+			//		saveM[v] = util.ObjToString((*info)[0]["website_url"])
+			//	}
+			//}
+		}
+		saveM["comeintime"] = time.Now()
+		saveM["updatetime"] = time.Now()
+		saveM["sourcetype"] = 3
+		MysqlB.Insert("company_baseinfo", saveM)
+		//savePool <- saveM
+	} else {
+		util.Debug("company_id err", tid)
+	}
+}
+
+func taskB_1(tid string) {
+	saveM := make(map[string]interface{})
+	info, _ := MongoTool1.FindOneByField("company_base", map[string]interface{}{"company_id": tid}, "")
+	if len(*info) == 0 {
+		info, _ = MongoTool1.FindOneByField("special_enterprise", map[string]interface{}{"company_id": tid}, "")
+	}
+	if len(*info) > 0 {
+		for _, v := range company_field {
+			if (*info)[v] != nil {
+				saveM[v] = (*info)[v]
+			}
+			if v == "area_code" {
+				if a := util.ObjToString((*info)["province_short"]); a != "" {
+					m := map[string]interface{}{"area": province_map[a]}
+					info := MysqlB.FindOne("code_area", m, "", "")
+					if info != nil && len(*info) > 0 {
+						saveM["area_code"] = (*info)["code"]
+					}
+
+				}
+			} else if (*info)["capital"] != nil {
+				text := util.ObjToString((*info)["capital"])
+				capital := ObjToMoney(text)
+				capital = capital / 10000
+				if capital != 0 {
+					saveM["capital"] = fmt.Sprint(capital)
+				}
+			}
+		}
+		saveM["comeintime"] = time.Now()
+		saveM["updatetime"] = time.Now()
+		saveM["sourcetype"] = 1
+		MysqlB.Insert("company_baseinfo", saveM)
+		//savePool <- saveM
+	} else {
+		util.Debug("company_id err", tid)
+	}
+}
+
+func taskC(tmp map[string]interface{}) {
+	tid := util.ObjToString(tmp["company_id"])
+	saveM := make(map[string]interface{})
+	for _, v := range company_field {
+		if tmp[v] != nil {
+			saveM[v] = tmp[v]
+		}
+		if v == "area_code" {
+			std, _ := MongoTool.FindOneByField("qyxy_std", map[string]interface{}{"_id": tid}, map[string]interface{}{"company_area": 1, "company_city": 1, "company_district": 1, "website_url": 1, "company_phone": 1, "company_email": 1, "capital": 1})
+			if len(*std) > 0 {
+				m := make(map[string]interface{})
+				for k, v := range map[string]string{"company_district": "district", "company_city": "city", "company_area": "area"} {
+					if v1 := util.ObjToString((*std)[k]); v1 != "" {
+						m = map[string]interface{}{v: v1}
+						info := MysqlB.FindOne("code_area", m, "", "")
+						if info != nil && len(*info) > 0 {
+							saveM["area_code"] = (*info)["code"]
+							break
+						}
+					}
+				}
+				if saveM["area_code"] == nil {
+					saveM["area_code"] = "000000"
+				}
+				if (*std)["website_url"] != nil && len(util.ObjToString((*std)["website_url"])) <= 255 {
+					saveM["website"] = (*std)["website_url"]
+				}
+				if (*std)["company_phone"] != nil {
+					saveM["company_phone"] = (*std)["company_phone"]
+				}
+				if (*std)["company_email"] != nil {
+					saveM["company_email"] = (*std)["company_email"]
+				}
+				if (*std)["capital"] != nil {
+					saveM["capital"] = fmt.Sprint((*std)["capital"])
+				}
+			}
+		} else if v == "employee_no" {
+			rep, _ := MongoTool1.Find("annual_report_base", map[string]interface{}{"company_id": tid}, map[string]interface{}{"_id": -1}, "", false, -1, -1)
+			if len(*rep) > 0 && util.ObjToString((*rep)[0]["employee_no"]) != "" {
+				i, err := strconv.Atoi(util.ObjToString((*rep)[0]["employee_no"]))
+				if err != nil {
+					saveM[v] = i
+				}
+			}
+		}
+	}
+	saveM["comeintime"] = time.Now()
+	saveM["updatetime"] = time.Now()
+	saveM["sourcetype"] = 2
+	//MysqlB.Insert("company_business_model", map[string]interface{}{"company_id": tmp["company_id"],
+	//	"business_model": util.IntAll(tmp["business_type"]), "company_field_code": "0101", "comeintime": time.Now()})
+	MysqlB.Insert("company_baseinfo", saveM)
+}
+
+func taskBiddingData() {
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+
+	wg := &sync.WaitGroup{}
+	//lock := &sync.Mutex{}
+
+	esquery := `{"query":{"bool":{"must":[],"must_not":[{"constant_score":{"filter":{"missing":{"field":"bid_field"}}}}],"should":[{"match_all":{}}]}},"from":0,"size":10,"sort":[],"facets":{}}`
+
+	q := es.NewRawStringQuery(esquery)
+	util.Debug(Es.Count("bidding", "bidding", q))
+	countDocs := 0
+	res, err := client.Scroll().Index("bidding").Type("bidding").Query(q).Size(200).Do() //查询一条获取游标
+	if err == nil {
+		taskInfoA(res, wg, &countDocs)
+		scrollId := res.ScrollId
+		for {
+			searchResult, err := client.Scroll("1m").ScrollId(scrollId).Size(200).Do() //查询
+			if err != nil {
+				util.Debug("Es Search Data Error：", err)
+				break
+			}
+			taskInfoA(searchResult, wg, &countDocs)
+			scrollId = searchResult.ScrollId
+		}
+		wg.Wait()
+		util.Debug("over---", countDocs)
+		_, _ = client.ClearScroll().ScrollId(scrollId).Do() //清理游标
+	} else {
+		util.Debug(err)
+	}
+
+	c := make(chan bool, 1)
+	<-c
+}
+
+func taskInfoA(searchResult *es.SearchResult, wg *sync.WaitGroup, countDocs *int) {
+	for _, hit := range searchResult.Hits.Hits {
+		//开始处理数据
+		wg.Add(1)
+		ChEs <- true
+		go func(tmpHit *es.SearchHit) {
+			defer func() {
+				<-ChEs
+				wg.Done()
+			}()
+			tmp := make(map[string]interface{})
+			if json.Unmarshal(*tmpHit.Source, &tmp) == nil {
+				id := util.ObjToString(tmp["_id"])
+				if tmp["entidlist"] != nil && tmp["purchasinglist"] != nil {
+					for _, eid := range util.ObjArrToStringArr(tmp["entidlist"].([]interface{})) {
+						if util.ObjToString(eid) != "" && util.ObjToString(eid) != "-" {
+							saveMap := make(map[string]interface{})
+							saveMap["infoid"] = id
+							saveMap["company_id"] = eid
+							for _, p1 := range tmp["purchasinglist"].([]interface{}) {
+								p2 := p1.(map[string]interface{})
+								if p2["itemname"] != nil {
+									b := method3(id, util.ObjToString(p2["itemname"]))
+									if b {
+										saveMap["itemname"] = p2["itemname"]
+										if p2["brandname"] != nil {
+											saveMap["brand"] = p2["brandname"]
+										}
+										if p2["model"] != nil {
+											saveMap["model"] = p2["model"]
+										}
+										MongoTool.Save("bidding_p_list_0907", saveMap)
+									}
+								}
+							}
+						}
+					}
+				}
+
+			}
+		}(hit)
+		*countDocs += 1
+
+		if *countDocs%20000 == 0 {
+			util.Debug("Current：", *countDocs)
+		}
+	}
+}
+
+func method3(id, itemname string) bool {
+	q := map[string]interface{}{"infoid": id, "itemname": itemname}
+	info := MysqlB.FindOne("bid_purchasinginfo", q, "id", "")
+	if info != nil && len(*info) > 0 {
+		q1 := map[string]interface{}{"bid_purchasing_id": (*info)["id"]}
+		info1 := MysqlB.FindOne("bid_purchasing_field_record", q1, "id", "")
+		if info1 != nil && len(*info1) > 0 {
+			return true
+		}
+	}
+	return false
+}

fieldproject_com/util.go

@@ -7,18 +7,18 @@ import (
 )
 
 var (
-	regNumFloat, _  	= regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
-	regStrUnit, _   	= regexp.Compile(`[元|万|亿]`)
-	contentUnit, _ 		= regexp.Compile(`(万元|单位/万)`)
-	numCapitals, _		= regexp.Compile(`([〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]{4,40})`)
-	regStrChar      	= `[〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
-	moneyRegChar, _ 	= regexp.Compile(regStrChar)
-	regQianw, _    		= regexp.Compile(`\d{1,2}千万`)
+	regNumFloat, _  = regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
+	regStrUnit, _   = regexp.Compile(`[元|万|亿]`)
+	contentUnit, _  = regexp.Compile(`(万元|单位/万)`)
+	numCapitals, _  = regexp.Compile(`([〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]{4,40})`)
+	regStrChar      = `[〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
+	moneyRegChar, _ = regexp.Compile(regStrChar)
+	regQianw, _     = regexp.Compile(`\d{1,2}千万`)
 
-	cutAllSpace, _ 		= regexp.Compile(`\s*`)
-	spaces        		= []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n"}
+	cutAllSpace, _ = regexp.Compile(`\s*`)
+	spaces         = []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n"}
 
-	moneyChar      = map[string]interface{}{ //"〇": "0", "零": "0",
+	moneyChar = map[string]interface{}{ //"〇": "0", "零": "0",
 		"一": float64(1), "壹": float64(1), "二": float64(2), "贰": float64(2), "三": float64(3), "叁": float64(3), "四": float64(4), "肆": float64(4), "五": float64(5), "伍": float64(5),
 		"六": float64(6), "陆": float64(6), "七": float64(7), "柒": float64(7), "八": float64(8), "捌": float64(8), "九": float64(9), "玖": float64(9), "十": float64(10), "拾": float64(10),
 		"百": float64(100), "佰": float64(100), "千": float64(1000), "仟": float64(1000), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000),
@@ -270,3 +270,53 @@ func replaceSymbol(con string, rep []string) string {
 	return con
 }
 
+var Bidstatus = map[string]int{
+	"预告": 0,
+	"拟建": 1,
+	"招标": 2,
+	"中标": 3,
+	"成交": 4,
+	"废标": 5,
+	"流标": 6,
+	"合同": 7,
+	"其它": 8,
+}
+
+var Bidtype = map[string]int{
+	"招标": 0,
+	"邀标": 1,
+	"单一": 2,
+	"竞价": 3,
+	"竞谈": 4,
+	"询价": 5,
+}
+
+var Month = map[string]int{
+	"January":   1,
+	"February":  2,
+	"March":     3,
+	"April":     4,
+	"May":       5,
+	"June":      6,
+	"July":      7,
+	"August":    8,
+	"September": 9,
+	"October":   10,
+	"November":  11,
+	"December":  12,
+}
+
+var Quarter = map[string]int{
+	"January":   1,
+	"February":  1,
+	"March":     1,
+	"April":     2,
+	"May":       2,
+	"June":      2,
+	"July":      3,
+	"August":    3,
+	"September": 3,
+	"October":   4,
+	"November":  4,
+	"December":  4,
+}

fieldproject_inc_data/common.toml

@@ -1,3 +1,9 @@
+[udp]
+locport = ":1582"
+[udp.next]
+addr = "127.0.0.1"
+port = 1581
+stype = "dispose"
 
 [db]
 
@@ -8,13 +14,6 @@ size = 5
 user = "root"
 password = "root"
 
-[db.mongo1]
-addr = "192.168.3.207:27092"
-dbname = "wjh"
-size = 5
-user = ""
-password = ""
-
 [db.es]
 addr = "http://192.168.3.206:9800"
 size = 5
@@ -23,6 +22,11 @@ typem = "medical_institution"
 indexs = "supplier_product_v1"
 types = "supplier_product"
 
+[mail]
+send = false
+to = "wangjianghan@topnet.net.cn"
+api = "http://172.17.145.179:19281/_send/_mail"
+
 # 日志
 [log]
 # 日志路径，为空将输出控制台

fieldproject_inc_data/config/conf.go

@@ -23,12 +23,36 @@ func Init(conf string) {
 }
 
 type conf struct {
-	DB  Db
-	Log Log
+	Serve serve
+	DB    db
+	Udp   udp
+	Mail  mail
+	Log   log
+}
+
+type serve struct {
+	GrpcAddr string
+}
+
+type udp struct {
+	LocPort string
+	Next    udpNext
+}
+
+type udpNext struct {
+	Addr  string
+	Port  int
+	Stype string
+}
+
+type mail struct {
+	Send bool
+	To   string
+	Api  string
 }
 
 // Log Config
-type Log struct {
+type log struct {
 	LogPath    string
 	MaxSize    int
 	Compress   bool
@@ -38,11 +62,9 @@ type Log struct {
 	Format     string
 }
 
-type Db struct {
+type db struct {
 	Mongo  mgo
 	Mongo1 mgo
-	Mongo2 mgo
-	Mysql  mysql
 	Es     es
 }
 
@@ -54,30 +76,11 @@ type mgo struct {
 	Password string
 }
 
-type mysql struct {
-	Addr          string
-	DbnameBasic   string
-	DbnameMedical string
-	Size          int
-	User          string
-	Password      string
-
-	Drivename    string
-	Dsn          string
-	MaxIdle      int
-	MaxConn      int
-	MaxQueryTime duration
-}
-
 type es struct {
 	Addr   string
 	Size   int
-	IndexM string
-	TypeM  string
 	IndexS string
 	TypeS  string
-	FieldM map[string]interface{}
-	FieldS map[string]interface{}
 }
 
 type duration struct {

fieldproject_inc_data/field_inc_data

@@ -0,0 +1 @@
+ELF

fieldproject_inc_data/go.mod

@@ -5,5 +5,8 @@ go 1.16
 require (
 	app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f
 	github.com/BurntSushi/toml v1.2.0
+	github.com/robfig/cron v1.2.0 // indirect
 	github.com/spf13/cobra v1.5.0
+	go.mongodb.org/mongo-driver v1.10.2 // indirect
+	go.uber.org/zap v1.23.0 // indirect
 )

fieldproject_inc_data/go.sum

@@ -2,7 +2,9 @@ app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f h1
 app.yhyue.com/data_processing/common_utils v0.0.0-20220830011833-76d58ef43f4f/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -10,42 +12,64 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
 github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
+github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
 github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.10.2 h1:4Wk3cnqOrQCn0P92L3/mmurMxzdvWWs5J9jinAVKD+k=
+go.mongodb.org/mongo-driver v1.10.2/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
@@ -54,8 +78,11 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
 go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
+go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -64,8 +91,10 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -78,6 +107,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -89,9 +119,11 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/olivere/elastic.v2 v2.0.61 h1:7cpl3MW8ysa4GYFBXklpo5mspe4NK0rpZTdyZ+QcD4U=
 gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

fieldproject_inc_data/init.go

@@ -4,20 +4,21 @@ import (
 	"app.yhyue.com/data_processing/common_utils/elastic"
 	"app.yhyue.com/data_processing/common_utils/log"
 	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/udp"
 	"fieldproject_inc_data/config"
 	"fmt"
 	"os"
+	"sync"
 )
 
 var (
-	MongoTool, MongoTool1, MongoTool2 *mongodb.MongodbSim
-	Es                                *elastic.Elastic
+	MongoTool *mongodb.MongodbSim
+	Es        *elastic.Elastic
+
+	UdpClient  udp.UdpClient
+	UdpTaskMap = &sync.Map{}
 
 	saveSize     int
-	savePool     chan map[string]interface{}
-	saveSp       chan bool
-	EsSaveCache  chan map[string]interface{}
-	SP           chan bool
 	updateEsPool chan []map[string]interface{}
 	updateEsSp   chan bool
 )
@@ -25,14 +26,8 @@ var (
 func init() {
 	config.Init("./common.toml")
 	InitLog()
-	InitMgo()
-	InitEs()
 
 	saveSize = 200
-	savePool = make(chan map[string]interface{}, 5000)
-	saveSp = make(chan bool, 5)
-	EsSaveCache = make(chan map[string]interface{}, 1000)
-	SP = make(chan bool, 5)
 	updateEsPool = make(chan []map[string]interface{}, 5000)
 	updateEsSp = make(chan bool, 1)
 
@@ -66,24 +61,6 @@ func InitMgo() {
 		Password:    config.Conf.DB.Mongo.Password,
 	}
 	MongoTool.InitPool()
-
-	MongoTool1 = &mongodb.MongodbSim{
-		MongodbAddr: config.Conf.DB.Mongo1.Addr,
-		Size:        config.Conf.DB.Mongo1.Size,
-		DbName:      config.Conf.DB.Mongo1.Dbname,
-		UserName:    config.Conf.DB.Mongo1.User,
-		Password:    config.Conf.DB.Mongo1.Password,
-	}
-	MongoTool1.InitPool()
-
-	MongoTool2 = &mongodb.MongodbSim{
-		MongodbAddr: config.Conf.DB.Mongo2.Addr,
-		Size:        config.Conf.DB.Mongo2.Size,
-		DbName:      config.Conf.DB.Mongo2.Dbname,
-		UserName:    config.Conf.DB.Mongo2.User,
-		Password:    config.Conf.DB.Mongo2.Password,
-	}
-	MongoTool2.InitPool()
 }
 
 func InitEs() {

fieldproject_inc_data/main.go

@@ -1,33 +1,261 @@
 package main
 
 import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/udp"
+	"encoding/json"
+	"fieldproject_inc_data/config"
 	"fmt"
+	"github.com/robfig/cron"
 	"github.com/spf13/cobra"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.uber.org/zap"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"sync"
+	"time"
 )
 
-func init() {
-
-}
-
 func main() {
 	rootCmd := &cobra.Command{Use: "my cmd"}
 	rootCmd.AddCommand(timeTask())
-	//rootCmd.AddCommand(bidding())
+	rootCmd.AddCommand(fieldTask())
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println("rootCmd.Execute failed", err.Error())
 	}
 }
 
-// @Description 供应商/经销商数据
+// @Description 定时任务 id段
 // @Author J 2022/8/11 16:49
 func timeTask() *cobra.Command {
 	cmdClient := &cobra.Command{
-		Use:   "time task",
+		Use:   "time",
+		Short: "Start scheduled task",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			InitMgo()
+			go checkMapJob()
+
+			UdpClient := udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
+			UdpClient.Listen(func(b byte, data []byte, add *net.UDPAddr) {
+				switch b {
+				case udp.OP_NOOP:
+					ok := string(data)
+					if ok != "" {
+						log.Info("udp re", zap.String("data:", ok))
+						UdpTaskMap.Delete(ok)
+					}
+				}
+			})
+
+			c := cron.New()
+			_ = c.AddFunc("0 */10 * * * ?", func() {
+				log.Info("start process")
+				info, _ := MongoTool.Find("field_data_record", nil, `{"_id": -1}`, nil, true, -1, -1)
+				if info != nil && len(*info) > 0 {
+					if util.IntAll((*info)[0]["status"]) == 0 {
+						mapInfo := make(map[string]interface{})
+						var next = &net.UDPAddr{
+							IP:   net.ParseIP(config.Conf.Udp.Next.Addr),
+							Port: util.IntAll(config.Conf.Udp.Next.Port),
+						}
+						mapInfo["stype"] = config.Conf.Udp.Next.Stype
+						mapInfo["gtid"] = util.ObjToString((*info)[0]["gtid"])
+						mapInfo["lteid"] = util.ObjToString((*info)[0]["lteid"])
+						key := fmt.Sprintf("%s-%s-%s", util.ObjToString(mapInfo["gtid"]), util.ObjToString(mapInfo["lteid"]), config.Conf.Udp.Next.Stype)
+						mapInfo["key"] = key
+						log.Info("udp next node", zap.Any("mapinfo:", mapInfo))
+						datas, _ := json.Marshal(mapInfo)
+						node := &UdpNode{datas, next, time.Now().Unix(), 0}
+						UdpTaskMap.Store(key, node)
+						_ = UdpClient.WriteUdp(datas, udp.OP_TYPE_DATA, next)
+					} else {
+						log.Info("timeTask not find ids")
+					}
+				}
+			})
+			c.Start()
+
+			ch := make(chan bool, 1)
+			<-ch
+		},
+	}
+	return cmdClient
+}
+
+// @Description 后续处理
+// @Author J 2022/9/13 10:50
+func fieldTask() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "field",
 		Short: "Start processing inc field data",
 		Run: func(cmd *cobra.Command, args []string) {
-			//task()
+			InitMgo()
+			InitEs()
+
+			task()
 		},
 	}
-	//cmdClient.Flags().StringVarP(&cfg, "conf", "c", "", "server config [toml]")
 	return cmdClient
 }
+
+func task() {
+	go updateEsMethod()
+
+	UdpClient = udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
+	UdpClient.Listen(processUdpMsg)
+	log.Info("Udp服务监听", zap.String("port:", config.Conf.Udp.LocPort))
+
+	ch := make(chan bool, 1)
+	<-ch
+}
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	defer util.Catch()
+	switch act {
+	case udp.OP_TYPE_DATA: //上个节点的数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Info("processUdpMsg", zap.Any("mapInfo:", mapInfo))
+		gtid, _ := mapInfo["gtid"].(string)
+		lteid, _ := mapInfo["lteid"].(string)
+		if err != nil || gtid == "" || lteid == "" {
+			UdpClient.WriteUdp([]byte("udp error"), udp.OP_NOOP, ra) //udp失败回写
+		} else {
+			//udp成功回写
+			if k := util.ObjToString(mapInfo["key"]); k != "" {
+				UdpClient.WriteUdp([]byte(k), udp.OP_NOOP, ra)
+			} else {
+				k = fmt.Sprintf("%s-%s-%s", gtid, lteid, util.ObjToString(mapInfo["stype"]))
+				UdpClient.WriteUdp([]byte(k), udp.OP_NOOP, ra)
+			}
+			log.Info("start dispose ...")
+			disposeFunc(gtid, lteid)
+		}
+	case udp.OP_NOOP: //回应
+		ok := string(data)
+		if ok != "" {
+			log.Info("udp re", zap.String("data:", ok))
+			UdpTaskMap.Delete(ok)
+		}
+	}
+}
+
+func disposeFunc(gtid, lteid string) {
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+
+	ch := make(chan bool, 2)
+	wg := &sync.WaitGroup{}
+
+	q := bson.M{"_id": bson.M{"$gt": mongodb.StringTOBsonId(gtid), "$lte": mongodb.StringTOBsonId(lteid)}}
+	field := map[string]interface{}{"bid_field": 1}
+	query := sess.DB("qfw").C("bidding").Find(q).Select(field).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%500 == 0 {
+			util.Debug("current ---", count, tmp["_id"])
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			if f := util.ObjToString(tmp["bid_field"]); f != "" {
+				id := mongodb.BsonIdToSId(tmp["_id"])
+				update := make(map[string]interface{})
+				update["bid_field"] = f
+				updateEsPool <- []map[string]interface{}{{
+					"_id": id,
+				},
+					update,
+				}
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+
+	up := bson.M{"$set": bson.M{"status": 1}}
+	MongoTool.Update("field_data_record", map[string]interface{}{"gtid": gtid}, up, false, false)
+	util.Debug("over ---", count)
+}
+
+type UdpNode struct {
+	data      []byte
+	addr      *net.UDPAddr
+	timestamp int64
+	retry     int
+}
+
+func checkMapJob() {
+	if config.Conf.Mail.Send {
+		log.Info("checkMapJob", zap.String("to:", config.Conf.Mail.To))
+		for {
+			UdpTaskMap.Range(func(k, v interface{}) bool {
+				now := time.Now().Unix()
+				node, _ := v.(*UdpNode)
+				if now-node.timestamp > 120 {
+					node.retry++
+					if node.retry > 5 {
+						UdpTaskMap.Delete(k)
+						res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", config.Conf.Mail.Api, config.Conf.Mail.To, "project-send-fail", k.(string)))
+						if err == nil {
+							defer res.Body.Close()
+							read, err := ioutil.ReadAll(res.Body)
+							log.Info("send mail ...", zap.String("r:", string(read)), zap.Any("err:", err))
+						}
+					} else {
+						log.Info("udp重发", zap.Any("k:", k))
+						UdpClient.WriteUdp(node.data, udp.OP_TYPE_DATA, node.addr)
+					}
+				} else if now-node.timestamp > 10 {
+					log.Info("udp任务超时中..", zap.Any("k:", k))
+				}
+				return true
+			})
+			time.Sleep(60 * time.Second)
+		}
+	}
+}
+
+func updateEsMethod() {
+	arru := make([][]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updateEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk("bidding", "bidding", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk("bidding", "bidding", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

medical_project/common.toml

@@ -0,0 +1,45 @@
+[server]
+thread = 2
+validtime = 150
+statusdays = 15
+[udp]
+locport = ":17821"
+
+[db]
+[db.mongoB]
+addr = "127.0.0.1:27092"
+dbname = "wjh"
+coll = "bidding"
+size = 10
+user = ""
+password = ""
+
+[db.mongoP]
+addr = "127.0.0.1:27092"
+dbname = "wjh"
+coll = "projectset_medical"
+size = 5
+user = ""
+password = ""
+
+[mail]
+send = false
+to = "wangjianghan@topnet.net.cn"
+api = "http://172.17.145.179:19281/_send/_mail"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"

medical_project/config/conf.go

@@ -0,0 +1,83 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	// Conf crocodile conf
+	Conf *conf
+)
+
+// Init Config
+func Init(conf string) {
+	_, err := toml.DecodeFile(conf, &Conf)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+}
+
+type conf struct {
+	Server server
+	DB     db
+	Udp    udp
+	Mail   mail
+	Log    log
+}
+
+type server struct {
+	Thread     int
+	ValidTime  int
+	StatusDays int
+}
+
+type udp struct {
+	LocPort string
+}
+
+type mail struct {
+	Send bool
+	To   string
+	Api  string
+}
+
+// Log Config
+type log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type db struct {
+	MongoB mgo
+	MongoP mgo
+}
+
+type mgo struct {
+	Addr     string
+	Dbname   string
+	Coll     string
+	Size     int
+	User     string
+	Password string
+}
+
+type duration struct {
+	time.Duration
+}
+
+// UnmarshalText parse 10s to time.Time
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}

medical_project/config/conf_test.go

@@ -0,0 +1,194 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var confs = `
+[udp]
+locport = ":1783"
+jyaddr = "127.0.0.1"
+jyport = 11118
+
+[db]
+[db.mongoB]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+coll = "bidding"
+size = 15
+user = ""
+password = ""
+
+[db.mongoP]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+coll = "projectset"
+size = 15
+user = ""
+password = ""
+
+[db.mongoQ]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+coll = "projectset"
+size = 15
+user = ""
+password = ""
+
+[db.es]
+addr = "http://192.168.3.206:9800"
+size = 5
+indexb = "bidding"
+typeb = "bidding"
+indexp = "projectset"
+typep = "projectset"
+indexwinner = "winner"
+typewinner = "winner"
+indexbuyer = "buyer"
+typebuyer = "buyer"
+detailfilter = ["(招标网|千里马|采招网|招标采购导航网|招标与采购网|中国招投标网|中国采购与招标网|中国采购与招标|优质采)[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}(qianlima|zhaobiao|okcis|zbytb|infobidding|bidcenter|youzhicai|chinabidding|Chinabidding｜CHINABIDDING)[a-z0-9.\\/\\/]{0,40}",
+    "招标网[\\w\\W]{0,15}[http|https|htpps]?[a-z0-9:\\/\\/.]{0,20}zhaobiao[a-z0-9.\\/\\/]{0,40}",
+    "千里马[\\w\\W]{0,15}[a-z0-9:\\/\\/.]{0,20}qianlima[a-z0-9.\\/\\/]{0,10}",
+    "[\\(（]?(网址)?[:：；;]?(http|https|htpps)*[:：]?(\\/\\/)?(www|jinan|WWW)?.(zhaobiao|chinabidding|Chinabidding|CHINABIDDING|infobidding|zbytb|okcis|qianlima|youzhicai).(com|cn|COM|CN)?(.cn|.CN)?\\/?[\\)）]?",
+    "[\\(（]?(网址)?(:：)?(http|https|htpps)*(:|：)?\\/\\/www.bidcenter.com.cn\\/",
+    "千里马(平台|网站)+", "[“\"]?优质采(平台|电子交易平台|云采购平台|交易平台)?[”\"]?", "《?(中国采购与|中国)?招(投)?标(与采购|采购导航)?网》?",
+    "《?元博网(采购与招标网)?》?", "《?(中国)?招标采购导航网》?", "中\\W{0,3}国采\\W{0,3}招\\W{0,3}网\\W*[(（]?(bidcenter.com.cn)?[)）]?", "已方宝", "中国招标与采购"]
+
+[mail]
+send = false
+to = "wangjianghan@topnet.net.cn"
+api = "http://172.17.145.179:19281/_send/_mail"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"
+
+[db.es.fieldes]
+"_id" = ""
+"buyerzipcode" = "string"
+"winnertel" = "string"
+"winnerperson" = "string"
+"contractcode" = "string"
+"winneraddr" = "string"
+"agencyaddr" = "string"
+"buyeraddr" = "string"
+"signaturedate" = "int64"
+"projectperiod" = "string"
+"projectaddr" = "string"
+"agencytel" = "string"
+"agencyperson" = "string"
+"buyerperson" = "string"
+"agency" = "string"
+"projectscope" = "string"
+"projectcode" = "string"
+"bidopentime" = "int64"
+"supervisorrate" = "float64"
+"buyertel" = "string"
+"bidamount" = "float64"
+"winner" = "string"
+"buyer" = "string"
+"budget" = "float64"
+"projectname" = "string"
+"bidstatus" = "string"
+"buyerclass" = "string"
+"topscopeclass" = ""
+"s_topscopeclass" = "string"
+"s_subscopeclass" = "string"
+"area" = "string"
+"city" = "string"
+"district" = "string"
+"s_winner" = "string"
+"title" = "string"
+"detail" = "string"
+"site" = "string"
+"comeintime" = "int64"
+"href" = "string"
+"infoformat" = "int32"
+"publishtime" = "int64"
+"s_sha" = "string"
+"spidercode" = "string"
+"subtype" = "string"
+"toptype" = "string"
+"projectinfo" = ""
+"purchasing" = "string"
+"purchasinglist" = ""
+"channel" = "string"
+"winnerorder" = ""
+"project_scale" = "string"
+"project_duration" = "int32"
+"project_timeunit" = "string"
+"project_startdate" = "int64"
+"project_completedate" = "int64"
+"payway" = "string"
+"contract_guarantee" = "bool"
+"bid_guarantee" = "bool"
+"qualifies" = ""
+"entidlist" = ""
+"funds" = "string"
+"review_experts" = "string"
+"bidmethod" = "string"
+"bidendtime" = "int64"
+"bidopenaddress" = "string"
+"docamount" = "float64"
+"agencyrate" = "float64"
+"agencyfee" = "float64"
+"bidway" = "string"
+"getdocmethod" = "string"
+"china_bidding" = "string"
+"purchasing_tag" = "string"
+"multipackage" = "int32"
+"isValidFile" = "bool"
+"bid_field" = "string"
+[db.es.fieldprojectinfo]
+"approvecode" = "string"
+"approvecontent" = "string"
+"approvestatus" = "string"
+"approvetime" = "string"
+"approvedept" = "string"
+"approvenumber" = "string"
+"projecttype" = "string"
+"approvecity" = "string"
+[db.es.fieldpurchasinglist]
+"itemname" = "string"
+"item" = "string"
+"brandname" = "string"
+"model" = "string"
+"unitname" = "string"
+"number" = "float64"
+"unitprice" = "float64"
+"totalprice" = "float64"
+[db.es.fieldprocurementlist]
+"projectname" = "string"
+"buyer" = "string"
+"item" = "string"
+"projectscope" = "string"
+"expurasingtime" = "string"
+"totalprice" = "float64"
+[db.es.fieldwinnerorder]
+"sort" = "int"
+"sortstr" = "string"
+"entname" = "string"
+`
+
+func TestInit(t *testing.T) {
+	testfile := "/tmp/crocodile.toml"
+	ioutil.WriteFile(testfile, []byte(confs), 0644)
+	Init(testfile)
+	//t.Logf("%+v", Conf.DB.Es.FieldPurchasingList)
+	os.Remove(testfile)
+}

medical_project/deepcopy.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"reflect"
+	"time"
+)
+
+// Interface for delegating copy process to type
+type Interface interface {
+	DeepCopy() interface{}
+}
+
+// Iface is an alias to Copy; this exists for backwards compatibility reasons.
+func Iface(iface interface{}) interface{} {
+	return Copy(iface)
+}
+
+// Copy creates a deep copy of whatever is passed to it and returns the copy
+// in an interface{}.  The returned value will need to be asserted to the
+// correct type.
+func Copy(src interface{}) interface{} {
+	if src == nil {
+		return nil
+	}
+
+	// Make the interface a reflect.Value
+	original := reflect.ValueOf(src)
+
+	// Make a copy of the same type as the original.
+	cpy := reflect.New(original.Type()).Elem()
+
+	// Recursively copy the original.
+	copyRecursive(original, cpy)
+
+	// Return the copy as an interface.
+	return cpy.Interface()
+}
+
+// copyRecursive does the actual copying of the interface. It currently has
+// limited support for what it can handle. Add as needed.
+func copyRecursive(original, cpy reflect.Value) {
+	// check for implement deepcopy.Interface
+	if original.CanInterface() {
+		if copier, ok := original.Interface().(Interface); ok {
+			cpy.Set(reflect.ValueOf(copier.DeepCopy()))
+			return
+		}
+	}
+
+	// handle according to original's Kind
+	switch original.Kind() {
+	case reflect.Ptr:
+		// Get the actual value being pointed to.
+		originalValue := original.Elem()
+
+		// if  it isn't valid, return.
+		if !originalValue.IsValid() {
+			return
+		}
+		cpy.Set(reflect.New(originalValue.Type()))
+		copyRecursive(originalValue, cpy.Elem())
+
+	case reflect.Interface:
+		// If this is a nil, don't do anything
+		if original.IsNil() {
+			return
+		}
+		// Get the value for the interface, not the pointer.
+		originalValue := original.Elem()
+
+		// Get the value by calling Elem().
+		copyValue := reflect.New(originalValue.Type()).Elem()
+		copyRecursive(originalValue, copyValue)
+		cpy.Set(copyValue)
+
+	case reflect.Struct:
+		t, ok := original.Interface().(time.Time)
+		if ok {
+			cpy.Set(reflect.ValueOf(t))
+			return
+		}
+		// Go through each field of the struct and copy it.
+		for i := 0; i < original.NumField(); i++ {
+			// The Type's StructField for a given field is checked to see if StructField.PkgPath
+			// is set to determine if the field is exported or not because CanSet() returns false
+			// for settable fields.  I'm not sure why.  -mohae
+			if original.Type().Field(i).PkgPath != "" {
+				continue
+			}
+			copyRecursive(original.Field(i), cpy.Field(i))
+		}
+
+	case reflect.Slice:
+		if original.IsNil() {
+			return
+		}
+		// Make a new slice and copy each element.
+		cpy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))
+		for i := 0; i < original.Len(); i++ {
+			copyRecursive(original.Index(i), cpy.Index(i))
+		}
+
+	case reflect.Map:
+		if original.IsNil() {
+			return
+		}
+		cpy.Set(reflect.MakeMap(original.Type()))
+		for _, key := range original.MapKeys() {
+			originalValue := original.MapIndex(key)
+			copyValue := reflect.New(originalValue.Type()).Elem()
+			copyRecursive(originalValue, copyValue)
+			copyKey := Copy(key.Interface())
+			cpy.SetMapIndex(reflect.ValueOf(copyKey), copyValue)
+		}
+
+	default:
+		cpy.Set(original)
+	}
+}

medical_project/go.mod

@@ -0,0 +1,10 @@
+module medical_project
+
+go 1.16
+
+require (
+	app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d
+	github.com/BurntSushi/toml v1.2.0
+	go.mongodb.org/mongo-driver v1.10.3
+	go.uber.org/zap v1.22.0
+)

medical_project/go.sum

@@ -0,0 +1,115 @@
+app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d h1:Nh2rC3LBqh0alvam2vr4is/vbUaPkl0rbZxVETx3nmk=
+app.yhyue.com/data_processing/common_utils v0.0.0-20220927054143-d9e97522625d/go.mod h1:9PlRUNzirlF/LL1W7fA7koCudxJe3uO5nshDWlCnGo8=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.10.3 h1:XDQEvmh6z1EUsXuIkXE9TaVeqHw6SwS1uf93jFs0HBA=
+go.mongodb.org/mongo-driver v1.10.3/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

medical_project/init.go

@@ -0,0 +1,258 @@
+package main
+
+import (
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"medical_project/config"
+	"os"
+	"regexp"
+	"sync"
+)
+
+var (
+	MongoBid, MongoPro *mongodb.MongodbSim //mongodb连接
+
+	wg       sync.WaitGroup
+	findLock sync.Mutex
+	//采购单位、项目名称、项目编号
+	mapPb, mapPn, mapPc map[string]*Key
+	AllIdsMapLock       sync.Mutex
+	AllIdsMap           map[string]*ID
+
+	//bidtype、bidstatus 锁
+	mapBidLock sync.Mutex
+
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+	saveSize   int
+
+	currentType string
+	currentTime int64
+	pici        int64
+	validTime   int64
+	statusTime  int64
+	jgTime      int64
+
+	compareNoPass = map[string]bool{}
+	compareAB     = map[string]bool{}
+	compareAB2D   = map[string]bool{}
+	compareABD    = map[string]bool{}
+	compareAB2CD  = map[string]bool{}
+	compareABCD   = map[string]bool{}
+
+	_datereg   = regexp.MustCompile("20[0-2][0-9][年-][0-9]{1,2}[月-][0-9]{1,2}[日-]([0-9]{1,2}时[0-9]{0,2})?")
+	replaceStr = regexp.MustCompile("(工程|采购|项目|[?！、【】（）—()－-]|栏标价|中标候选人|招标代理)")
+	//纯数字或纯字母
+	StrOrNum2 = regexp.MustCompile("^[0-9_-]+$|^[a-zA-Z_-]+$")
+	//含分包词，招标未识别分包  合并到一个项目
+	KeyPackage  = regexp.MustCompile("[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}(包|段)|(包|段)[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}")
+	titleGetPc  = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)")
+	titleGetPc1 = regexp.MustCompile("[\\[【(（](.{0,6}(编号|编码|项号|包号|代码|标段?号)[:：为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】)）]")
+	titleGetPc2 = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$")
+	numCheckPc  = regexp.MustCompile("^[0-9-]{1,10}$")
+	StrOrNum    = regexp.MustCompile("^[0-9_-]{1,4}$|^[a-zA-Z_-]{1,4}$")
+	_zimureg1   = regexp.MustCompile("^[a-zA-Z-]{1,7}$")
+)
+
+func init() {
+	config.Init("./common.toml")
+	InitLog()
+	InitMgo()
+
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 1)
+	saveSize = 200
+
+	wg = sync.WaitGroup{}
+	mapPn = make(map[string]*Key, 5000000)
+	AllIdsMap = make(map[string]*ID, 5000000)
+	mapPb = make(map[string]*Key, 1500000)
+	mapPn = make(map[string]*Key, 5000000)
+	mapPc = make(map[string]*Key, 5000000)
+
+	validTime = int64(config.Conf.Server.ValidTime * 86400)
+	statusTime = int64(config.Conf.Server.StatusDays * 86400)
+	jgTime = int64(7 * 86400)
+
+	//加载项目数据
+	//---不能通过
+	vm := []string{"C", "D"}
+	//三个元素一致 [AB][AB][AB]，分值最高
+	vm = []string{"A", "B"}
+	for i := 0; i < 2; i++ {
+		for j := 0; j < 2; j++ {
+			for k := 0; k < 2; k++ {
+				key := vm[i] + vm[j] + vm[k]
+				compareAB[key] = true
+			}
+		}
+	}
+}
+
+func InitLog() {
+	logcfg := config.Conf.Log
+
+	err := log.InitLog(
+		log.Path(logcfg.LogPath),
+		log.Level(logcfg.LogLevel),
+		log.Compress(logcfg.Compress),
+		log.MaxSize(logcfg.MaxSize),
+		log.MaxBackups(logcfg.MaxBackups),
+		log.MaxAge(logcfg.MaxAge),
+		log.Format(logcfg.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func InitMgo() {
+	MongoBid = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoB.Addr,
+		DbName:      config.Conf.DB.MongoB.Dbname,
+		Size:        config.Conf.DB.MongoB.Size,
+		UserName:    config.Conf.DB.MongoB.User,
+		Password:    config.Conf.DB.MongoB.Password,
+	}
+	MongoBid.InitPool()
+	MongoPro = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoP.Addr,
+		DbName:      config.Conf.DB.MongoP.Dbname,
+		Size:        config.Conf.DB.MongoP.Size,
+		UserName:    config.Conf.DB.MongoP.User,
+		Password:    config.Conf.DB.MongoP.Password,
+	}
+	MongoPro.InitPool()
+}
+
+//招标信息实体类
+type Info struct {
+	Id                  string                   `json:"_id"`
+	Href                string                   `json:"href"` //源地址
+	Publishtime         int64                    `json:"publishtime"`
+	Comeintime          int64                    `json:"comeintime"`
+	Title               string                   `json:"title"`
+	TopType             string                   `json:"toptype"`
+	SubType             string                   `json:"subtype"`
+	ProjectName         string                   `json:"projectname"`
+	ProjectCode         string                   `json:"projectcode"`
+	ProjectScope        string                   `json:"projectscope"`
+	ContractCode        string                   `json:"contractcode"`
+	Buyer               string                   `json:"buyer"`
+	Buyerperson         string                   `json:"buyerperson"`
+	Buyertel            string                   `json:"buyertel"`
+	Agency              string                   `json:"agency"`
+	Area                string                   `json:"area"`
+	City                string                   `json:"city"`
+	District            string                   `json:"district"`
+	Infoformat          int                      `json:"infoformat"`
+	ReviewExperts       []string                 `json:"review_experts"`
+	Purchasing          string                   `json:"purchasing"`
+	WinnerOrder         []map[string]interface{} `json:"winnerorder"`
+	ProjectScale        string                   `json:"project_scale"`
+	ProjectDuration     int                      `json:"project_duration"`
+	ProjectTimeUnit     string                   `json:"project_timeunit"`
+	ProjectStartDate    int64                    `json:"project_startdate"`
+	ProjectCompleteDate int64                    `json:"project_completedate"`
+	Payway              string                   `json:"payway"`
+	ContractGuarantee   bool                     `json:"contract_guarantee"`
+	BidGuarantee        bool                     `json:"bid_guarantee"`
+	Qualifies           []map[string]interface{} `json:"qualifies"`
+	EntIdList           []string                 `json:"entidlist"`
+	HasPackage          bool                     // `json:"haspackage"`
+	Package             map[string]interface{}   `json:"package"`
+	Topscopeclass       []string                 `json:"topscopeclass"`
+	Subscopeclass       []string                 `json:"subscopeclass"`
+	Buyerclass          string                   `json:"buyerclass"`
+	Bidopentime         int64                    `json:"bidopentime"`
+	Budget              float64                  `json:"budget"`
+	Bidamount           float64                  `json:"bidamount"`
+	Winners             []string
+	dealtype            int
+	PTC                 string //从标题中抽的项目编号
+	pnbval              int    //项目名称、编号、采购单位存在的个数
+	LenPC               int    //项目编号长度
+	LenPN               int    //项目名称长度
+	LenPTC              int    //标题抽的项目编号长度
+	//以下三个元素做对比，计算包含时候使用
+	PNBH  int //0初始，+包含，-被包含
+	PCBH  int
+	PTCBH int
+}
+
+// 内存 项目信息
+type Project struct {
+	Id            primitive.ObjectID `json:"_id"`
+	Ids           []string           `json:"ids,omitempty"`
+	FirstTime     int64              `json:"firsttime,omitempty"`   //项目的最早时间
+	LastTime      int64              `json:"lasttime,omitempty"`    //项目的最后时间
+	ProjectName   string             `json:"projectname,omitempty"` //项目名称
+	ProjectCode   string             `json:"projectcode,omitempty"` //项目代码唯一（纯数字的权重低）
+	Buyer         string             `json:"buyer,omitempty"`       //采购单位唯一
+	Agency        string             `json:"agency"`                //代理机构
+	Area          string             `json:"area"`                  //地区
+	City          string             `json:"city"`                  //地市
+	District      string             `json:"district"`              //区县
+	Bidamount     float64            `json:"bidamount,omitempty"`   //中标金额
+	Budget        float64            `json:"budget,omitempty"`      //预算
+	score         int
+	comStr        string
+	MPN           []string `json:"mpn,omitempty"` //合并后多余的项目名称
+	MPC           []string `json:"mpc,omitempty"` //合并后多余的项目编号
+	resVal, pjVal int
+
+	Topscopeclass      []string               `json:"topscopeclass,omitempty"`
+	Subscopeclass      []string               `json:"subscopeclass,omitempty"` //子行业分类
+	Winners            string                 `json:"s_winner,omitempty"`      //中标人
+	ContractCode       string                 `json:"contractcode,omitempty"`  //合同编号
+	Buyerperson        string                 `json:"buyerperson"`             //采购联系人
+	Buyertel           string                 `json:"buyertel"`                //采购联系人电话
+	Bidstatus          string                 `json:"bidstatus"`               //
+	Bidtype            string                 `json:"bidtype"`                 //
+	ReviewExperts      []string               `json:"review_experts"`          // 项目评审专家
+	Purchasing         string                 `json:"purchasing"`              // 标的物
+	Package            map[string]interface{} `json:"package,omitempty"`       //分包的对比对象
+	Buyerclass         string                 `json:"buyerclass"`              //采购单位分类
+	Bidopentime        int64                  `json:"bidopentime,omitempty"`   //开标时间
+	Jgtime             int64                  `json:"jgtime"`                  //结果中标时间
+	Zbtime             int64                  `json:"zbtime"`                  //招标时间
+	Winnerorder        []string               `json:"winnerorder"`             //中标候选人
+	ProjectScale       string                 `json:"project_scale"`           //项目规模
+	ProjectDuration    int                    `json:"project_duration"`        //工期时长
+	ProjectTimeunit    string                 `json:"project_timeunit"`        //工期时长单位
+	ProjectStartDate   int64                  `json:"project_startdate"`       //开工日期
+	ProjctCompleteDate int64                  `json:"projct_completedate"`     //竣工日期
+	Payway             string                 `json:"payway"`                  //付款方式
+	ContractGuarantee  bool                   `json:"contract_guarantee"`      //履约保证金 是否支持包含
+	BidGuarantee       bool                   `json:"bid_guarantee"`           //投标保证金 是否支持包含
+	Qualifies          string                 `json:"qualifies"`               //资质条件
+	EntIdList          []string               `json:"entidlist"`               //企业id
+	//FirstCooperation   []string               `json:"first_cooperation"`       //first_cooperation
+	InfoFiled    map[string]InfoField `json:"infofield"`    //逻辑处理需要的info字段
+	Budgettag    int                  `json:"budgettag"`    //预算是否有效标记
+	Bidamounttag int                  `json:"bidamounttag"` //中标金额是否有效标记
+}
+
+//存储部分招标信息字段，业务逻辑处理需要
+type InfoField struct {
+	Budget       float64 `json:"budget"`
+	Bidamount    float64 `json:"bidamount"`
+	ContractCode string  `json:"contractcode"`
+	ProjectName  string  `json:"projectname"`
+	ProjectCode  string  `json:"projectcode"`
+	Bidstatus    string  `json:"bidstatus"`
+}
+
+type ID struct {
+	Id   string
+	Lock sync.Mutex
+	P    *Project
+}
+
+type Key struct {
+	Arr  []string
+	Lock sync.Mutex
+}

medical_project/load_data.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"medical_project/config"
+	"time"
+)
+
+func loadData(starttime int64) {
+	log.Println("load project start..", starttime)
+	findLock.Lock()
+	defer findLock.Unlock()
+	AllIdsMapLock.Lock()
+	defer AllIdsMapLock.Unlock()
+	sess := MongoPro.GetMgoConn()
+	defer MongoPro.DestoryMongoConn(sess)
+	loadOver := make(chan bool)
+	q := map[string]interface{}{
+		"lasttime": map[string]interface{}{"$gte": starttime},
+	}
+	//field := map[string]interface{}{"list": 0}
+	it := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.Coll).Find(&q).Select(nil).Iter()
+	n := 0
+	pool := make(chan *Project, 1000)
+	go func() {
+		for {
+			select {
+			case tmp := <-pool:
+				n++
+				if n%10000 == 0 {
+					log.Println("current", n, "\n", tmp.Id, len(mapPn), len(mapPc), len(mapPb)) //, tmp.ProjectName, tmp.MPN, tmp.ProjectCode, tmp.MPC, tmp.Buyer)
+				}
+				if tmp != nil {
+					id := tmp.Id.Hex()
+					for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) {
+						if v != "" {
+							//v = pcReplace.ReplaceAllString(v, "")
+							if v != "" {
+								k := mapPn[v]
+								if k == nil {
+									k = &Key{Arr: []string{id}}
+									mapPn[v] = k
+								} else {
+									k.Arr = append(k.Arr, id)
+								}
+							}
+						}
+					}
+					for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) {
+						if v != "" {
+							//v = pcReplace.ReplaceAllString(v, "")
+							if v != "" {
+								k := mapPc[v]
+								if k == nil {
+									k = &Key{Arr: []string{id}}
+									mapPc[v] = k
+								} else {
+									k.Arr = append(k.Arr, id)
+								}
+							}
+						}
+					}
+					if tmp.Buyer != "" && len([]rune(tmp.Buyer)) > 2 {
+						k := mapPb[tmp.Buyer]
+						if k == nil {
+							k = &Key{Arr: []string{id}}
+							mapPb[tmp.Buyer] = k
+						} else {
+							k.Arr = append(k.Arr, id)
+						}
+					}
+					AllIdsMap[id] = &ID{Id: id, P: tmp}
+				}
+			case <-loadOver:
+				return
+			}
+		}
+	}()
+	for {
+		result := make(map[string]interface{})
+		if it.Next(&result) {
+			go func(res map[string]interface{}) {
+				bys, _ := json.Marshal(result)
+				var tmp *Project
+				_ = json.Unmarshal(bys, &tmp)
+				saveFiled(result, tmp)
+				pool <- tmp
+			}(result)
+		} else {
+			break
+		}
+	}
+	time.Sleep(2 * time.Second)
+	loadOver <- true
+	log.Println("load project over..", n)
+}
+
+func saveFiled(res map[string]interface{}, tmp *Project) {
+	tmpMap := make(map[string]InfoField)
+	infoMap := res["infofield"].(map[string]interface{})
+	for _, v := range infoMap {
+		var field InfoField
+		b, _ := json.Marshal(v)
+		_ = json.Unmarshal(b, &field)
+		tmpMap[tmp.Id.Hex()] = field
+	}
+	tmp.InfoFiled = tmpMap
+}

medical_project/main.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/udp"
+	"encoding/json"
+	"go.uber.org/zap"
+	"medical_project/config"
+	"net"
+	"time"
+)
+
+var (
+	udpClient udp.UdpClient
+
+	SingleThread = make(chan bool, 1)
+	UdpChan      = make(chan map[string]interface{}, 500)
+)
+
+func main() {
+	go updateAllQueue()
+
+	loadData(0)
+
+	udpClient = udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
+	udpClient.Listen(processUdpMsg)
+	log.Info("Udp服务监听", zap.String("port:", config.Conf.Udp.LocPort))
+
+	for {
+		mapinfo, ok := <-UdpChan
+		if !ok {
+			continue
+		}
+		SingleThread <- true
+		tasktype := util.ObjToString(mapinfo["stype"])
+		switch tasktype {
+		case "ql": //全量合并
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+				currentType = tasktype
+				pici = time.Now().Unix()
+				taskQl(mapinfo)
+			}()
+		case "project": //增量合并，
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+				currentType = tasktype
+				pici = time.Now().Unix()
+				taskZl(mapinfo)
+			}()
+		case "project_history": //增量合并， id段历史数据
+			go func() {
+				defer func() {
+					<-SingleThread
+				}()
+				currentType = tasktype
+				pici = time.Now().Unix()
+				taskZl(mapinfo)
+			}()
+		default:
+			<-SingleThread
+		}
+	}
+}
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case udp.OP_TYPE_DATA: //上个节点的数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Info("err:", zap.Error(err), zap.Any("mapInfo:", mapInfo))
+		if err != nil {
+			_ = udpClient.WriteUdp([]byte("err:"+err.Error()), udp.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			key, _ := mapInfo["key"].(string)
+			if key == "" {
+				key = "udpok"
+			}
+			go udpClient.WriteUdp([]byte(key), udp.OP_NOOP, ra)
+			UdpChan <- mapInfo
+		}
+	case udp.OP_NOOP: //下个节点回应
+		ok := string(data)
+		if ok != "" {
+			log.Info("re", zap.String("ok:", ok))
+		}
+	}
+}

medical_project/merge_comparepnc.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"strings"
+)
+
+//对比项目名称、项目编号
+
+func comparePNC(info *Info, compareProject *Project) (compareStr string, score int) {
+	if info.ProjectName != "" {
+		pns := []string{}
+		if compareProject.ProjectName != "" {
+			pns = append(pns, compareProject.ProjectName)
+		}
+		if len(compareProject.MPN) > 0 {
+			pns = append(pns, compareProject.MPN...)
+		}
+		ifind := 0
+		templen := 0
+		buyer := info.Buyer
+		if buyer == "" {
+			buyer = compareProject.Buyer
+		}
+		pn := info.ProjectName
+		if buyer != "" {
+			pn = strings.Replace(pn, buyer, "", -1)
+		}
+		for _, v := range pns {
+			if info.ProjectName == v {
+				ifind = 1
+				break
+			} else {
+				//if strings.Contains(info.ProjectName, v) || strings.Contains(v, info.ProjectName) ||
+				retv := CheckContain(info.ProjectName, v)
+				if retv == 1 {
+					ifind = 1
+					break
+				} else {
+					if buyer != "" {
+						v = strings.Replace(v, buyer, "", -1)
+					}
+					//v1 := CosineSimilar(pn, v)
+					if retv == 2 {
+						templen = len([]rune(v))
+						ifind = 2
+					} else if ifind == 0 {
+						ifind = 3
+					}
+				}
+			}
+		}
+		switch ifind {
+		case 0:
+			compareStr = "D"
+		case 1:
+			compareStr = "A"
+			score += 4
+			if len([]rune(info.ProjectName)) > 18 {
+				score += 2
+			}
+		case 2:
+			compareStr = "B"
+			score += 2
+			if templen > info.LenPN {
+				templen = info.LenPN
+			}
+			info.PNBH = templen
+			if templen > 12 {
+				score += 1
+			}
+		case 3:
+			compareStr = "C"
+		}
+	} else {
+		compareStr = "D"
+	}
+
+	/*
+				项目编号 - －()（） 要注意
+				init_text = ["号","(重)","(第二次)","（重）"]
+		all_clean_mark = ["[","(","【","（","〖","]",")","】","）","〗","-","〔","〕","《","［","］","｛","｝","{","—"," ","－","﹝","﹞","–"]
+	*/
+	for index, pc := range []string{info.ProjectCode, info.PTC} {
+		if pc != "" {
+			pcs := []string{}
+			if compareProject.ProjectCode != "" {
+				pcs = append(pcs, compareProject.ProjectCode)
+			}
+			if len(compareProject.MPC) > 0 {
+				pcs = append(pcs, compareProject.MPC...)
+			}
+			ifind := 0
+			templen := 0
+			for _, v := range pcs {
+				if pc == v {
+					ifind = 1
+					break
+				} else {
+					// math.Abs(float64(len([]rune(pc))-len([]rune(v)))) < 6
+					//if !_numreg1.MatchString(pc) && !_zimureg1.MatchString(pc) && !_numreg1.MatchString(v) && !_zimureg1.MatchString(v)
+					if strings.Contains(pc, v) || strings.Contains(v, pc) {
+						t1 := pc
+						t2 := v
+						if len(v) > len(pc) {
+							t1 = v
+							t2 = pc
+						}
+						t3 := strings.Replace(t1, t2, "", -1)
+						t3 = _datereg.ReplaceAllString(t3, "")
+						if t3 == "" {
+							ifind = 1
+							break
+						} else {
+							ifind = 2
+							templen = len([]rune(v))
+						}
+					} else if ifind == 0 {
+						ifind = 3
+					}
+				}
+			}
+			switch ifind {
+			case 0:
+				compareStr += "D"
+			case 1:
+				compareStr += "A"
+				score += 4
+				if len([]rune(pc)) > 18 {
+					score += 2
+				}
+			case 2:
+				compareStr += "B"
+				score += 2
+				if index == 0 {
+					if templen > info.LenPC {
+						templen = info.LenPC
+					}
+					info.PCBH = templen
+					if templen > 12 {
+						score += 1
+					}
+
+				} else {
+					if templen > info.LenPTC {
+						templen = info.LenPTC
+					}
+					info.PTCBH = templen
+					if templen > 12 {
+						score += 1
+					}
+				}
+
+			case 3:
+				compareStr += "C"
+			}
+
+		} else {
+			compareStr += "D"
+		}
+	}
+	return
+}
+
+func CheckContain(b1, b2 string) (res int) {
+	b1 = replaceStr.ReplaceAllString(b1, "")
+	b2 = replaceStr.ReplaceAllString(b2, "")
+
+	if b1 == b2 {
+		res = 1 //相等
+		return
+	}
+	bs1 := []rune(b1)
+	bs2 := []rune(b2)
+	tmp := ""
+	for i := 0; i < len(bs1); i++ {
+		for j := 0; j < len(bs2); j++ {
+			if bs1[i] == bs2[j] {
+				tmp += string(bs1[i])
+			} else if tmp != "" {
+				b1 = strings.Replace(b1, tmp, "", -1)
+				b2 = strings.Replace(b2, tmp, "", -1)
+				tmp = ""
+			}
+		}
+	}
+	if tmp != "" {
+		b1 = strings.Replace(b1, tmp, "", -1)
+		b2 = strings.Replace(b2, tmp, "", -1)
+	}
+	if b1 == b2 {
+		res = 1 //相等
+	} else if b1 == "" || b2 == "" {
+		res = 2 //包含
+	} else {
+		res = 3 //不相等
+	}
+	return
+}

medical_project/merge_select.go

@@ -0,0 +1,510 @@
+package main
+
+//根据字符特征打分
+//3为最高分，pj为评级 A AD A  AA AA AB
+func Select(compareStr string, info *Info, compareInfo *Project) (res, pj int) {
+	//没有可对比的项目名称、或项目编号 //评级
+	if compareNoPass[compareStr] {
+
+	} else {
+		switch compareStr {
+		case "AAA":
+			res = 3
+			pj = 3
+		case "AAB":
+			res = 3
+			pj = 3
+		case "ABA":
+			res = 3
+			pj = 3
+		case "ABB":
+			if info.LenPN > 10 || info.PCBH > 8 || info.PTCBH > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BAA":
+			if info.PNBH > 10 || info.LenPC > 8 || info.LenPTC > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BAB":
+			if info.PNBH > 10 || info.LenPTC > 8 || info.PTCBH > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BBA":
+			if info.PNBH > 10 || info.PCBH > 8 || info.LenPC > 8 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "BBB":
+			v := 0
+			if info.PNBH > 10 {
+				v++
+			}
+			if info.PCBH > 8 {
+				v++
+			}
+			if info.PTCBH > 8 {
+				v++
+			}
+			if v > 1 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 2
+		case "AAD":
+			if info.LenPC > 8 || info.LenPN > 12 {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 3
+		case "ABD":
+			if info.LenPN > 10 && info.PCBH > 8 {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BAD":
+			if info.LenPC > 13 || (info.PNBH > 10 && info.LenPC > 8) {
+				res = 3
+				pj = 3
+			} else if info.PNBH > 10 || info.LenPC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BBD":
+			if info.PNBH > 12 && info.PCBH > 10 {
+				res = 3
+				pj = 1
+			} else if info.PNBH > 10 && info.PCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "ADA":
+			if info.LenPN > 12 || (info.LenPTC > 8 && !StrOrNum2.MatchString(info.PTC)) {
+				res = 3
+			} else {
+				res = 2
+			}
+			pj = 2
+		case "ADB":
+			if info.LenPN > 10 && info.PTCBH > 8 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PTCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BDA":
+			if info.PNBH > 10 && info.LenPTC > 8 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.PNBH > 10 || info.LenPTC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BDB":
+			if info.PNBH > 12 && info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+			} else if info.PNBH > 10 && info.PTCBH > 8 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 2
+		case "ADD":
+			if info.LenPN > 18 {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "BDD":
+			if info.PNBH > 10 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "DAA":
+			if info.LenPTC > 8 || info.LenPC > 8 {
+				res = 3
+				pj = 2
+			} else {
+				res = 2
+				pj = 3
+			}
+		case "DAB":
+			if info.LenPC > 8 && info.PTCBH > 8 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 || info.PTCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DAD":
+			if info.LenPC > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "DBA":
+			if info.PCBH > 8 && info.LenPC > 8 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.PCBH > 8 || info.LenPC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DBB":
+			if info.PCBH > 10 && info.PTCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 1
+			} else if info.PCBH > 8 && info.PTCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "DBD":
+			if info.PCBH > 12 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+				pj = 1
+			} else {
+				res = 1
+				pj = 1
+			}
+		case "DDA":
+			if info.LenPTC > 14 && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 1
+			} else if info.LenPTC > 8 {
+				res = 2
+				pj = 1
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "DDB":
+			if info.PTCBH > 12 && !StrOrNum2.MatchString(info.PTC) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "ACA":
+			if info.LenPN > 10 && info.LenPTC > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.LenPTC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "ACB":
+			if info.LenPN > 10 && info.PTCBH > 8 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PTCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BCA":
+			if (info.PNBH > 10 && info.LenPTC > 8) || info.LenPTC > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 2
+			} else if info.PNBH > 10 || info.LenPTC > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BCB":
+			if info.PNBH > 12 && info.PTCBH > 12 && info.LenPC != len([]rune(compareInfo.ProjectCode)) && !StrOrNum2.MatchString(info.PTC) {
+				res = 3
+				pj = 1
+			} else if info.PNBH > 10 || info.PTCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "AAC":
+			if (info.LenPN > 10 && info.LenPC > 8) || info.LenPN > 14 || (info.LenPC > 10 && !StrOrNum2.MatchString(info.ProjectCode)) {
+				res = 3
+				pj = 3
+			} else {
+				res = 2
+				pj = 3
+			}
+		case "ABC":
+			if info.LenPN > 14 && info.PCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.LenPN > 10 || info.PCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BAC":
+			if info.PNBH > 14 && info.LenPC > 8 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 2
+			} else if info.PNBH > 10 || info.LenPC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "BBC":
+			if info.PNBH > 14 && info.PCBH > 10 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+				pj = 1
+			} else if info.PNBH > 10 || info.PCBH > 8 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "ACC":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPN > 16 {
+					res = 2
+					pj = 1
+				} else {
+					res = 1
+					pj = 2
+				}
+			}
+		case "ACD":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPN > 16 {
+
+					res = 2
+				} else {
+					res = 1
+
+				}
+				pj = 1
+			}
+
+		case "ADC":
+			if info.LenPN > 16 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "BCC":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PNBH > 12 {
+					res = 1
+				}
+				pj = 1
+			}
+		case "BCD":
+			//项目编号不一致
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PNBH > 8 {
+					res = 1
+				}
+				pj = 1
+			}
+		case "BDC":
+			if info.PNBH > 7 {
+				res = 1
+			}
+			pj = 1
+		case "CAA":
+			if info.LenPC > 12 || info.LenPTC > 12 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 8 || info.LenPTC > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "CAB":
+			if info.LenPC > 12 && info.PTCBH > 8 {
+				res = 3
+				pj = 2
+			} else if info.LenPC > 12 || info.PTCBH > 8 {
+				res = 2
+				pj = 3
+			} else {
+				res = 1
+				pj = 3
+			}
+		case "CAC":
+			if info.LenPC > 9 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 1
+			}
+		case "CAD":
+			if info.LenPC > 9 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "CBA":
+			if info.LenPTC > 14 && info.PCBH > 12 {
+				res = 3
+				pj = 2
+			} else if info.LenPTC > 12 || info.PCBH > 10 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "CBB":
+			if info.PCBH > 13 && info.PTCBH > 13 {
+				res = 3
+				pj = 1
+			} else if info.PCBH > 9 || info.PTCBH > 9 {
+				res = 2
+				pj = 2
+			} else {
+				res = 1
+				pj = 2
+			}
+		case "CBC":
+			if info.PCBH > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else if info.PCBH > 5 {
+				res = 1
+			}
+			pj = 1
+		case "CBD":
+			if info.PCBH > 14 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 2
+			} else if info.PCBH > 5 {
+				res = 1
+			}
+			pj = 1
+		case "CCA":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPTC > 12 && !StrOrNum2.MatchString(info.PTC) {
+					res = 2
+				} else if info.LenPTC > 5 {
+					res = 1
+				}
+				pj = 1
+			}
+		case "CCB":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
+					res = 1
+				}
+				pj = 1
+			}
+		case "CDA":
+			if info.LenPTC > 12 && !StrOrNum2.MatchString(info.PTC) {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "CDB":
+			if info.PTCBH > 10 && !StrOrNum2.MatchString(info.PTC) {
+				res = 1
+				pj = 1
+			}
+		case "DAC":
+			if info.LenPC > 13 && !StrOrNum2.MatchString(info.ProjectCode) {
+				res = 3
+			} else if info.LenPC > 8 {
+				res = 2
+			} else {
+				res = 1
+			}
+			pj = 1
+		case "DBC":
+			if info.PCBH > 8 {
+				res = 1
+			}
+			pj = 1
+		case "DCA":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.LenPTC > 10 {
+					res = 2
+				} else {
+					res = 1
+				}
+				pj = 1
+			}
+		case "DCB":
+			if info.LenPC != len([]rune(compareInfo.ProjectCode)) {
+				if info.PTCBH > 8 && !StrOrNum2.MatchString(info.PTC) {
+					res = 1
+				}
+				pj = 1
+			}
+		}
+
+	}
+	return
+}

medical_project/project.go

@@ -0,0 +1,1213 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.uber.org/zap"
+	"math"
+	"sort"
+	"strings"
+	"time"
+)
+
+//从对应map中获取对比的项目id
+func getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
+	//	p.ConCurrentLock(n1, n2, n3, n4)
+	//	defer p.ConCurrentUnLock(n1, n2, n3, n4)
+	wg.Add(1)
+	//查找到id数组
+	res = []*Key{}
+	//是否查找到，并标识位置。-1代表值为空或已经存在。
+	bpn, bpc, bptc, bpb = -1, -1, -1, -1
+	if pn != "" {
+		ids := mapPn[pn]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			mapPn[pn] = ids
+			bpn = 0
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if pc != "" {
+		ids := mapPc[pc]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			mapPc[pc] = ids
+			bpc = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if ptc != "" {
+		ids := mapPc[ptc]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			mapPc[ptc] = ids
+			bptc = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+
+	if pb != "" {
+		ids := mapPb[pb]
+		if ids == nil {
+			ids = &Key{Arr: []string{}}
+			mapPb[pb] = ids
+			bpb = len(res)
+		}
+		ids.Lock.Lock()
+		res = append(res, ids)
+	}
+	repeatId := map[string]bool{}
+	idArr = []string{} //项目id
+	IDArr = []*ID{}    //项目信息
+	for _, m := range res {
+		for _, id := range m.Arr {
+			if !repeatId[id] {
+				repeatId[id] = true
+				//_, _ = strconv.ParseInt(id[0:8], 16, 64)
+				AllIdsMapLock.Lock()
+				Id := AllIdsMap[id]
+				AllIdsMapLock.Unlock()
+				if Id != nil {
+					Id.Lock.Lock()
+					idArr = append(idArr, id)
+					IDArr = append(IDArr, Id)
+				}
+			}
+		}
+	}
+	return
+}
+
+func startProjectMerge(info *Info, tmp map[string]interface{}) {
+	findLock.Lock()
+	defer findLock.Unlock()
+
+	//只有或没有采购单位的无法合并
+	//bpn, bpc, bptc, bpb 是否查找到，并标识位置。-1代表未查找到。
+	//pids 是项目id数组集合
+	//IDArr，是单个项目ID对象集合
+	bpn, bpc, bptc, bpb, pids, _, IDArr := getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer) //, info.LenPN, info.LenPC, info.LenPTC, len([]rune(info.Buyer)))
+	defer wg.Done()
+	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock
+
+	for _, m := range pids {
+		defer m.Lock.Unlock()
+	}
+	for _, id := range IDArr {
+		defer id.Lock.Unlock()
+	}
+
+	bFindProject := false
+	findPid := ""
+	//获取完id，进行计算
+	//定义两组
+	comRes1 := []*Project{} //优先级最高的对比结果数组
+	comRes2 := []*Project{} //优化级其次
+	comRes3 := []*Project{}
+	for _, v := range IDArr {
+		comStr := ""
+		compareProject := v.P
+		compareProject.score = 0
+		//问题出地LastTime!!!!!
+		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
+		if diffTime <= validTime {
+			//代理机构完全不相同，直接新建项目
+			if CheckContain(compareProject.Agency, info.Agency) == 3 {
+				continue
+			}
+			//地区(省、市、区)不同，直接新建项目
+			if ComparePlace(compareProject, info) {
+				continue
+			}
+
+			info.PNBH = 0
+			info.PCBH = 0
+			info.PTCBH = 0
+			compareStr, score := comparePNC(info, compareProject)
+
+			resVal, pjVal := Select(compareStr, info, compareProject)
+			//---------------------------------------
+			if resVal > 0 {
+				compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount, score2 := compareBCTABB(info, compareProject, diffTime, score)
+
+				//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
+				comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
+				compareProject.comStr = comStr
+				compareProject.pjVal = pjVal
+				compareProject.resVal = resVal
+				//log.Println(compareProject.comStr)
+				eqV := compareResult(resVal, pjVal, score2, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount)
+				if eqV == 1 {
+					comRes1 = append(comRes1, compareProject)
+				} else if eqV == 2 {
+					comRes2 = append(comRes2, compareProject)
+				} else if eqV == 3 {
+					comRes3 = append(comRes3, compareProject)
+				}
+			}
+		}
+	}
+	//--------------------------------对比完成-----------------------
+
+	//更新数组、更新项目
+	for kv, resN := range [][]*Project{comRes1, comRes2, comRes3} {
+		if len(resN) > 0 {
+			if len(resN) > 1 {
+				sort.Slice(resN, func(i, j int) bool {
+					return resN[i].score > resN[j].score
+				})
+			}
+
+			ex := 0
+			resArr := []*Project{}
+			for i, res := range resN {
+				choose, e := CompareStatus(resN[i], info)
+				if !choose {
+					ex = e
+					resArr = append(resArr, res)
+				}
+			}
+			if len(resArr) > 0 {
+				bFindProject = true
+				findPid = resArr[0].Id.Hex()
+				UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr, ex)
+				for k2, bv := range []int{bpn, bpc, bptc, bpb} {
+					if bv > -1 {
+						pids[bv].Arr = append(pids[bv].Arr, findPid)
+						if k2 == 0 {
+							if resArr[0].ProjectName == "" {
+								resArr[0].ProjectName = info.ProjectName
+							} else {
+								if resArr[0].MPN == nil {
+									resArr[0].MPN = []string{info.ProjectName}
+								} else {
+									resArr[0].MPN = append(resArr[0].MPN, info.ProjectName)
+								}
+							}
+						} else if k2 < 3 {
+							if resArr[0].ProjectCode == "" {
+								resArr[0].ProjectCode = util.If(k2 == 1, info.ProjectCode, info.PTC).(string)
+							} else {
+								if resArr[0].MPC == nil {
+									resArr[0].MPC = []string{util.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
+								} else {
+									resArr[0].MPC = append(resArr[0].MPC, util.If(k2 == 1, info.ProjectCode, info.PTC).(string))
+								}
+							}
+						} else {
+							if resArr[0].Buyer == "" {
+								resArr[0].Buyer = info.Buyer
+							}
+						}
+					}
+				}
+			} else {
+				bFindProject = false
+				findPid = ""
+			}
+			break
+		}
+	}
+
+	if !bFindProject {
+		id, p1 := NewProject(tmp, info)
+		AllIdsMapLock.Lock()
+		AllIdsMap[id] = &ID{Id: id, P: p1}
+		AllIdsMapLock.Unlock()
+		for _, m := range pids {
+			m.Arr = append(m.Arr, id)
+		}
+	}
+}
+
+func compareBCTABB(info *Info, cp *Project, diffTime int64, score int) (compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string, score2 int) {
+	compareBuyer = "D"
+	if len([]rune(info.Buyer)) > 3 && len([]rune(cp.Buyer)) > 3 {
+		v := CheckContain(info.Buyer, cp.Buyer)
+		if v == 1 {
+			compareBuyer = "A"
+			score += 3
+		} else {
+			//v1 := CosineSimilar(info.Buyer, cp.Buyer)
+			if v == 2 {
+				compareBuyer = "B"
+				score += 1
+			} else {
+				compareBuyer = "C"
+			}
+		}
+	}
+	//---------------------------------------
+
+	compareCity = ""
+	if info.Area != "全国" && info.Area != "" && info.Area == cp.Area {
+		compareCity += "A"
+		score += 2
+	} else if info.Area == "全国" || cp.Area == "全国" {
+		compareCity += "B"
+		score += 1
+	} else {
+		compareCity += "C"
+	}
+	if compareCity != "C" {
+		if info.City != "" && info.City == cp.City {
+			compareCity += "A"
+			score += 2
+		} else {
+			if info.Area == "全国" || cp.Area == "全国" {
+				compareCity += "B"
+			} else if info.City == compareCity {
+				compareCity += "B"
+			} else {
+				compareCity += "C"
+			}
+		}
+	} else {
+		compareCity += "C"
+	}
+	score2 = 0
+	if compareCity == "AA" {
+		if info.District != "" && info.District == cp.District {
+			score2 = 1
+		}
+	}
+
+	compareTime = "D"
+	if diffTime < 45*86400 {
+		compareTime = "A"
+		score += 2
+	} else if diffTime < 90*86400 {
+		compareTime = "B"
+		score += 1
+	}
+
+	compareAgency = "D"
+	if info.Agency != "" {
+		if info.Agency == cp.Agency {
+			compareAgency = "A"
+			score += 2
+			score2 += 1
+		} else if cp.Agency != "" {
+			if strings.Contains(info.Agency, cp.Agency) || strings.Contains(cp.Agency, info.Agency) {
+				compareAgency = "B"
+				score += 1
+				score2 += 1
+			} else {
+				compareAgency = "C"
+			}
+		}
+	}
+	compareBudget = "C"
+	if info.Budget > 0 && (info.Budget == cp.Budget || (cp.Bidamount > 0 && info.Budget > cp.Bidamount && (info.Budget-cp.Bidamount) < (0.15*info.Budget))) {
+		compareBudget = "A"
+		score += 1
+		score2 += 1
+	}
+	//	else if info.Budget == 0 && cp.Budget == 0 {
+	//		compareBudget = "B"
+	//	}
+	compareBidmount = "C"
+	if info.Bidamount > 0 && (info.Bidamount == cp.Bidamount || (cp.Budget > 0 && cp.Budget > info.Bidamount && (cp.Budget-info.Bidamount) < 0.15*cp.Budget)) {
+		compareBidmount = "A"
+		score += 1
+		score2 += 1
+	}
+	//	else if info.Bidamount == 0 && cp.Bidamount == 0 {
+	//		compareBidmount = "B"
+	//	}
+
+	cp.score = score
+	return
+}
+
+func compareResult(resVal, pjVal, score2 int, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string) int {
+	eqV := 0
+	switch resVal {
+	case 3:
+		if pjVal == 3 && comStr[3:] != "CCCDCCC" {
+			eqV = 1
+		} else if compareBuyer < "C" {
+			if pjVal > 1 {
+				eqV = 1
+			} else { //if (compareCity[1:1] != "C" || compareTime != "D") && score2 > 0
+				eqV = 2
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && (compareCity[1:1] != "C" || score2 > 0) {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+		} else {
+			if pjVal == 3 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if pjVal == 2 && compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			} else if compareCity == "AA" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+		}
+	case 2:
+		if compareBuyer < "C" {
+			if pjVal > 1 {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" || score2 > 0 {
+				eqV = 3
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+
+		} else {
+			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		}
+	case 1:
+		if compareBuyer < "C" {
+			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		} else {
+			if pjVal > 1 && compareTime == "A" && score2 > 0 && (compareBudget == "A" || compareBidmount == "A") && compareCity[1:1] != "C" {
+				eqV = 3
+			}
+		}
+	}
+	return eqV
+}
+
+func ComparePlace(project *Project, info *Info) bool {
+	if info.Area == "全国" || info.Area == "" {
+		return false
+	}
+	if project.Area == "全国" || project.Area == "" {
+		return false
+	}
+	if info.Area == project.Area {
+		if info.City == "" || project.City == "" {
+			return false
+		} else if info.City == project.City {
+			return false
+		}
+	} else {
+		return true
+	}
+	return true
+}
+
+func CompareStatus(project *Project, info *Info) (bool, int) {
+	if info.TopType == "拟建" || info.TopType == "预告" || info.TopType == "招标" {
+		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
+			return false, 0
+		} else if project.Bidstatus == "废标" || project.Bidstatus == "流标" {
+			return false, 0
+		} else {
+			return true, 0
+		}
+	} else if info.TopType == "结果" {
+		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
+			return false, 0
+		} else if project.Bidstatus == info.SubType {
+			//状态一样，根据发布时间判断是否合并
+			if (info.Publishtime - project.LastTime) > statusTime {
+				return true, 0
+			} else {
+				return false, 0
+			}
+		} else if project.Bidstatus == "成交" && info.SubType == "中标" {
+			return true, 0
+		} else if project.Bidstatus == "流标" || project.Bidstatus == "废标" {
+			return false, 2
+		} else if project.Bidstatus == "合同" || project.Bidstatus == "其它" {
+			return false, 3
+		} else {
+			return false, 0
+		}
+	} else {
+		return false, 0
+	}
+}
+
+func NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidtype, bidstatus string) Project {
+	p1 := Project{
+		Id:          id,
+		Ids:         []string{thisinfo.Id},
+		ProjectName: thisinfo.ProjectName,
+		ProjectCode: thisinfo.ProjectCode,
+		Buyer:       thisinfo.Buyer,
+		Agency:      thisinfo.Agency,
+		Area:        thisinfo.Area,
+		City:        thisinfo.City,
+		District:    thisinfo.District,
+		MPN:         []string{},
+		MPC:         []string{},
+		FirstTime:   thisinfo.Publishtime,
+		LastTime:    thisinfo.Publishtime,
+		Budget:      thisinfo.Budget,
+		Bidamount:   thisinfo.Bidamount,
+		Bidstatus:   bidstatus,
+		Bidtype:     bidtype,
+	}
+	if thisinfo.LenPTC > 5 {
+		p1.MPC = append(p1.MPC, thisinfo.PTC)
+	}
+	return p1
+}
+
+func NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *Project) {
+	pId := mongodb.StringTOBsonId(thisinfo.Id)
+	set := map[string]interface{}{}
+	set["_id"] = pId
+	for _, f := range FIELDS {
+		if tmp[f] != nil && tmp[f] != "" {
+			set[f] = tmp[f]
+		}
+	}
+	bidopentime := util.Int64All(tmp["bidopentime"])
+	if bidopentime > 0 {
+		set["bidopentime"] = bidopentime
+	}
+	//异常标记
+	if thisinfo.TopType != "招标" && thisinfo.TopType != "拟建" && thisinfo.TopType != "预告" {
+		set["exception"] = 1
+	}
+	//合同编号
+	if thisinfo.ContractCode != "" {
+		set["contractcode"] = thisinfo.ContractCode
+	}
+
+	bt := util.ObjToString(tmp["toptype"])
+	bs := util.ObjToString(tmp["subtype"])
+	mapBidLock.Lock()
+	if thisinfo.Infoformat == 2 || thisinfo.SubType == "拟建" {
+		set["bidstatus"] = "拟建"
+		bt = "拟建"
+	} else {
+		if bidtype[bs] != "" {
+			set["bidtype"] = bidtype[bs]
+		} else {
+			set["bidtype"] = "招标"
+		}
+		if bt == "招标" {
+			set["projectscope"] = util.ObjToString(tmp["projectscope"])
+			set["bidstatus"] = bt
+		} else {
+			if bidstatus[bs] != "" {
+				set["bidstatus"] = thisinfo.SubType
+				bt = thisinfo.SubType
+			} else if bs == "" {
+				set["bidstatus"] = ""
+				bt = ""
+			} else {
+				set["bidstatus"] = "其它"
+				bt = "其它"
+			}
+		}
+	}
+	mapBidLock.Unlock()
+
+	pkg := PackageFormat(thisinfo, nil)
+	p1 := NewCachePinfo(pId, thisinfo, bs, bt)
+
+	now := time.Now().Unix()
+	set["createtime"] = now
+	//set["sourceinfoid"] = thisinfo.Id
+	set["firsttime"] = tmp["publishtime"]
+	set["lasttime"] = tmp["publishtime"]
+	//增量用系统时间，全量(历史)入库时间
+	if currentType == "project" {
+		set["pici"] = pici
+	} else {
+		set["pici"] = tmp["comeintime"]
+	}
+	set["ids"] = []string{thisinfo.Id}
+	if thisinfo.TopType == "招标" {
+		//if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
+		set["zbtime"] = tmp["publishtime"]
+		p1.Zbtime = util.Int64All(tmp["publishtime"])
+		//}
+	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
+		set["jgtime"] = tmp["publishtime"]
+		p1.Jgtime = thisinfo.Publishtime
+	}
+	if len(thisinfo.Subscopeclass) > 0 {
+		set["s_subscopeclass"] = strings.Join(thisinfo.Subscopeclass, ",")
+	}
+	if len(thisinfo.Winners) > 0 {
+		set["s_winner"] = strings.Join(thisinfo.Winners, ",")
+		p1.Winners = strings.Join(thisinfo.Winners, ",")
+
+		if util.ObjToString(tmp["winnertel"]) != "" {
+			set["winnertel"] = tmp["winnertel"]
+		}
+		if util.ObjToString(tmp["winnerperson"]) != "" {
+			set["winnerperson"] = tmp["winnerperson"]
+		}
+	}
+	if thisinfo.HasPackage {
+		set["multipackage"] = 1
+		set["package"] = pkg
+	} else {
+		set["multipackage"] = 0
+	}
+	//项目评审专家
+	if len(thisinfo.ReviewExperts) > 0 {
+		set["review_experts"] = thisinfo.ReviewExperts
+		p1.ReviewExperts = thisinfo.ReviewExperts
+	}
+	//标的物
+	if thisinfo.Purchasing != "" {
+		list := Duplicate(strings.Split(thisinfo.Purchasing, ","))
+		p := strings.Join(util.ObjArrToStringArr(list), ",")
+		set["purchasing"] = p
+		p1.Purchasing = p
+	}
+	if tmp["purchasinglist"] != nil && len(util.ObjArrToMapArr(tmp["purchasinglist"].([]interface{}))) > 0 {
+		if thisinfo.TopType == "招标" || thisinfo.TopType == "预告" {
+			set["zb_plist"] = tmp["purchasinglist"]
+		} else {
+			set["jg_plist"] = tmp["purchasinglist"]
+		}
+	}
+	set["sourceinfoid"] = thisinfo.Id
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, util.ObjToString(v["entname"])) == -1 {
+				list = append(list, util.ObjToString(v["entname"]))
+			}
+		}
+		set["winnerorder"] = list
+		p1.Winnerorder = list
+	}
+	p1.InfoFiled = make(map[string]InfoField)
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+	p1.InfoFiled[thisinfo.Id] = infofield
+	res := StructToMap(infofield)
+	set["infofield"] = map[string]interface{}{
+		thisinfo.Id: res,
+	}
+	if tmp["budget"] != nil && tmp["budget"] != "" {
+		set["budget"] = thisinfo.Budget
+		p1.Budgettag = 0
+		set["budgettag"] = 0
+	} else {
+		p1.Budgettag = 1
+		set["budgettag"] = 1
+	}
+	if tmp["bidamount"] != nil && tmp["bidamount"] != "" {
+		set["bidamount"] = thisinfo.Bidamount
+		p1.Bidamounttag = 0
+		set["bidamounttag"] = 0
+	} else {
+		p1.Bidamounttag = 1
+		set["bidamounttag"] = 1
+	}
+	if p1.Bidamount > 0 {
+		set["sortprice"] = p1.Bidamount
+	} else {
+		if p1.Budget > 0 {
+			set["sortprice"] = p1.Budget
+		}
+	}
+	push := PushListInfo(tmp, thisinfo.Id)
+	push["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	set["list"] = []bson.M{
+		push,
+	}
+	updatePool <- []map[string]interface{}{
+		{
+			"_id": pId,
+		},
+		{
+			"$set": set,
+		},
+	}
+	return pId.Hex(), &p1
+}
+
+//项目中list的信息
+func PushListInfo(tmp map[string]interface{}, infoid string) bson.M {
+	res := bson.M{
+		"infoid": infoid,
+	}
+	for _, k := range INFOFIELDS {
+		if tmp[k] != nil {
+			res[k] = tmp[k]
+		}
+	}
+	return res
+}
+
+func PackageFormat(info *Info, project *Project) map[string]interface{} {
+	p1 := map[string]interface{}{}
+	if project != nil && project.Package != nil && len(project.Package) > 0 {
+		packageCopy := Copy(project.Package).(map[string]interface{})
+		p1 = packageCopy
+		for k, v := range info.Package {
+			if v1, ok := v.(map[string]interface{}); ok {
+				v2 := map[string]interface{}{}
+				v2 = packageEle(v1, info.Id)
+				if v2["bidstatus"] == nil {
+					v2["bidstatus"] = info.SubType
+				}
+				addFlag := false
+				for k1, v3 := range p1 {
+					if v4, ok := v3.([]map[string]interface{}); ok {
+						//if qu.ObjToString(v4[0]["origin"]) == qu.ObjToString(v2["origin"]) && qu.ObjToString(v4[0]["name"]) == qu.ObjToString(v2["name"]) {
+						if k1 == k {
+							v4 = append(v4, v2)
+							p1[k1] = v4
+							addFlag = true
+							break
+						}
+					}
+				}
+				if !addFlag {
+					p1[k] = []map[string]interface{}{v2}
+				}
+			}
+		}
+	} else {
+		for k, v := range info.Package {
+			v1, _ := v.(map[string]interface{})
+			p2 := map[string]interface{}{}
+			p2 = packageEle(v1, info.Id)
+			if p2["bidstatus"] == nil {
+				p2["bidstatus"] = info.SubType
+			}
+			p1[k] = []map[string]interface{}{p2}
+		}
+	}
+	return p1
+}
+
+func packageEle(map1 map[string]interface{}, id string) map[string]interface{} {
+	p2 := map[string]interface{}{}
+	for _, k := range PackageEle {
+		if map1[k] != nil {
+			p2[k] = map1[k]
+		}
+		infoid := p2["infoid"]
+		if infoid == nil {
+			p2["infoid"] = id
+		}
+	}
+	return p2
+}
+
+func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *Project, weight int, comStr string, ex int) {
+	if currentType != "updateInfo" {
+		if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
+			log.Info("repeat", zap.String("info", thisinfo.Id), zap.Any("pid", pInfo.Id))
+			return
+		}
+	}
+
+	set := map[string]interface{}{}
+	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
+	//zbtime、lasttime、jgtime
+	pInfo.LastTime = thisinfo.Publishtime
+	set["lasttime"] = thisinfo.Publishtime
+	if thisinfo.TopType == "招标" {
+		if pInfo.Zbtime <= 0 {
+			pInfo.Zbtime = thisinfo.Publishtime
+			set["zbtime"] = tmp["publishtime"]
+		}
+		if pInfo.Jgtime > 0 {
+			pInfo.Jgtime = int64(0)
+			set["jgtime"] = int64(0)
+		}
+	} else if thisinfo.TopType == "结果" {
+		if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+			if pInfo.Jgtime > 0 {
+				//jg1 := int64(math.Abs(float64(pInfo.Jgtime - thisinfo.Publishtime)))
+				//公告状态和项目状态同样都是中标或者成交，
+				if thisinfo.SubType == "成交" && pInfo.Bidstatus == "中标" {
+					if jgTime < thisinfo.Publishtime {
+						pInfo.Jgtime = thisinfo.Publishtime
+						set["jgtime"] = tmp["publishtime"]
+					}
+					//公告状态和项目状态同样是流标或者废标
+				} else if (thisinfo.SubType == "流标" || thisinfo.SubType == "废标") && (pInfo.Bidstatus == "流标" || pInfo.Bidstatus == "废标") {
+					if jgTime < thisinfo.Publishtime {
+						pInfo.Jgtime = thisinfo.Publishtime
+						set["jgtime"] = tmp["publishtime"]
+					}
+				}
+			} else {
+				pInfo.Jgtime = thisinfo.Publishtime
+				set["jgtime"] = tmp["publishtime"]
+			}
+		}
+	} else if thisinfo.SubType == "合同" {
+		if pInfo.Bidstatus == "中标" || pInfo.Bidstatus == "成交" || pInfo.Bidstatus == "" {
+			//中标、成交不更新jgtime
+		} else {
+			pInfo.Jgtime = thisinfo.Publishtime
+			set["jgtime"] = tmp["publishtime"]
+		}
+	}
+	if thisinfo.Bidopentime > pInfo.Bidopentime {
+		pInfo.Bidopentime = thisinfo.Bidopentime
+		set["bidopentime"] = pInfo.Bidopentime
+	}
+
+	bt := util.ObjToString(tmp["toptype"])
+	bs := util.ObjToString(tmp["subtype"])
+	mapBidLock.Lock()
+	if bt == "招标" {
+		//招标状态，更新projectscope
+		if tmp["projectscope"] != nil {
+			set["projectscope"] = util.ObjToString(tmp["projectscope"])
+		}
+		set["bidstatus"] = bt
+		pInfo.Bidstatus = bt
+		if bidtype[bs] != "" {
+			set["bidtype"] = bidtype[bs]
+			pInfo.Bidtype = bidtype[bs]
+		} else {
+			set["bidtype"] = "招标"
+			pInfo.Bidtype = "招标"
+		}
+	} else {
+		if bidstatus[bs] != "" {
+			set["bidstatus"] = thisinfo.SubType
+			pInfo.Bidstatus = thisinfo.SubType
+		} else if thisinfo.Infoformat == 2 {
+			set["bidstatus"] = "拟建"
+			pInfo.Bidstatus = "拟建"
+		} else if bs == "" && bt == "结果" {
+			if pInfo.Bidstatus == "招标" {
+				set["bidstatus"] = ""
+				pInfo.Bidstatus = ""
+			}
+		} else {
+			set["bidstatus"] = "其它"
+			pInfo.Bidstatus = "其它"
+		}
+	}
+	mapBidLock.Unlock()
+
+	//相同城市的公告才会合并到一起(全国列外)
+	if thisinfo.Area != "全国" {
+		pInfo.Area = thisinfo.Area
+		set["area"] = thisinfo.Area
+		if thisinfo.City != "" {
+			pInfo.City = thisinfo.City
+			set["city"] = thisinfo.City
+		}
+		if thisinfo.District != "" {
+			pInfo.District = thisinfo.District
+			set["district"] = thisinfo.District
+		}
+	}
+	//项目名称
+	//if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+	if pInfo.ProjectName == "" && thisinfo.ProjectName != "" {
+		pInfo.ProjectName = thisinfo.ProjectName
+		set["projectname"] = thisinfo.ProjectName
+	}
+	//7--项目编号
+	//if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+	if pInfo.ProjectCode == "" && thisinfo.ProjectCode != "" {
+		pInfo.ProjectCode = thisinfo.ProjectCode
+		set["projectcode"] = thisinfo.ProjectCode
+	}
+	//7--采购单位
+	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
+		pInfo.Buyer = thisinfo.Buyer
+		set["buyer"] = thisinfo.Buyer
+
+		pInfo.Buyerclass = thisinfo.Buyerclass
+		set["buyerclass"] = thisinfo.Buyerclass
+	}
+	if pInfo.Buyer == "" {
+		pInfo.Buyerclass = ""
+		set["buyerclass"] = ""
+	}
+	//采购单位联系人
+	if thisinfo.Buyerperson != "" {
+		pInfo.Buyerperson = thisinfo.Buyerperson
+		set["buyerperson"] = thisinfo.Buyerperson
+	} else {
+		pInfo.Buyerperson = ""
+		set["buyerperson"] = ""
+	}
+	//采购单位電話
+	if thisinfo.Buyertel != "" {
+		pInfo.Buyertel = thisinfo.Buyertel
+		set["buyertel"] = pInfo.Buyertel
+	} else {
+		pInfo.Buyertel = ""
+		set["buyertel"] = ""
+	}
+	if thisinfo.ContractCode != "" {
+		if pInfo.ContractCode == "" {
+			set["contractcode"] = thisinfo.ContractCode
+		} else {
+			list := strings.Split(pInfo.ContractCode, ",")
+			if BinarySearch(list, thisinfo.ContractCode) == -1 {
+				list = append(list, thisinfo.ContractCode)
+				sort.Strings(list)
+			}
+			set["contractcode"] = strings.Join(list, ",")
+		}
+	}
+
+	//8--代理机构
+	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
+		pInfo.Agency = thisinfo.Agency
+		set["agency"] = thisinfo.Agency
+	}
+
+	if len(thisinfo.Topscopeclass) > 0 {
+		sort.Strings(pInfo.Topscopeclass)
+		for _, k := range thisinfo.Topscopeclass {
+			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
+				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
+				sort.Strings(pInfo.Topscopeclass)
+			}
+		}
+		set["topscopeclass"] = pInfo.Topscopeclass
+	}
+
+	if thisinfo.Purchasing != "" {
+		if pInfo.Purchasing == "" {
+			list := Duplicate(strings.Split(thisinfo.Purchasing, ",")) //标的物 去重 03/03
+			p := strings.Join(util.ObjArrToStringArr(list), ",")
+			pInfo.Purchasing = p
+			set["purchasing"] = p
+		} else {
+			list := strings.Split(pInfo.Purchasing, ",")
+			list = util.ObjArrToStringArr(Duplicate(list))
+			for _, k := range list {
+				if BinarySearch(strings.Split(thisinfo.Purchasing, ","), k) == -1 {
+					list = append(list, k)
+					sort.Strings(list)
+				}
+			}
+			set["purchasing"] = strings.Join(list, ",")
+		}
+	}
+	// 标的物
+	if tmp["purchasinglist"] != nil && len(util.ObjArrToMapArr(tmp["purchasinglist"].([]interface{}))) > 0 {
+		if thisinfo.TopType == "招标" || thisinfo.TopType == "预告" {
+			set["zb_plist"] = tmp["purchasinglist"]
+		} else {
+			set["jg_plist"] = tmp["purchasinglist"]
+		}
+	}
+	set["sourceinfoid"] = thisinfo.Id
+	//中标候选人
+	if len(thisinfo.WinnerOrder) > 0 {
+		var list = []string{}
+		for _, v := range thisinfo.WinnerOrder {
+			if BinarySearch(list, util.ObjToString(v["entname"])) == -1 {
+				list = append(list, util.ObjToString(v["entname"]))
+			}
+		}
+		set["winnerorder"] = list
+		pInfo.Winnerorder = list
+	}
+
+	if len(thisinfo.Subscopeclass) > 0 {
+		sort.Strings(pInfo.Subscopeclass)
+		for _, k := range thisinfo.Subscopeclass {
+			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
+				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
+				sort.Strings(pInfo.Subscopeclass)
+			}
+		}
+		set["subscopeclass"] = pInfo.Subscopeclass
+		set["s_subscopeclass"] = strings.Join(pInfo.Subscopeclass, ",")
+	}
+
+	if len(thisinfo.Winners) > 0 {
+		var winners []string
+		if pInfo.Winners == "" {
+			set["winner"] = util.ObjToString(tmp["winner"])
+			for _, k := range thisinfo.Winners {
+				if k == "" {
+					continue
+				}
+				if BinarySearch(winners, k) == -1 {
+					winners = append(winners, k)
+				}
+			}
+		} else {
+			winners = strings.Split(pInfo.Winners, ",")
+			for _, k := range thisinfo.Winners {
+				if k == "" {
+					continue
+				}
+				if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+					if BinarySearch(winners, k) != -1 {
+						deleteSlice(winners, k, "")
+					}
+				} else {
+					if BinarySearch(winners, k) == -1 {
+						winners = append(winners, k)
+					}
+				}
+			}
+		}
+		set["s_winner"] = strings.Join(util.ObjArrToStringArr(Duplicate(winners)), ",")
+		if util.ObjToString(tmp["winnertel"]) != "" {
+			set["winnertel"] = tmp["winnertel"]
+		}
+		if util.ObjToString(tmp["winnerperson"]) != "" {
+			set["winnerperson"] = tmp["winnerperson"]
+		}
+	}
+
+	if thisinfo.HasPackage { //多包处理
+		set["multipackage"] = 1
+		pkg := PackageFormat(thisinfo, pInfo)
+		pInfo.Package = pkg
+		set["package"] = pInfo.Package
+	}
+	//处理多包后，计算预算金额、中标金额
+	CountAmount(pInfo, thisinfo, tmp)
+	if pInfo.Budget >= 0 && pInfo.Budgettag != 1 {
+		pInfo.Budget = pInfo.Budget
+		set["budget"] = pInfo.Budget
+		set["budgettag"] = 0
+	}
+	if pInfo.Bidamount >= 0 && pInfo.Bidamounttag != 1 {
+		pInfo.Bidamount = pInfo.Bidamount
+		set["bidamount"] = pInfo.Bidamount
+		set["bidamounttag"] = 0
+	}
+	if pInfo.Bidamount > 0 {
+		set["sortprice"] = pInfo.Bidamount
+	} else {
+		if pInfo.Budget > 0 {
+			set["sortprice"] = pInfo.Budget
+		}
+	}
+
+	infofield := InfoField{
+		Budget:       thisinfo.Budget,
+		Bidamount:    thisinfo.Bidamount,
+		ContractCode: thisinfo.ContractCode,
+		ProjectName:  thisinfo.ProjectName,
+		ProjectCode:  thisinfo.ProjectCode,
+		Bidstatus:    bs,
+	}
+	copyMap := Copy(pInfo.InfoFiled).(map[string]InfoField)
+	copyMap[thisinfo.Id] = infofield
+	tmpMap := make(map[string]interface{})
+	for k, v := range copyMap {
+		tmpMap[k] = StructToMap(v)
+	}
+	tmpMap[thisinfo.Id] = StructToMap(infofield)
+	pInfo.InfoFiled = copyMap
+	set["infofield"] = tmpMap
+
+	if currentType == "project" {
+		set["pici"] = pici
+	} else {
+		set["pici"] = tmp["comeintime"]
+	}
+	update := map[string]interface{}{}
+	if len(set) > 0 {
+		update["$set"] = set
+	}
+	push := PushListInfo(tmp, thisinfo.Id)
+	push["s_winner"] = strings.Join(thisinfo.Winners, ",")
+	push["compareStr"] = comStr
+	push["resVal"] = pInfo.resVal
+	push["pjVal"] = pInfo.pjVal
+	update["$push"] = map[string]interface{}{
+		"list": push,
+		"ids":  thisinfo.Id,
+	}
+
+	if len(update) > 0 {
+		updateInfo := []map[string]interface{}{
+			{
+				"_id": pInfo.Id,
+			},
+			update,
+		}
+		updatePool <- updateInfo
+	}
+}
+
+// 计算预算(budget)、中标金额(bidamount)
+func CountAmount(project *Project, info *Info, tmp map[string]interface{}) {
+	if info.HasPackage {
+		budget := 0.0
+		for _, v := range project.Package {
+			v1, _ := v.([]map[string]interface{})
+			for _, v2 := range v1 {
+				if v2["budget"] != nil {
+					b1 := util.Float64All(v2["budget"])
+					if b1 > 0 {
+						budget = budget + b1
+						break
+					}
+				} else {
+					project.Budgettag = 1
+				}
+			}
+		}
+		if budget > 0 {
+			project.Budget = budget
+			project.Budgettag = 0
+		} else if budget == 0 && info.Budget > 0 {
+			project.Budget = info.Budget
+			project.Budgettag = 0
+		}
+	} else {
+		//招标没有多包
+		k := KeyPackage.FindStringSubmatch(info.ProjectName)
+		if len(k) > 0 {
+			//招标是单包
+			if len(project.Package) > 0 {
+				//项目有多包
+				flag := false
+				for _, v := range project.Package {
+					v1, _ := v.([]map[string]interface{})
+					if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
+						flag = true
+					}
+				}
+				if !flag {
+					project.Budget = project.Budget + info.Budget
+					project.Budgettag = 0
+				}
+			} else {
+				//项目没有多包
+				if info.Budget > 0 {
+					project.Budget = project.Budget + info.Budget
+					project.Budgettag = 0
+				} else if info.Budget == 0 && tmp["budget"] != nil {
+					project.Budgettag = 0
+				}
+			}
+		} else {
+			if info.Budget > 0 && project.Budget < info.Budget {
+				project.Budget = info.Budget
+				project.Budgettag = 0
+			}
+		}
+	}
+	if info.SubType == "中标" || info.SubType == "成交" || info.SubType == "合同" {
+		if info.HasPackage {
+			bidamount := 0.0
+			for _, v := range project.Package {
+				v1, _ := v.([]map[string]interface{})
+				for _, v2 := range v1 {
+					b1 := util.Float64All(v2["bidamount"])
+					if b1 > 0 {
+						bidamount = bidamount + b1
+						break
+					}
+				}
+			}
+			if bidamount > 0 {
+				project.Bidamount = bidamount
+				project.Bidamounttag = 0
+			} else if bidamount == 0 && info.Bidamount > 0 {
+				project.Bidamount = info.Bidamount
+				project.Bidamounttag = 0
+			}
+		} else {
+			//招标没有多包
+			k := KeyPackage.FindStringSubmatch(info.ProjectName)
+			if len(k) > 0 {
+				//招标是单包
+				if len(project.Package) > 0 {
+					//项目有多包
+					flag := false
+					for _, v := range project.Package {
+						v1, _ := v.([]map[string]interface{})
+						if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
+							flag = true
+						}
+					}
+					if !flag {
+						project.Bidamount = project.Bidamount + info.Bidamount
+						project.Bidamounttag = 0
+					}
+				} else {
+					//项目没有多包
+					if info.Bidamount > 0 {
+						project.Bidamount = project.Bidamount + info.Bidamount
+						project.Bidamounttag = 0
+					} else if info.Bidamount == 0 && tmp["bidamount"] != nil {
+						project.Bidamounttag = 0
+					} else {
+						project.Bidamounttag = 1
+					}
+				}
+			} else {
+				if info.SubType == "中标" || info.SubType == "成交" {
+					if info.Bidamount > 0 {
+						project.Bidamount = info.Bidamount
+						project.Bidamounttag = 0
+					} else {
+						flag := false
+						if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
+							for _, res := range project.InfoFiled {
+								if res.ContractCode != "" && res.ContractCode == info.ContractCode {
+									flag = true
+									break
+								}
+								if res.Bidamount == project.Bidamount {
+									flag = true
+									break
+								}
+							}
+							if !flag {
+								project.Bidamount = project.Bidamount + info.Bidamount
+								project.Bidamounttag = 0
+							} else {
+								if info.Budget > 0 && project.Bidamount > info.Bidamount {
+									project.Bidamount = info.Bidamount
+									project.Bidamounttag = 0
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	} else {
+		project.Bidamounttag = 1
+	}
+
+}

medical_project/task.go

@@ -0,0 +1,305 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"encoding/json"
+	"fmt"
+	"go.uber.org/zap"
+	"medical_project/config"
+	"strings"
+	"time"
+)
+
+var (
+	queryClose     = make(chan bool)
+	queryCloseOver = make(chan bool)
+	pool           = make(chan bool, 2)
+)
+
+func updateAllQueue() {
+	arru := make([][]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoPro.UpSertBulk(config.Conf.DB.MongoP.Coll, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1 * time.Second):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MongoPro.UpSertBulk(config.Conf.DB.MongoP.Coll, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+//全量合并
+func taskQl(udpInfo map[string]interface{}) {
+	defer util.Catch()
+	q := make(map[string]interface{})
+	gtid, _ := udpInfo["gtid"].(string)
+	lteid, _ := udpInfo["lteid"].(string)
+	if mongodb.IsObjectIdHex(gtid) && mongodb.IsObjectIdHex(lteid) {
+		q["_id"] = map[string]interface{}{
+			"$lte": mongodb.StringTOBsonId(lteid),
+			"$gte": mongodb.StringTOBsonId(gtid),
+		}
+	}
+	//生成查询语句执行
+	log.Info("查询语句：", zap.Any("q", q))
+	Enter(q)
+}
+
+//增量合并
+func taskZl(udpInfo map[string]interface{}) {
+	defer util.Catch()
+	//开始id和结束id
+	q, _ := udpInfo["query"].(map[string]interface{})
+	gtid := udpInfo["gtid"].(string)
+	lteid := udpInfo["lteid"].(string)
+	q = map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  mongodb.StringTOBsonId(gtid),
+			"$lte": mongodb.StringTOBsonId(lteid),
+		},
+	}
+	Enter(q)
+	//if udpInfo["stop"] == nil {
+	//	for i := 0; i < 1; i++ {
+	//		sp <- true
+	//	}
+	//	for i := 0; i < 1; i++ {
+	//		<-sp
+	//	}
+	//}
+}
+
+func Enter(q map[string]interface{}) {
+	defer util.Catch()
+	count := 0
+
+	sess := MongoBid.GetMgoConn()
+	defer MongoBid.DestoryMongoConn(sess)
+	infoPool := make(chan map[string]interface{}, 2000)
+	over := make(chan bool)
+	go func() {
+	L:
+		for {
+			select {
+			case tmp := <-infoPool:
+				pool <- true
+				go func(tmp map[string]interface{}) {
+					defer func() {
+						<-pool
+					}()
+					info := ParseInfo(tmp)
+					currentTime = info.Publishtime
+					startProjectMerge(info, tmp)
+				}(tmp)
+			default:
+				select {
+				case tmp := <-infoPool:
+					pool <- true
+					go func(tmp map[string]interface{}) {
+						defer func() {
+							<-pool
+						}()
+						info := ParseInfo(tmp)
+						currentTime = info.Publishtime
+						startProjectMerge(info, tmp)
+					}(tmp)
+				case <-over:
+					break L
+				}
+			}
+		}
+	}()
+	fields := map[string]interface{}{"kvtext": 0, "repeat_reason": 0, "field_source": 0, "detail": 0, "contenthtml": 0, "jsondata": 0}
+	ms := sess.DB(config.Conf.DB.MongoB.Dbname).C(config.Conf.DB.MongoB.Coll).Find(q).Select(fields).Sort("publishtime")
+	query := ms.Iter()
+	var lastid interface{}
+L:
+	for {
+		select {
+		case <-queryClose:
+			log.Error("receive interrupt sign")
+			queryCloseOver <- true
+			break L
+		default:
+			tmp := make(map[string]interface{})
+			if query.Next(&tmp) {
+				lastid = tmp["_id"]
+				if currentType == "ql" {
+					if count%50000 == 0 {
+						util.Debug("current", count, lastid)
+					}
+				} else {
+					if count%2000 == 0 {
+						util.Debug("current", count, lastid)
+					}
+				}
+				//if util.ObjToString(tmp["bid_field"]) == "0101" {
+				infoPool <- tmp
+				//}
+				count++
+			} else {
+				break L
+			}
+		}
+	}
+	time.Sleep(5 * time.Second)
+	over <- true
+	//阻塞
+	for n := 0; n < config.Conf.Server.Thread; n++ {
+		pool <- true
+	}
+	log.Info("所有线程执行完成...", zap.Int("count:", count))
+
+}
+
+func ParseInfo(tmp map[string]interface{}) (info *Info) {
+	bys, _ := json.Marshal(tmp)
+	var thisinfo *Info
+	_ = json.Unmarshal(bys, &thisinfo)
+	if thisinfo == nil {
+		return nil
+	}
+
+	// 处理publishtime为空
+	if thisinfo.Publishtime <= 0 {
+		for _, d := range DateTimeSelect {
+			if tmp[d] != nil {
+				thisinfo.Publishtime = util.Int64All(tmp[d])
+				tmp["publishtime"] = tmp[d]
+				break
+			}
+		}
+	}
+
+	if len(thisinfo.Topscopeclass) == 0 {
+		thisinfo.Topscopeclass = []string{}
+	}
+	if len(thisinfo.Subscopeclass) == 0 {
+		thisinfo.Subscopeclass = []string{}
+	}
+	if thisinfo.SubType == "" {
+		thisinfo.SubType = util.ObjToString(tmp["bidstatus"])
+	}
+
+	//从标题中查找项目编号
+	res := titleGetPc.FindStringSubmatch(thisinfo.Title)
+	if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
+		thisinfo.PTC = res[1]
+	} else {
+		res = titleGetPc1.FindStringSubmatch(thisinfo.Title)
+		if len(res) > 3 && len(res[3]) > 6 && thisinfo.ProjectCode != res[3] && !numCheckPc.MatchString(res[3]) && !_zimureg1.MatchString(res[3]) {
+			thisinfo.PTC = res[3]
+		} else {
+			res = titleGetPc2.FindStringSubmatch(thisinfo.Title)
+			if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
+				thisinfo.PTC = res[1]
+			}
+		}
+	}
+
+	if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
+		//thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
+		//if thisinfo.ProjectName != "" {
+		thisinfo.pnbval++
+		//}
+	}
+
+	if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
+		if thisinfo.ProjectCode != "" {
+			//thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
+			if thisinfo.pnbval == 0 && len([]rune(thisinfo.ProjectCode)) < 5 {
+				thisinfo.ProjectCode = StrOrNum.ReplaceAllString(thisinfo.ProjectCode, "")
+			}
+		} else {
+			//thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
+			if thisinfo.pnbval == 0 && len([]rune(thisinfo.PTC)) < 5 {
+				thisinfo.PTC = StrOrNum.ReplaceAllString(thisinfo.PTC, "")
+			}
+		}
+		thisinfo.pnbval++
+	}
+	if thisinfo.ProjectCode == thisinfo.PTC || strings.Index(thisinfo.ProjectCode, thisinfo.PTC) > -1 {
+		thisinfo.PTC = ""
+	}
+
+	if thisinfo.Buyer != "" && len([]rune(thisinfo.Buyer)) > 2 {
+		thisinfo.pnbval++
+	} else {
+		thisinfo.Buyer = ""
+	}
+	//winners整理、清理
+	winner := util.ObjToString(tmp["winner"])
+	m1 := map[string]bool{}
+	winners := []string{}
+	if winner != "" {
+		m1[winner] = true
+		winners = append(winners, winner)
+	}
+	packageM, _ := tmp["package"].(map[string]interface{})
+	if packageM != nil {
+		thisinfo.HasPackage = true
+		for _, p := range packageM {
+			pm, _ := p.(map[string]interface{})
+			pw := util.ObjToString(pm["winner"])
+			if pw != "" && !m1[pw] {
+				m1[pw] = true
+				winners = append(winners, pw)
+			}
+		}
+	}
+	thisinfo.Winners = winners
+	//清理winnerorder
+	var wins []map[string]interface{}
+	for _, v := range thisinfo.WinnerOrder {
+		w := util.ObjToString(v["entname"])
+		if w != "" {
+			v["entname"] = w
+			wins = append(wins, v)
+		}
+	}
+	thisinfo.WinnerOrder = wins
+
+	thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
+	thisinfo.LenPTC = len([]rune(thisinfo.PTC))
+	thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
+
+	//处理分包中数据异常问题
+	for k, tmp := range thisinfo.Package {
+		if ps, ok := tmp.([]map[string]interface{}); ok {
+			for i, p := range ps {
+				name, _ := p["name"].(string)
+				if len([]rune(name)) > 100 {
+					p["name"] = fmt.Sprint([]rune(name[:100]))
+				}
+				ps[i] = p
+			}
+			thisinfo.Package[k] = ps
+		}
+	}
+	return thisinfo
+}

medical_project/tool.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"reflect"
+	"sort"
+	"time"
+)
+
+var DateTimeSelect = []string{"bidopentime", "bidendtime", "signaturedate", "comeintime"}
+
+//项目中的字段
+var FIELDS = []string{
+	"area",
+	"city",
+	"district",
+	"projectname",
+	"projectcode",
+	"buyer",
+	"buyerclass",
+	"buyerperson",
+	"buyertel",
+	"winner",
+	"agency",
+	"topscopeclass",
+	"subscopeclass",
+	"package",
+}
+
+//招标信息字段
+var INFOFIELDS = []string{
+	"projectname",
+	"projectcode",
+	"contractcode",
+	"title",
+	"publishtime",
+	"comeintime",
+	"bidopentime",
+	"toptype",
+	"subtype",
+	"buyer",
+	"buyerclass",
+	"agency",
+	"winner",
+	"winnertel",
+	"winnerperson",
+	"budget",
+	"bidamount",
+	"buyerperson",
+	"buyertel",
+	"area",
+	"city",
+	"district",
+	"purchasing",
+	"purchasinglist",
+}
+
+// 包信息
+var PackageEle = []string{
+	"origin",
+	"name",
+	//"text",
+	"budget",
+	"winner",
+	"bidamount",
+	"bidamounttype",
+	"currency",
+	"bidstatus",
+}
+
+var bidstatus = map[string]string{
+	"预告": "预告",
+	"中标": "中标",
+	"成交": "成交",
+	"废标": "废标",
+	"流标": "流标",
+	"合同": "合同",
+}
+
+var bidtype = map[string]string{
+	"招标": "招标",
+	"邀标": "邀标",
+	"询价": "询价",
+	"单一": "单一",
+	"竞价": "竞价",
+	"竞谈": "竞谈",
+}
+
+func Duplicate(a interface{}) (ret []interface{}) {
+	va := reflect.ValueOf(a)
+	for i := 0; i < va.Len(); i++ {
+		if i > 0 && reflect.DeepEqual(va.Index(i-1).Interface(), va.Index(i).Interface()) {
+			continue
+		}
+		ret = append(ret, va.Index(i).Interface())
+	}
+	return ret
+}
+
+//二分字符串查找
+func BinarySearch(s []string, k string) int {
+	sort.Strings(s)
+	lo, hi := 0, len(s)-1
+	for lo <= hi {
+		m := (lo + hi) >> 1
+		if s[m] < k {
+			lo = m + 1
+		} else if s[m] > k {
+			hi = m - 1
+		} else {
+			return m
+		}
+	}
+	return -1
+}
+
+//结构体转map
+func StructToMap(filed InfoField) map[string]interface{} {
+	//先转json
+	result, err := json.Marshal(filed)
+	if err != nil {
+		return nil
+	}
+	//json转map
+	res := make(map[string]interface{})
+	err = json.Unmarshal(result, &res)
+	return res
+}
+
+//从数组中删除元素
+func deleteSlice(arr []string, v, stype string) []string {
+	for k, v1 := range arr {
+		if v1 == v {
+			ts := time.Now().Unix()
+			arr = append(arr[:k], arr[k+1:]...)
+			rt := time.Now().Unix() - ts
+			if rt > 0 {
+				log.Println("deleteSlice", stype, rt, v, len(arr))
+			}
+			return arr
+		}
+	}
+	return arr
+}

proposed_project/common.toml

@@ -0,0 +1,78 @@
+
+[serve]
+pici = 1669942800
+thread = 1
+tagrule = "nzj_rule"
+jyhref = "https://www.jianyu360.cn/article/content/%s.html"
+
+[udp]
+locport = ":17821"
+
+[db]
+[db.mysql]
+addr = "192.168.3.217:4000"
+dbnameBasic = "global_common_data"
+dbnameMedical = "Jianyu_subjectdb"
+size = 5
+user = "root"
+password = "=PDT49#80Z!RVv52_z"
+
+maxidle = 10
+maxconn = 20
+maxquerytime = "10s"
+
+[db.mongoB]
+addr = "192.168.3.207:29099"
+dbname = "wjh"
+coll = "bidding"
+size = 15
+user = ""
+password = ""
+
+[db.mongoP]
+addr = "192.168.3.207:29099"
+dbname = "wjh"
+proposedColl = "projectset_proposed"
+combColl = "projectset_comb"
+projectColl = "projectset_20221103"
+size = 15
+user = ""
+password = ""
+
+[redis]
+addr = "192.168.3.207:2679"
+project = "project"
+proposed = "proposed"
+dbT = 1
+dbD = 2
+
+[db.es]
+addr = "http://192.168.3.149:9200"
+user = ""
+password = ""
+size = 5
+indexP = "proposed_v1"
+# minimum_should_match
+minSdMh = "7<90%"
+
+[mail]
+send = false
+to = "wangjianghan@topnet.net.cn"
+api = "http://172.17.145.179:19281/_send/_mail"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"

proposed_project/config/conf.go

@@ -0,0 +1,115 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	// Conf crocodile conf
+	Conf *conf
+)
+
+// Init Config
+func Init(conf string) {
+	_, err := toml.DecodeFile(conf, &Conf)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+}
+
+type conf struct {
+	Serve serve
+	DB    db
+	Udp   udp
+	Mail  mail
+	Log   log
+}
+
+type serve struct {
+	Pici    int64
+	Thread  int
+	TagRule string
+	JyHref  string
+}
+
+type udp struct {
+	LocPort string
+}
+
+type es struct {
+	Addr     string
+	User     string
+	Password string
+	Size     int
+	IndexP   string
+	MinSdMh  string
+}
+
+type mail struct {
+	Send bool
+	To   string
+	Api  string
+}
+
+// Log Config
+type log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type db struct {
+	MongoB mgo
+	MongoP mgo
+	Mysql  mysql
+	Es     es
+	Redis  redis
+}
+
+type mgo struct {
+	Addr         string
+	Dbname       string
+	ProposedColl string
+	CombColl     string
+	ProjectColl  string
+	Size         int
+	User         string
+	Password     string
+}
+
+type redis struct {
+	Addr     string
+	Project  string
+	Proposed string
+	Dbt      int
+	Dbd      int
+}
+
+type mysql struct {
+	Addr          string
+	DbnameBasic   string
+	DbnameMedical string
+	Size          int
+	User          string
+	Password      string
+}
+
+type duration struct {
+	time.Duration
+}
+
+// UnmarshalText parse 10s to time.Time
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}

proposed_project/config/conf_test.go

@@ -0,0 +1,87 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var confs = `
+[udp]
+locport = ":1783"
+jyaddr = "127.0.0.1"
+jyport = 11118
+
+[db]
+[db.mysql]
+addr = "192.168.3.14:4000"
+dbnameBasic = "global_common_data"
+dbnameMedical = "Jianyu_subjectdb"
+size = 5
+user = "root"
+password = "Tibi#20211222"
+
+maxidle = 10
+maxconn = 20
+maxquerytime = "10s"
+
+[db.mongoB]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+coll = "bidding"
+size = 15
+user = ""
+password = ""
+
+[db.mongoP]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+coll = "projectset"
+size = 15
+user = ""
+password = ""
+
+[db.mongoQ]
+addr = "192.168.3.207:27092"
+dbname = "wjh"
+coll = "projectset"
+size = 15
+user = ""
+password = ""
+
+[db.redis]
+addr = "192.168.3.207:2679"
+pcode = "project"
+db = 1
+
+[mail]
+send = false
+to = "wangjianghan@topnet.net.cn"
+api = "http://172.17.145.179:19281/_send/_mail"
+
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "text"
+
+`
+
+func TestInit(t *testing.T) {
+	testfile := "/tmp/crocodile.toml"
+	ioutil.WriteFile(testfile, []byte(confs), 0644)
+	Init(testfile)
+	t.Logf("%+v", Conf.DB.Redis.Addr)
+	os.Remove(testfile)
+}

proposed_project/go.mod

@@ -0,0 +1,13 @@
+module proposed_project
+
+go 1.16
+
+require (
+	app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061
+	github.com/BurntSushi/toml v1.2.0
+	github.com/go-ego/gse v0.80.2
+	github.com/robfig/cron v1.2.0
+	github.com/spf13/cobra v1.6.1
+	go.mongodb.org/mongo-driver v1.10.3
+	go.uber.org/zap v1.23.0
+)

proposed_project/go.sum

@@ -0,0 +1,230 @@
+app.yhyue.com/data_processing/common_utils v0.0.0-20230308013947-0ef15d576c68 h1:560oa2np5fPKQOGvsOYMV9vNQ8Y6y9O+WlUoNbnZ0UU=
+app.yhyue.com/data_processing/common_utils v0.0.0-20230308013947-0ef15d576c68/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
+app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061 h1:UaE1gor8fIhSBAXcZsNOQDMI5sD9LxGECXFh7Ik4rk4=
+app.yhyue.com/data_processing/common_utils v0.0.0-20230427103005-4289580ee061/go.mod h1:XMSY6tIzDnO/YQFjSb0OrOKl93ViGE0ejqcSCTlyHUs=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-ego/gse v0.80.2 h1:3LRfkaBuwlsHsmkOZvnhTcsYPXUAhiP06Sqcid7mO1M=
+github.com/go-ego/gse v0.80.2/go.mod h1:kesekpZfcFQ/kwd9b27VZHUOH5dQUjaaQUZ4OGt4Hj4=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
+github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
+github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
+github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/vcaesar/cedar v0.20.1 h1:cDOmYWdprO7ZW8cngJrDi8Zivnscj9dA/y8Y+2SB1P0=
+github.com/vcaesar/cedar v0.20.1/go.mod h1:iMDweyuW76RvSrCkQeZeQk4iCbshiPzcCvcGCtpM7iI=
+github.com/vcaesar/tt v0.20.0 h1:9t2Ycb9RNHcP0WgQgIaRKJBB+FrRdejuaL6uWIHuoBA=
+github.com/vcaesar/tt v0.20.0/go.mod h1:GHPxQYhn+7OgKakRusH7KJ0M5MhywoeLb8Fcffs/Gtg=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.mongodb.org/mongo-driver v1.10.3 h1:XDQEvmh6z1EUsXuIkXE9TaVeqHw6SwS1uf93jFs0HBA=
+go.mongodb.org/mongo-driver v1.10.3/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
+go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

proposed_project/init.go

@@ -0,0 +1,203 @@
+package main
+
+import (
+	"app.yhyue.com/data_processing/common_utils/elastic"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/mysqldb"
+	"fmt"
+	"os"
+	"proposed_project/config"
+	"sync"
+)
+
+var (
+	MgoBid, MgoPro        *mongodb.MongodbSim //mongodb连接
+	MysqlTool, MysqlTool1 *mysqldb.Mysql
+	saveSize              int
+
+	LastId     string //增量拟建数据最后id
+	TaskSingle bool
+
+	pici int64 // 拟建增量数据pici时间
+
+	updatePool chan []map[string]interface{}
+	updateSp   chan bool
+	savePool   chan map[string]interface{}
+	saveSp     chan bool
+
+	AllPidMap     map[string]*ID
+	AllPidMapLock sync.Mutex
+	AllPnMap      map[string]*Pname
+	AllPnMapLock  sync.Mutex
+
+	AllCodeMap     map[string]string
+	AllCodeMapLock sync.Mutex
+)
+
+func init() {
+	config.Init("./common.toml")
+	InitLog()
+	InitMgo()
+
+	TaskSingle = true
+
+	saveSize = 200
+	AllPidMap = make(map[string]*ID, 20000)
+	AllPnMap = make(map[string]*Pname, 20000)
+	AllCodeMap = make(map[string]string, 50000)
+
+	updatePool = make(chan []map[string]interface{}, 5000)
+	updateSp = make(chan bool, 1)
+	savePool = make(chan map[string]interface{}, 5000)
+	saveSp = make(chan bool, 2)
+}
+
+func InitLog() {
+	logcfg := config.Conf.Log
+
+	err := log.InitLog(
+		log.Path(logcfg.LogPath),
+		log.Level(logcfg.LogLevel),
+		log.Compress(logcfg.Compress),
+		log.MaxSize(logcfg.MaxSize),
+		log.MaxBackups(logcfg.MaxBackups),
+		log.MaxAge(logcfg.MaxAge),
+		log.Format(logcfg.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func InitMgo() {
+	MgoBid = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoB.Addr,
+		DbName:      config.Conf.DB.MongoB.Dbname,
+		Size:        config.Conf.DB.MongoB.Size,
+		UserName:    config.Conf.DB.MongoB.User,
+		Password:    config.Conf.DB.MongoB.Password,
+	}
+	MgoBid.InitPool()
+	MgoPro = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoP.Addr,
+		DbName:      config.Conf.DB.MongoP.Dbname,
+		Size:        config.Conf.DB.MongoP.Size,
+		UserName:    config.Conf.DB.MongoP.User,
+		Password:    config.Conf.DB.MongoP.Password,
+	}
+	MgoPro.InitPool()
+}
+
+func InitMysql() {
+	dbcfg := config.Conf.DB.Mysql
+	MysqlTool = &mysqldb.Mysql{
+		Address:  dbcfg.Addr,
+		DBName:   dbcfg.DbnameMedical,
+		UserName: dbcfg.User,
+		PassWord: dbcfg.Password,
+	}
+	MysqlTool.Init()
+
+	MysqlTool1 = &mysqldb.Mysql{
+		Address:  dbcfg.Addr,
+		DBName:   dbcfg.DbnameBasic,
+		UserName: dbcfg.User,
+		PassWord: dbcfg.Password,
+	}
+	MysqlTool1.Init()
+}
+
+func InitEs() {
+	Es = &elastic.Elastic{
+		S_esurl:  config.Conf.DB.Es.Addr,
+		I_size:   config.Conf.DB.Es.Size,
+		Username: config.Conf.DB.Es.User,
+		Password: config.Conf.DB.Es.Password,
+	}
+	Es.InitElasticSize()
+}
+
+type Pname struct {
+	Id   map[string]string
+	Lock sync.Mutex
+}
+
+type Key struct {
+	Id   string
+	Lock sync.Mutex
+}
+
+type ID struct {
+	Lock sync.Mutex
+	P    *Project
+}
+
+// Project 内存 项目信息
+type Project struct {
+	Id                  string   `json:"_id"`
+	Ids                 []string `json:"ids,omitempty"`
+	FirstTime           int64    `json:"firsttime,omitempty"`   //项目的最早时间
+	LastTime            int64    `json:"lasttime,omitempty"`    //项目的最后时间
+	ProjectName         string   `json:"projectname,omitempty"` //项目名称
+	Owner               string   `json:"owner"`
+	ApproveCode         string   `json:"approvecode"` // 审批代码
+	ApproveNumber       string   `json:"approvenumber"`
+	ApproveDept         string   `json:"approvedept"`
+	TotalInvestment     string   `json:"total_investment"`
+	ProjectAddr         string   `json:"projectaddr"`
+	Funds               string   `json:"funds"`
+	ProjectPeriod       string   `json:"projectperiod"`
+	ProjectScale        string   `json:"project_scale"`
+	ProjectStartDate    int64    `json:"project_startdate"`
+	ProjectCompleteDate int64    `json:"project_completedate"`
+	ConstructionArea    string   `json:"construction_area"`
+	FollowNum           int      `json:"follow_num"`
+	FloorArea           string   `json:"floor_area"`
+	ProjectPhone        string   `json:"project_phone"`
+	ProjectPerson       string   `json:"project_person"`
+	Area                string   `json:"area"`     //地区
+	City                string   `json:"city"`     //地市
+	District            string   `json:"district"` //区县
+	Site                string   `json:"site"`
+}
+
+// Info 拟在建数据信息
+type Info struct {
+	Id                  string   `json:"_id"`
+	Href                string   `json:"href"`
+	Publishtime         int64    `json:"publishtime"`
+	Title               string   `json:"title"`
+	ProjectName         string   `json:"projectname"`
+	Area                string   `json:"area"`
+	City                string   `json:"city"`
+	District            string   `json:"district"`
+	Infoformat          int      `json:"infoformat"`
+	Site                string   `json:"site"`
+	Topscopeclass       []string `json:"topscopeclass"`
+	Subscopeclass       []string `json:"subscopeclass"`
+	Buyerclass          string   `json:"buyerclass"`
+	Owner               string   `json:"owner"`
+	ApproveCode         string   `json:"approvecode"`
+	ApproveDept         string   `json:"approvedept"`
+	ApproveStatus       string   `json:"approvestatus"`
+	ApproveTime         string   `json:"approvetime"`
+	ApproveNumber       string   `json:"approvenumber"`
+	ApproveContent      string   `json:"approvecontent"`
+	ProjectType         string   `json:"projecttype"`
+	ApproveCity         string   `json:"approvecity"`
+	ProjectScale        string   `json:"project_scale"`
+	TotalInvestment     string   `json:"total_investment"`
+	ProjectAddr         string   `json:"projectaddr"`
+	ProjectPeriod       string   `json:"projectperiod"`
+	Funds               string   `json:"funds"`
+	ProjectStartDate    int64    `json:"project_startdate"`
+	ProjectCompleteDate int64    `json:"project_completedate"`
+	ConstructionArea    string   `json:"construction_area"`
+	FloorArea           string   `json:"floor_area"`
+	ProjectPhone        string   `json:"project_phone"`
+	ProjectPerson       string   `json:"project_person"`
+}
+
+var Project_Stage = []string{"可研", "初设", "环评", "审批备案", "设计", "施工准备", "施工", "竣工验收"}

proposed_project/load_data.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"app.yhyue.com/data_processing/common_utils/log"
+	"encoding/json"
+	"fmt"
+	"go.uber.org/zap"
+	"proposed_project/config"
+	"sync"
+	"time"
+)
+
+func loadProject() {
+	sess := MgoPro.GetMgoConn()
+	defer MgoPro.DestoryMongoConn(sess)
+	f := map[string]interface{}{"list": 0}
+	it := sess.DB(MgoPro.DbName).C(config.Conf.DB.MongoP.ProposedColl).Find(nil).Select(f).Iter()
+	count := 0
+
+	ch := make(chan bool, 3)
+	wg := &sync.WaitGroup{}
+	for result := make(map[string]interface{}); it.Next(result); count++ {
+		if count%40000 == 0 {
+			log.Info(fmt.Sprintf("iter --- %d", count))
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			bys, _ := json.Marshal(tmp)
+			var p *Project
+			_ = json.Unmarshal(bys, &p)
+			AllPidMapLock.Lock()
+			AllPidMap[p.Id] = &ID{P: p}
+			AllPidMapLock.Unlock()
+			AllCodeMapLock.Lock()
+			AllCodeMap[p.ApproveCode] = p.Id
+			AllCodeMapLock.Unlock()
+
+			AllPnMapLock.Lock()
+			res := AllPnMap[p.Id]
+			if res != nil {
+				res.Id[p.Id] = p.ProjectName
+			} else {
+				res = &Pname{
+					Id: map[string]string{p.Id: p.ProjectName},
+				}
+			}
+			AllPnMapLock.Unlock()
+		}(result)
+		result = make(map[string]interface{})
+	}
+	wg.Wait()
+	time.Sleep(2 * time.Second)
+	log.Info("loadProject", zap.Int("count", count), zap.Int("pcode", len(AllCodeMap)))
+}

proposed_project/main.go

@@ -0,0 +1,626 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/elastic"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/redis"
+	"app.yhyue.com/data_processing/common_utils/udp"
+	"encoding/json"
+	"fmt"
+	"github.com/robfig/cron"
+	"github.com/spf13/cobra"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.uber.org/zap"
+	"net"
+	"os"
+	"proposed_project/config"
+	"sync"
+	"time"
+)
+
+var (
+	UdpClient udp.UdpClient
+
+	Es         *elastic.Elastic
+	EsBulkSize = 200
+	saveEsPool = make(chan map[string]interface{}, 5000)
+	saveEsSp   = make(chan bool, 3)
+)
+
+func main() {
+
+	rootCmd := &cobra.Command{Use: "my cmd"}
+	rootCmd.AddCommand(tags())
+	rootCmd.AddCommand(project())
+	rootCmd.AddCommand(nzjData())
+	rootCmd.AddCommand(projectAdd())
+	rootCmd.AddCommand(tidbSave())
+	rootCmd.AddCommand(tidbAddSave())
+	rootCmd.AddCommand(redisSave())
+	rootCmd.AddCommand(esSave())
+	rootCmd.AddCommand(projectComb())
+	rootCmd.AddCommand(projectCombTidb())
+	rootCmd.AddCommand(projectCombAdd())
+
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println("rootCmd.Execute failed", err.Error())
+	}
+}
+
+func tags() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "tags",
+		Short: "Start processing tags data",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitRule()
+			go UpdateMethod()
+			taskRun()
+
+			c := make(chan bool, 1)
+			<-c
+		},
+	}
+	return cmdClient
+}
+
+func project() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "project",
+		Short: "Start processing project data",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitRule()
+
+			loadProject()
+			go updateAllQueue()
+
+			UdpClient = udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
+			UdpClient.Listen(processUdpMsg)
+			log.Info("Udp服务监听", zap.String("port:", config.Conf.Udp.LocPort))
+
+			c := make(chan bool, 1)
+			<-c
+		},
+	}
+	return cmdClient
+}
+
+func nzjData() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "nzj-data",
+		Short: "Start processing project data",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			if LastId == "" {
+				_ = cmd.Help()
+				os.Exit(0)
+			}
+
+			go saveMethod()
+
+			sid, eid := taskQ()
+			if sid != "" && eid != "" {
+				doTask1(sid, eid)
+				LastId = eid
+			}
+
+			crn := cron.New()
+			cronstr := "0 */10 * * * ?" // 每10min执行一次
+			_ = crn.AddFunc(cronstr, func() {
+				if TaskSingle {
+					sid, eid := taskQ()
+					if sid != "" && eid != "" {
+						TaskSingle = false
+						doTask1(sid, eid)
+						LastId = eid
+					}
+				} else {
+					log.Info("上次任务未执行完成")
+				}
+
+			})
+			crn.Start()
+
+			c := make(chan bool, 1)
+			<-c
+		},
+	}
+	cmdClient.Flags().StringVarP(&LastId, "lastid", "s", "", "data start id")
+	return cmdClient
+}
+
+func projectAdd() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "project-add",
+		Short: "Start processing project data",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			if LastId == "" {
+				_ = cmd.Help()
+				os.Exit(0)
+			}
+
+			InitRule()
+			loadProject()
+			go updateAllQueue()
+
+			info, _ := MgoBid.Find("nzj_bidding", nil, `{"_id": -1}`, nil, true, -1, 1)
+			doTask(LastId, mongodb.BsonIdToSId((*info)[0]["_id"]))
+			LastId = mongodb.BsonIdToSId((*info)[0]["_id"])
+
+			crn := cron.New()
+			cronstr := "0 */30 * * * ?" // 每30min执行一次
+			_ = crn.AddFunc(cronstr, func() {
+				if TaskSingle {
+					info, _ := MgoBid.Find("nzj_bidding", nil, `{"_id": -1}`, nil, true, -1, 1)
+					TaskSingle = false
+					doTask(LastId, mongodb.BsonIdToSId((*info)[0]["_id"]))
+					LastId = mongodb.BsonIdToSId((*info)[0]["_id"])
+				} else {
+					log.Info("上次任务未执行完成")
+				}
+
+			})
+			crn.Start()
+
+			c := make(chan bool, 1)
+			<-c
+		},
+	}
+	cmdClient.Flags().StringVarP(&LastId, "lastid", "s", "", "data start id")
+	return cmdClient
+}
+
+func tidbSave() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "tidb",
+		Short: "Start processing project save to tidb",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitMysql()
+			InitArea()
+			go SaveFunc("dwd_f_nzj_baseinfo_new", BaseField)
+			go SaveRFunc("dwd_f_nzj_follw_record_new", RecordField)
+			go SaveCFunc("dwd_f_nzj_contact_new", ContactField)
+			go SaveCyFunc("dwd_f_nzj_category_tags_new", CategoryField)
+			go SaveEntFunc("dwd_f_nzj_ent_new", EntField)
+
+			redis.InitRedis1("ent_id=172.17.4.189:8379", 6) // name_id
+
+			taskTidb(nil)
+			c := make(chan bool, 1)
+			<-c
+		},
+	}
+	return cmdClient
+}
+
+func tidbAddSave() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "tidb-add",
+		Short: "Start processing project save to tidb",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitMysql()
+			InitArea()
+			//go SaveFunc("dwd_f_nzj_baseinfo", BaseField)
+			//go SaveRFunc("dwd_f_nzj_follw_record", RecordField)
+			//go SaveCFunc("dwd_f_nzj_contact", ContactField)
+			//go SaveCyFunc("dwd_f_nzj_category_tags", CategoryField)
+
+			redis.InitRedis1("ent_id=172.17.4.189:8379", 6) // name_id
+
+			taskTidb_add(bson.M{"pici": bson.M{"$gt": pici}})
+
+			crn := cron.New()
+			cronstr := "0 */30 * * * ?" // 每30min执行一次
+			_ = crn.AddFunc(cronstr, func() {
+				if TaskSingle {
+					TaskSingle = false
+					taskTidb_add(bson.M{"pici": bson.M{"$gt": pici}})
+				} else {
+					log.Info("上次任务未执行完成")
+				}
+
+			})
+			crn.Start()
+
+			c := make(chan bool, 1)
+			<-c
+		},
+	}
+	cmdClient.Flags().Int64VarP(&pici, "pici", "c", 0, "pici time")
+	return cmdClient
+}
+
+func redisSave() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "redis",
+		Short: "Start processing project save to tidb",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitMysql()
+			redis.InitRedis1("ent_id=172.17.4.189:8379", 6) // name_id
+
+			redisDisp()
+		},
+	}
+	return cmdClient
+}
+
+func esSave() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "es-save",
+		Short: "Start processing project save to es",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitMysql()
+			InitArea()
+			InitTagCode()
+			InitEs()
+
+			go SaveEs()
+			esDisp()
+
+			crn := cron.New()
+			cronstr := "0 */30 * * * ?" // 每30min执行一次
+			_ = crn.AddFunc(cronstr, func() {
+				if TaskSingle {
+					TaskSingle = false
+					esDisp()
+				} else {
+					log.Info("上次任务未执行完成")
+				}
+
+			})
+			crn.Start()
+
+			c := make(chan bool, 1)
+			<-c
+		},
+	}
+	cmdClient.Flags().Int64VarP(&pici, "pici", "c", 0, "pici time")
+	return cmdClient
+}
+
+// @Description 拟在建项目关联
+// @Author J 2023/4/17 09:05
+func projectComb() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "project-comb",
+		Short: "Start processing combined project",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			initSeg()
+			InitEs()
+
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Project, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbt)
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Proposed, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbd)
+
+			go SavePpMethod()
+			taskComb()
+			c := make(chan bool, 1)
+			<-c
+		},
+	}
+	return cmdClient
+}
+
+// @Description 拟在建项目关联后入tidb处理
+// @Author J 2023/4/17 09:06
+func projectCombTidb() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "project-comb-tidb",
+		Short: "Start processing combined project",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			InitMysql()
+			initStage()
+
+			redis.InitRedis1("ent_id=172.17.4.189:8379", 6) // name_id
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Project, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbt)
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Proposed, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbd)
+			go SaveEntFunc1("dwd_f_nzj_ent", EntField)
+			taskD()
+			c := make(chan bool, 1)
+			<-c
+		},
+	}
+	return cmdClient
+}
+
+// @Description 拟在建关联数据 增量数据处理
+// @Author J 2023/4/24 13:51
+func projectCombAdd() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "project-comb-add",
+		Short: "Start processing combined project add",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			initSeg()
+			InitEs()
+			InitMysql()
+			initStage()
+
+			redis.InitRedis1("ent_id=172.17.4.189:8379", 6) // name_id
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Project, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbt)
+			redis.InitRedis1(fmt.Sprintf("%s=%s", config.Conf.DB.Redis.Proposed, config.Conf.DB.Redis.Addr), config.Conf.DB.Redis.Dbd)
+			go SaveEntFunc1("dwd_f_nzj_ent", EntField)
+
+			go taskAA()
+			go taskBB()
+
+			crn := cron.New()
+			cronstr := "0 0 22 * * ?" // 每天10点执行一次
+			_ = crn.AddFunc(cronstr, func() {
+				if TaskSingle {
+					go taskAA()
+					go taskBB()
+				} else {
+					log.Info("上次任务未执行完成")
+				}
+
+			})
+			crn.Start()
+
+			c := make(chan bool, 1)
+			<-c
+		},
+	}
+	return cmdClient
+}
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case udp.OP_TYPE_DATA:
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Info("processUdpMsg", zap.Any("mapInfo:", mapInfo))
+		gtid, _ := mapInfo["gtid"].(string)
+		lteid, _ := mapInfo["lteid"].(string)
+		if err != nil || gtid == "" || lteid == "" {
+			UdpClient.WriteUdp([]byte("tidb udp error"), udp.OP_NOOP, ra)
+		} else {
+			//udp成功回写
+			if k := util.ObjToString(mapInfo["key"]); k != "" {
+				UdpClient.WriteUdp([]byte(k), udp.OP_NOOP, ra)
+			} else {
+				k = fmt.Sprintf("%s-%s-%s", gtid, lteid, util.ObjToString(mapInfo["stype"]))
+				UdpClient.WriteUdp([]byte(k), udp.OP_NOOP, ra)
+			}
+			log.Info("start merge ...")
+			doTask(gtid, lteid)
+		}
+	case udp.OP_NOOP: //下个节点回应
+	}
+}
+
+func taskQ() (string, string) {
+	log.Info("taskQ", zap.String("lastid", LastId))
+	query := bson.M{"gtid": bson.M{"$gt": LastId}}
+	info, _ := MgoBid.Find("field_data_record", query, `{"_id": -1}`, nil, false, -1, 1)
+	if len(*info) > 0 {
+		newid := util.ObjToString((*info)[0]["lteid"])
+		log.Info("taskQ", zap.String("新lastid", newid))
+		return LastId, newid
+	} else {
+		log.Info("taskQ 未发现新数据")
+		return "", ""
+	}
+}
+
+func redisDisp() {
+	pool := make(chan bool, 5) //控制线程数
+	wg := &sync.WaitGroup{}
+
+	finalId := 0
+	lastInfo := MysqlTool1.SelectBySql(fmt.Sprintf("SELECT id FROM %s ORDER BY id DESC LIMIT 1", "dws_f_ent_baseinfo"))
+	if len(*lastInfo) > 0 {
+		finalId = util.IntAll((*lastInfo)[0]["id"])
+	}
+	util.Debug("taskIterateSql---", "finally id", finalId)
+	lastid, count := 0, 0
+	for {
+		util.Debug("重新查询，lastid---", lastid)
+		q := fmt.Sprintf("SELECT id, name, name_id, area_code, city_code FROM %s WHERE id > %d ORDER BY id ASC limit 100000", "dws_f_ent_baseinfo", lastid)
+		rows, err := MysqlTool1.DB.Query(q)
+		if err != nil {
+			util.Debug("taskIterateSql---", err)
+		}
+		columns, err := rows.Columns()
+		if finalId == lastid {
+			util.Debug("----finish----------", count)
+			break
+		}
+		for rows.Next() {
+			scanArgs := make([]interface{}, len(columns))
+			values := make([]interface{}, len(columns))
+			ret := make(map[string]interface{})
+			for k := range values {
+				scanArgs[k] = &values[k]
+			}
+			err = rows.Scan(scanArgs...)
+			if err != nil {
+				util.Debug("taskIterateSql---", err)
+				break
+			}
+			for i, col := range values {
+				if v, ok := col.([]uint8); ok {
+					ret[columns[i]] = string(v)
+				} else {
+					ret[columns[i]] = col
+				}
+			}
+			lastid = util.IntAll(ret["id"])
+			count++
+			if count%20000 == 0 {
+				util.Debug("current-------", count, lastid)
+			}
+			pool <- true
+			wg.Add(1)
+			go func(tmp map[string]interface{}) {
+				defer func() {
+					<-pool
+					wg.Done()
+				}()
+				redis.PutCKV("ent_id", util.ObjToString(tmp["name"]),
+					fmt.Sprintf("%s_%s_%s", util.ObjToString(tmp["name_id"]), util.ObjToString(tmp["area_code"]), util.ObjToString(tmp["city_code"])))
+
+			}(ret)
+			ret = make(map[string]interface{})
+		}
+		_ = rows.Close()
+		wg.Wait()
+	}
+}
+
+var EsField = []string{"_id", "projectname", "owner", "area", "city", "district", "nature_code", "ownerclass_code", "project_stage_code", "category_code", "total_investment", "lasttime", "proposed_id"}
+
+func esDisp() {
+	sess := MgoPro.GetMgoConn()
+	defer MgoPro.DestoryMongoConn(sess)
+
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	var q bson.M
+	if pici > 0 {
+		q = bson.M{"pici": bson.M{"$gt": pici}}
+	}
+
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(q).Select(SelectF).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%200 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		if t := util.Int64All(tmp["pici"]); t > pici {
+			pici = t
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			save := make(map[string]interface{})
+			for _, f := range EsField {
+				if tmp[f] == nil {
+					continue
+				}
+				if f == "_id" {
+					save["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+					save["_id"] = mongodb.BsonIdToSId(tmp["_id"])
+				} else if f == "area" {
+					save[f] = tmp[f]
+					save["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+					if util.ObjToString(tmp["city"]) != "" {
+						save["area_city"] = tmp["city"]
+					} else {
+						save["area_city"] = tmp["area"]
+					}
+				} else if f == "lasttime" {
+					save[f] = util.Int64All(tmp[f]) * 1000
+				} else if f == "nature_code" {
+					save[f] = tmp[f]
+					save["nature"] = TagCode["nature"].(map[string]interface{})[util.ObjToString(tmp[f])]
+				} else if f == "ownerclass_code" {
+					save[f] = tmp[f]
+					save["ownerclass"] = TagCode["owner"].(map[string]interface{})[util.ObjToString(tmp[f])]
+				} else if f == "project_stage_code" {
+					save[f] = tmp[f]
+					save["project_stage"] = TagCode["project_stage"].(map[string]interface{})[util.ObjToString(tmp[f])]
+				} else if f == "category_code" {
+					save[f] = tmp[f]
+					save["category"] = TagCode["category"].(map[string]interface{})[util.ObjToString(tmp[f])]
+				} else if f == "total_investment" {
+					text := util.ObjToString(tmp[f])
+					c := ObjToMoney(text)
+					c = c / 10000
+					if c != 0 {
+						c, _ = util.FormatFloat(c, 4)
+						save[f] = c
+					}
+				} else {
+					save[f] = tmp[f]
+				}
+			}
+			saveEsPool <- save
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count))
+	TaskSingle = true
+}
+
+func saveMethod() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MgoBid.SaveBulk("nzj_bidding", arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveSp
+					}()
+					MgoBid.SaveBulk("nzj_bidding", arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveEs() {
+	arru := make([]map[string]interface{}, EsBulkSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == EsBulkSize {
+				saveEsSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsSp
+					}()
+					Es.BulkSave(config.Conf.DB.Es.IndexP, arru)
+				}(arru)
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveEsSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEsSp
+					}()
+					Es.BulkSave(config.Conf.DB.Es.IndexP, arru)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, EsBulkSize)
+				indexu = 0
+			}
+		}
+	}
+}

proposed_project/merge.go

@@ -0,0 +1,274 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"fmt"
+	"go.uber.org/zap"
+	"math/rand"
+	"reflect"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+	"unicode/utf8"
+)
+
+// @Description 参与合并流程
+// @Author J 2022/10/20 13:57
+func startProjectMerge(tmp map[string]interface{}, info *Info) string {
+	IDArr := getCompareIds(info.ProjectName, info.Site)
+	isMerge := false
+	var mergeProject *Project
+	for _, v := range IDArr {
+		mergeProject = v.P
+		if info.Area != "" && mergeProject.Area != "" {
+			if info.Area != "全国" && mergeProject.Area != "全国" && info.Area != mergeProject.Area {
+				continue
+			} else {
+				isMerge = true
+				break
+			}
+		} else {
+			isMerge = true
+			break
+		}
+	}
+	if isMerge && mergeProject != nil {
+		updateProject(tmp, *info, mergeProject)
+		return mergeProject.Id
+	} else {
+		_, pinfo := newProject(tmp, *info)
+		AllPidMapLock.Lock()
+		AllPidMap[info.Id] = &ID{P: pinfo}
+		AllPidMapLock.Unlock()
+		AllPnMapLock.Lock()
+		res := AllPnMap[info.Site]
+		if res != nil {
+			res.Lock.Lock()
+			res.Id[info.ProjectName] = info.Id
+			res.Lock.Unlock()
+		} else {
+			res = &Pname{
+				Id:   map[string]string{info.ProjectName: info.Id},
+				Lock: sync.Mutex{},
+			}
+		}
+		AllPnMap[info.Site] = res
+		AllPnMapLock.Unlock()
+	}
+	return info.Id
+}
+
+// @Description 通过项目名称找可以与之合并的项目集合
+// @Author J 2022/10/20 14:22
+func getCompareIds(pname, site string) (IDArr []*ID) {
+	AllPnMapLock.Lock()
+	res := AllPnMap[site]
+	defer AllPnMapLock.Unlock()
+
+	pid := ""
+	IDArr = []*ID{} //项目信息
+	if res != nil {
+		res.Lock.Lock()
+		pid = res.Id[pname]
+		res.Lock.Unlock()
+
+		AllPidMapLock.Lock()
+		ID := AllPidMap[pid]
+		AllPidMapLock.Unlock()
+		if ID != nil {
+			IDArr = append(IDArr, ID)
+		}
+	}
+	return IDArr
+
+}
+
+func newProject(tmp map[string]interface{}, info Info) (string, *Project) {
+	pId := mongodb.StringTOBsonId(info.Id)
+	set := make(map[string]interface{})
+	set["_id"] = pId
+	for _, f := range FIELDS {
+		if tmp[f] != nil && tmp[f] != "" {
+			if reflect.TypeOf(tmp[f]).String() == "string" {
+				if utf8.RuneCountInString(util.ObjToString(tmp[f])) < 1000 {
+					set[f] = tmp[f]
+				}
+			} else {
+				set[f] = tmp[f]
+			}
+		}
+	}
+	p1 := NewCachePinfo(info)
+	set["firsttime"] = util.IntAll(info.Publishtime)
+	set["lasttime"] = util.IntAll(info.Publishtime)
+	set["ids"] = []string{info.Id}
+	set["sourceinfourl"] = tmp["href"]
+	set["follow_num"] = 1
+	set["pici"] = time.Now().Unix()
+	p1.FollowNum = 1
+	now := time.Now().Unix()
+	st := util.FormatDateByInt64(&now, util.Date_yyyyMMdd)
+	parseSt := strconv.FormatInt(util.Int64All(st), 8)                                       // 转8进制
+	rd := fmt.Sprintf("%04v", rand.New(rand.NewSource(time.Now().UnixNano())).Int63n(10000)) // 4位随机数
+	set["proposed_number"] = fmt.Sprintf("NZJ%s%s", parseSt, rd)
+	push := PushListInfo(tmp, info.Id)
+	push["follow_num"] = 1
+	set["list"] = []map[string]interface{}{
+		push,
+	}
+	updatePool <- []map[string]interface{}{
+		{
+			"_id": pId,
+		},
+		{
+			"$set": set,
+		},
+	}
+	return info.Id, &p1
+}
+
+// @Description 合并字段信息
+// @Author J 2022/10/20 13:57
+func updateProject(tmp map[string]interface{}, info Info, pInfo *Project) {
+	if BinarySearch(pInfo.Ids, info.Id) > -1 {
+		log.Error("updateProject", zap.String("repeat", info.Id), zap.String("pid", mongodb.BsonIdToSId(pInfo.Id)))
+		return
+	}
+	set := make(map[string]interface{})
+	pInfo.Ids = append(pInfo.Ids, info.Id)
+
+	pInfo.LastTime = info.Publishtime
+	set["lasttime"] = info.Publishtime
+	pInfo.FollowNum += 1
+	set["follow_num"] = pInfo.FollowNum
+	if pInfo.Owner == "" && info.Owner != "" {
+		pInfo.Owner = info.Owner
+		set["owner"] = info.Owner
+	}
+	if pInfo.ApproveCode == "" && info.ApproveCode != "" {
+		pInfo.ApproveCode = info.ApproveCode
+		set["approvecode"] = info.ApproveCode
+	}
+	if pInfo.ApproveNumber == "" && info.ApproveNumber != "" {
+		pInfo.ApproveNumber = info.ApproveNumber
+		set["approvenumber"] = info.ApproveNumber
+	}
+	if pInfo.ApproveDept == "" && info.ApproveDept != "" {
+		pInfo.ApproveDept = info.ApproveDept
+		set["approvedept"] = info.ApproveDept
+	}
+	if pInfo.TotalInvestment == "" && info.TotalInvestment != "" && utf8.RuneCountInString(info.TotalInvestment) < 1000 {
+		pInfo.TotalInvestment = info.TotalInvestment
+		set["total_investment"] = info.TotalInvestment
+	}
+	if pInfo.Funds == "" && info.Funds != "" && utf8.RuneCountInString(info.Funds) < 1000 {
+		pInfo.Funds = info.Funds
+		set["funds"] = info.Funds
+	}
+	if pInfo.ProjectAddr == "" && info.ProjectAddr != "" && utf8.RuneCountInString(info.ProjectAddr) < 1000 {
+		pInfo.ProjectAddr = info.ProjectAddr
+		set["projectaddr"] = info.ProjectAddr
+	}
+	if pInfo.ProjectPeriod == "" && info.ProjectPeriod != "" && utf8.RuneCountInString(info.ProjectPeriod) < 1000 {
+		pInfo.ProjectPeriod = info.ProjectPeriod
+		set["projectperiod"] = info.ProjectPeriod
+	}
+	if pInfo.ProjectScale == "" && info.ProjectScale != "" && utf8.RuneCountInString(info.ProjectScale) < 1000 {
+		pInfo.ProjectPeriod = info.ProjectScale
+		set["project_scale"] = info.ProjectScale
+	}
+	if pInfo.ProjectStartDate == 0 && info.ProjectStartDate > 0 {
+		pInfo.ProjectStartDate = info.ProjectStartDate
+		set["project_startdate"] = info.ProjectStartDate
+	}
+	if pInfo.ProjectCompleteDate == 0 && info.ProjectCompleteDate > 0 {
+		pInfo.ProjectCompleteDate = info.ProjectCompleteDate
+		set["project_completedate"] = info.ProjectCompleteDate
+	}
+	if pInfo.ConstructionArea == "" && info.ConstructionArea != "" && utf8.RuneCountInString(info.ConstructionArea) < 1000 {
+		pInfo.ConstructionArea = info.ConstructionArea
+		set["construction_area"] = info.ConstructionArea
+	}
+	if pInfo.FloorArea == "" && info.FloorArea != "" && utf8.RuneCountInString(info.FloorArea) < 1000 {
+		pInfo.FloorArea = info.FloorArea
+		set["floor_area"] = info.FloorArea
+	}
+	if pInfo.ProjectPerson == "" && info.ProjectPerson != "" {
+		pInfo.ProjectPerson = info.ProjectPerson
+		set["project_person"] = info.ProjectPerson
+	}
+	if pInfo.ProjectPhone == "" && info.ProjectPhone != "" {
+		pInfo.ProjectPhone = info.ProjectPhone
+		set["project_phone"] = info.ProjectPhone
+	}
+
+	for _, f := range []string{"ownerclass_code", "category_code", "nature_code", "project_stage_code"} {
+		if c := util.ObjToString(tmp[f]); c != "" {
+			set[f] = c
+		}
+	}
+	set["pici"] = time.Now().Unix()
+	push := PushListInfo(tmp, info.Id)
+	push["follow_num"] = pInfo.FollowNum
+	update := map[string]interface{}{}
+	set["updatetime"] = time.Now().Unix()
+	update["$set"] = set
+	update["$push"] = map[string]interface{}{
+		"list": push,
+		"ids":  info.Id,
+	}
+	updateInfo := []map[string]interface{}{
+		{
+			"_id": mongodb.StringTOBsonId(pInfo.Id),
+		},
+		update,
+	}
+	updatePool <- updateInfo
+}
+
+// @Description list中存放的信息
+// @Author J 2022/10/23 10:16
+func PushListInfo(tmp map[string]interface{}, infoid string) map[string]interface{} {
+	res := map[string]interface{}{
+		"infoid": infoid,
+	}
+	for _, k := range INFOFIELDS {
+		if tmp[k] != nil {
+			res[k] = tmp[k]
+		}
+	}
+	return res
+}
+
+func NewCachePinfo(info Info) Project {
+	p1 := Project{
+		Id:          info.Id,
+		Ids:         []string{info.Id},
+		ProjectName: info.ProjectName,
+		ApproveCode: info.ApproveCode,
+		Area:        info.Area,
+		City:        info.City,
+		District:    info.District,
+	}
+	return p1
+}
+
+//二分字符串查找
+func BinarySearch(s []string, k string) int {
+	sort.Strings(s)
+	lo, hi := 0, len(s)-1
+	for lo <= hi {
+		m := (lo + hi) >> 1
+		if s[m] < k {
+			lo = m + 1
+		} else if s[m] > k {
+			hi = m - 1
+		} else {
+			return m
+		}
+	}
+	return -1
+}

proposed_project/models.go

@@ -0,0 +1,44 @@
+package main
+
+type QueryObject struct {
+	Query  *BoolObject `json:"query,omitempty"`
+	Source []string    `json:"_source"`
+	Size   int         `json:"size"`
+}
+
+type QueryObject1 struct {
+	Query  *BoolObject1 `json:"query,omitempty"`
+	Source []string     `json:"_source"`
+	Size   int          `json:"size"`
+}
+
+type BoolObject struct {
+	Bool *MMSMObject `json:"bool,omitempty"`
+}
+
+type BoolObject1 struct {
+	Bool *MMSObject `json:"bool,omitempty"`
+}
+
+type MMSMObject struct {
+	Must       []interface{} `json:"must,omitempty"`
+	MustNot    []interface{} `json:"must_not,omitempty"`
+	Should     []interface{} `json:"should,omitempty"`
+	MinSdMatch string        `json:"minimum_should_match,omitempty"`
+}
+
+type MMSObject struct {
+	Must    []interface{} `json:"must,omitempty"`
+	MustNot []interface{} `json:"must_not,omitempty"`
+	Should  []interface{} `json:"should,omitempty"`
+}
+
+type ShouldObject struct {
+	MultiMatch *MultiMatch `json:"multi_match,omitempty"`
+}
+
+type MultiMatch struct {
+	Query  string   `json:"query,omitempty"`
+	Type   string   `json:"type,omitempty"`
+	Fields []string `json:"fields,omitempty"`
+}

proposed_project/proTask.go

@@ -0,0 +1,277 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"encoding/json"
+	"fmt"
+	"go.uber.org/zap"
+	"proposed_project/config"
+	"strings"
+	"sync"
+	"time"
+)
+
+func updateAllQueue() {
+	arru := make([][]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MgoPro.UpSertBulk(config.Conf.DB.MongoP.ProposedColl, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1 * time.Second):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MgoPro.UpSertBulk(config.Conf.DB.MongoP.ProposedColl, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func doTask(gtid, lteid string) {
+	defer util.Catch()
+
+	sess := MgoBid.GetMgoConn()
+	defer MgoBid.DestoryMongoConn(sess)
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	q := map[string]interface{}{"_id": map[string]interface{}{
+		"$gt":  mongodb.StringTOBsonId(gtid),
+		"$lte": mongodb.StringTOBsonId(lteid)}}
+	f := map[string]interface{}{
+		"contenthtml":  0,
+		"field_source": 0,
+		"nj_record":    0,
+		"kvtext":       0,
+	}
+	log.Info("doTask", zap.Any("q", q))
+	query := sess.DB(config.Conf.DB.MongoB.Dbname).C("nzj_bidding").Find(q).Select(f).Sort("publishtime").Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%5000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		//if util.ObjToString(tmp["toptype"]) != "拟建" {
+		//	continue
+		//}
+		ch <- true
+		wg.Add(1)
+		go func(temp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			taskTag(temp) // 增量数据打标签
+			info := ParseInfo(temp)
+			if info.ProjectName != "" {
+				MergeTask(temp, info)
+			} else {
+				log.Error("MergeTask", zap.Any("infoid", temp["_id"]))
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count), zap.String("LastId: ", LastId))
+	TaskSingle = true
+}
+
+func MergeTask(tmp map[string]interface{}, info *Info) {
+	if info.ApproveCode != "" && info.ApproveCode != "无" && info.ApproveCode != "暂无项目代码" {
+		AllCodeMapLock.Lock()
+		pid := AllCodeMap[info.ApproveCode]
+		AllCodeMapLock.Unlock()
+		if pid != "" {
+			AllPidMapLock.Lock()
+			res := AllPidMap[pid]
+			AllPidMapLock.Unlock()
+			if res != nil {
+				comparePro := res.P
+				updateProject(tmp, *info, comparePro)
+			} else {
+				startProjectMerge(tmp, info)
+			}
+		} else {
+			pid = startProjectMerge(tmp, info)
+			AllCodeMapLock.Lock()
+			AllCodeMap[info.ApproveCode] = pid
+			AllCodeMapLock.Unlock()
+		}
+	} else {
+		startProjectMerge(tmp, info)
+	}
+}
+
+func ParseInfo(tmp map[string]interface{}) (info *Info) {
+	bys, _ := json.Marshal(tmp)
+	var thisinfo *Info
+	_ = json.Unmarshal(bys, &thisinfo)
+	if thisinfo == nil {
+		return nil
+	}
+
+	if len(thisinfo.Topscopeclass) == 0 {
+		thisinfo.Topscopeclass = []string{}
+	}
+	if len(thisinfo.Subscopeclass) == 0 {
+		thisinfo.Subscopeclass = []string{}
+	}
+
+	return thisinfo
+}
+
+func taskTag(tmp map[string]interface{}) {
+	tag := taskFuc(tmp)
+	if tag["nature"] != "" {
+		tmp["nature_code"] = tag["nature"]
+	} else {
+		tmp["nature_code"] = "00"
+	}
+	if tag["project_stage"] != "" {
+		tmp["project_stage_code"] = tag["project_stage"]
+	} else {
+		tmp["project_stage_code"] = "00"
+	}
+	if tag["owner"] != "" {
+		tmp["ownerclass_code"] = tag["owner"]
+	} else {
+		tmp["ownerclass_code"] = "00"
+	}
+	if tag["sub_category"] != "" {
+		tmp["category_code"] = tag["sub_category"]
+	} else {
+		if tag["top_category"] != "" {
+			tmp["category_code"] = tag["top_category"]
+		}
+	}
+	if util.ObjToString(tmp["category_code"]) == "" {
+		tmp["category_code"] = "04"
+	}
+}
+
+func doTask1(gtid, lteid string) {
+	defer util.Catch()
+
+	sess := MgoBid.GetMgoConn()
+	defer MgoBid.DestoryMongoConn(sess)
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	q := map[string]interface{}{"_id": map[string]interface{}{
+		"$gt":  mongodb.StringTOBsonId(gtid),
+		"$lte": mongodb.StringTOBsonId(lteid)}}
+
+	log.Info("doTask", zap.Any("q", q))
+	query := sess.DB(config.Conf.DB.MongoB.Dbname).C("bidding").Find(q).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%20000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		if util.ObjToString(tmp["toptype"]) != "拟建" {
+			continue
+		}
+		ch <- true
+		wg.Add(1)
+		go func(temp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			//disField(temp)
+			savePool <- temp
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count))
+	TaskSingle = true
+}
+
+func disField(tmp map[string]interface{}) {
+	njInfo := map[string]int{
+		"approvecode":        1,
+		"approvedept":        1,
+		"approvestatus":      1,
+		"approvetime":        1,
+		"approvenumber":      1,
+		"approvecontent":     1,
+		"projecttype":        1,
+		"approvecity":        1,
+		"countryprojectcode": 1,
+		"total_investment":   1,
+		"owner":              1,
+		"projectperiod":      1,
+		"project_person":     1,
+		"projectaddr":        1,
+		"project_scale":      1,
+	}
+	code := util.ObjToString(tmp["spidercode"])
+	//只处理增量的lua采集的信息
+	if !strings.Contains(code, "_njpc") {
+
+		for f := range njInfo {
+			if tmp[f] != nil {
+				njInfo[f] = 0 //值为0表示该字段已抽取，不必再更新
+			}
+		}
+
+		//projectinfo
+		if projectinfo, ok := tmp["projectinfo"].(map[string]interface{}); ok && len(projectinfo) > 0 {
+			flag := false
+			for f, n := range njInfo {
+				if n == 1 && projectinfo[f] != nil {
+					tmp[f] = projectinfo[f]
+					flag = true
+				}
+			}
+			if !flag { //嵌套信息
+				tmpTime := ""
+				tmpK := "" //记录最终取哪个map
+				for k, v := range projectinfo {
+					if result, ok := v.(map[string]interface{}); ok && len(result) > 0 && k != "attachments" {
+						approvetime := util.ObjToString(result["approvetime"])
+						if tmpTime == "" || approvetime >= tmpTime { //取最新时间信息
+							tmpTime = approvetime
+							tmpK = k
+						}
+					}
+				}
+				if tmpK != "" {
+					if resultTmp, ok := projectinfo[tmpK].(map[string]interface{}); ok && len(resultTmp) > 0 {
+						for f, n := range njInfo {
+							if n == 1 && resultTmp[f] != nil {
+								tmp[f] = resultTmp[f]
+							}
+						}
+					}
+				}
+			}
+		}
+		//buyer
+		if buyer := tmp["buyer"]; buyer != nil && tmp["owner"] == nil {
+			tmp["owner"] = buyer
+		}
+	}
+}

proposed_project/projectTask.go

@@ -0,0 +1,894 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/redis"
+	"encoding/json"
+	"fmt"
+	"github.com/go-ego/gse"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.uber.org/zap"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"proposed_project/config"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+	"unicode/utf8"
+)
+
+var (
+	seg       gse.Segmenter
+	stopWords []string
+
+	regNum  = regexp.MustCompile(`[\dA-Za-z]{6,30}`)
+	regSymb = regexp.MustCompile("[•、,，.。?？'\"“”‘’·~！@#￥$%…&*（）()\\-—+=【】\\[\\]｛｝{}<>《》|\\/\\s]+")
+	regDel  = regexp.MustCompile("项目|工程|生产|中心")
+	reg1    = regexp.MustCompile("分布式光伏发电|自然人|出让|国有建设用地使用权")
+
+	field  = []string{"projectname.pname"}
+	sField = []string{"projectname", "bidstatus", "firsttime", "_id", "area", "city", "district", "ids"}
+
+	esQ = `{"query": {"bool": {"must": [{"multi_match": {"query": "%s","type": "phrase","fields": ["projectname"]}}]}},"size": 100}`
+
+	savePpPool = make(chan map[string]interface{}, 5000)
+	savePpSp   = make(chan bool, 3)
+)
+
+func initSeg() {
+	_ = seg.LoadDict("./t_1.txt")
+	//_ = seg.LoadDict()
+	seg.AddToken("渼陂", 3, "")
+	seg.LoadStop("./stopwords.txt")
+
+	//f, _ := os.Open("./stopwords.txt")
+	//defer f.Close()
+	//scanner := bufio.NewScanner(f)
+	//for scanner.Scan() {
+	//	stopWords = append(stopWords, scanner.Text())
+	//}
+	//sort.Strings(stopWords)
+}
+
+// @Description 关联
+// @Author J 2023/4/25 10:26
+func taskComb() {
+	sess := MgoPro.GetMgoConn()
+	defer MgoPro.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	f := map[string]interface{}{
+		"projectname":   1,
+		"approvecode":   1,
+		"approvenumber": 1,
+		"approvestatus": 1,
+		"area":          1,
+		"city":          1,
+		"district":      1,
+		"ids":           1,
+	}
+
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(nil).Select(f).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%2000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			if filterMethod(tmp) {
+				return
+			}
+
+			var mArr []map[string]interface{}
+			var eArr []map[string]interface{}
+			n1, n2 := 0, 0
+			// approvecode、approvenumber
+			//q := Method2(util.ObjToString(tmp["approvecode"]), util.ObjToString(tmp["approvenumber"]))
+			//if q != "" {
+			//	binfo := Es.Get("projectset", q)
+			//	if binfo != nil && len(*binfo) > 0 {
+			//		for _, m := range *binfo {
+			//			n1 = len(*binfo)
+			//			mArr = append(mArr, map[string]interface{}{"pid": util.ObjToString(m["_id"]), "projectname": m["projectname"], "source": 1})
+			//		}
+			//	}
+			//}
+
+			wds, q := Method1(util.ObjToString(tmp["projectname"]))
+			if q != "" {
+				binfo := Es.Get("projectset", q)
+				if binfo != nil && len(*binfo) > 0 {
+					n2 = len(*binfo)
+					for _, m := range *binfo {
+						if b, _ := redis.Exists(config.Conf.DB.Redis.Project, util.ObjToString(m["_id"])); b {
+							continue
+						}
+						if util.ObjToString(m["bidstatus"]) == "拟建" {
+							eArr = append(eArr, map[string]interface{}{"pid": util.ObjToString(m["_id"]), "projectname": m["projectname"], "bidstatus": m["bidstatus"]})
+							continue
+						}
+						if judgeArea(tmp, m, wds) {
+							eArr = append(eArr, map[string]interface{}{"pid": util.ObjToString(m["_id"]), "projectname": m["projectname"], "bidstatus": m["bidstatus"]})
+							continue
+						}
+
+						size := len(m["ids"].([]interface{}))
+						redis.PutCKV(config.Conf.DB.Redis.Project, util.ObjToString(m["_id"]), fmt.Sprintf("%s-%d", mongodb.BsonIdToSId(tmp["_id"]), size))
+						redis.PutCKV(config.Conf.DB.Redis.Proposed, mongodb.BsonIdToSId(tmp["_id"]), len(tmp["ids"].([]interface{})))
+						mArr = append(mArr, map[string]interface{}{"pid": util.ObjToString(m["_id"]), "projectname": m["projectname"], "source": 2})
+
+					}
+				}
+			}
+			//if mArr != nil && len(mArr) > 0 {
+			save := make(map[string]interface{})
+			save["_id"] = tmp["_id"]
+			save["ids"] = mArr
+			save["wds"] = wds
+			save["err"] = eArr
+			save["projectname"] = tmp["projectname"]
+			save["esearch"] = q
+			save["size_1"] = n1
+			save["size_2"] = n2
+			save["createtime"] = time.Now().Unix()
+			savePpPool <- save
+			//}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count))
+}
+
+func Method(pname string) string {
+	pname = regNum.ReplaceAllString(pname, "")
+	pname = regSymb.ReplaceAllString(pname, "")
+	//wds := seg.CutStop(pname, true)
+
+	util.Debug(pname)
+
+	surl := config.Conf.DB.Es.Addr + "/projectset_v2/_analyze"
+	URL, _ := url.Parse(surl)
+	Q := URL.Query()
+	Q.Add("pretty", "1")
+	Q.Add("analyzer", "my_ngram_title")
+	Q.Add("text", pname)
+	URL.RawQuery = Q.Encode()
+	resp, err := http.Get(URL.String())
+	if err != nil {
+		log.Info("")
+	}
+	result, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Info("")
+	}
+	var resMap map[string]interface{}
+	json.Unmarshal(result, &resMap)
+	if resMap == nil || len(resMap["tokens"].([]interface{})) == 0 {
+		log.Info("")
+	}
+	tokens := util.ObjArrToMapArr(resMap["tokens"].([]interface{}))
+	var should []interface{}
+	for _, t := range tokens {
+
+		wd := util.ObjToString(t["token"])
+		if in(wd) {
+			// 删除词 跳过
+			continue
+		}
+		mm := &ShouldObject{MultiMatch: &MultiMatch{
+			Query:  wd,
+			Type:   "phrase",
+			Fields: field,
+		}}
+		should = append(should, mm)
+	}
+	query := &QueryObject{Query: &BoolObject{Bool: &MMSMObject{Must: should}}}
+	jsr, err := json.Marshal(query)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+
+	return string(jsr)
+
+}
+
+//@Description jieba分词查询
+// @Desc projectname
+// @Desc minimum_should_match 当should分支总数小于等于指定的数量时，则必须匹配所有should分支，当should分支总数大于指定的数量时，则应用指定的说明符
+// @Author J 2023/3/7 17:13
+func Method1(pname string) ([]string, string) {
+	if pname == "" {
+		return nil, ""
+	}
+	var wds []string
+	p1 := regSymb.ReplaceAllString(pname, "")
+	p1 = regNum.ReplaceAllString(p1, "")
+	if utf8.RuneCountInString(p1) < 12 {
+		wds = append(wds, pname)
+	} else {
+		wds = seg.CutStop(pname, true)
+	}
+	if len(wds) > 0 {
+
+		wds = combArr(wds)
+		var should []interface{}
+		for _, t := range wds {
+			//if utf8.RuneCountInString(t) == 1 {
+			//	continue
+			//}
+			mm := &ShouldObject{MultiMatch: &MultiMatch{
+				Query:  t,
+				Type:   "phrase",
+				Fields: field,
+			}}
+			should = append(should, mm)
+		}
+		if should == nil || len(should) <= 0 {
+			return nil, ""
+		}
+		query := &QueryObject{Query: &BoolObject{Bool: &MMSMObject{Should: should, MinSdMatch: config.Conf.DB.Es.MinSdMh}}, Source: sField, Size: 1000}
+		jsr, err := json.Marshal(query)
+		if err != nil {
+			fmt.Printf("Err %v", err)
+			os.Exit(1)
+		}
+
+		return wds, strings.Replace(string(jsr), "\\u003c", "<", -1)
+	} else {
+		return nil, ""
+	}
+}
+
+// @Description
+// @Author J 2023/3/20 14:39
+func Method2(acode, anumb string) string {
+
+	if acode == "" && anumb == "" {
+		return ""
+	}
+	var should []interface{}
+	if acode != "" {
+		should = append(should, &ShouldObject{MultiMatch: &MultiMatch{
+			Query:  acode,
+			Type:   "phrase",
+			Fields: []string{"detail"},
+		}})
+	}
+	if anumb != "" {
+		should = append(should, &ShouldObject{MultiMatch: &MultiMatch{
+			Query:  anumb,
+			Type:   "phrase",
+			Fields: []string{"detail"},
+		}})
+	}
+	query := &QueryObject1{Query: &BoolObject1{Bool: &MMSObject{Should: should}}, Source: sField, Size: 500}
+	jsr, err := json.Marshal(query)
+	if err != nil {
+		fmt.Printf("Err %v", err)
+		os.Exit(1)
+	}
+	return string(jsr)
+}
+
+func judgeArea(tmp, btmp map[string]interface{}, wds []string) bool {
+	if util.ObjToString(tmp["area"]) == "全国" || util.ObjToString(btmp["area"]) == "全国" {
+		pname := util.ObjToString(btmp["projectname"])
+		for _, wd := range wds {
+			if !strings.Contains(strings.ToLower(pname), strings.ToLower(wd)) {
+				return true
+			}
+		}
+		return false
+	}
+	if tmp["district"] != nil && btmp["district"] != nil {
+		if util.ObjToString(tmp["district"]) == util.ObjToString(btmp["district"]) {
+			return false
+		} else {
+			return true
+		}
+	} else if tmp["city"] != nil && btmp["city"] != nil {
+		if util.ObjToString(tmp["city"]) == util.ObjToString(btmp["city"]) {
+			return false
+		} else {
+			return true
+		}
+	} else if tmp["area"] != nil && btmp["area"] != nil {
+		if util.ObjToString(tmp["area"]) == util.ObjToString(btmp["area"]) {
+			return false
+		} else {
+			return true
+		}
+	}
+	return false
+}
+
+var regs = []*regexp.Regexp{
+	regexp.MustCompile("[\\\\u4e00-\\\\u9fa5]{0,10}私宅"),
+	regexp.MustCompile("自然人|分布式光伏发电"),
+}
+
+// @Description 过滤数据
+// @Author J 2023/3/24 09:30
+func filterMethod(tmp map[string]interface{}) bool {
+	pname := regSymb.ReplaceAllString(util.ObjToString(tmp["projectname"]), "")
+	pname = regNum.ReplaceAllString(pname, "")
+	p1 := regDel.ReplaceAllString(pname, "")
+	p1 = regSymb.ReplaceAllString(p1, "")
+	if utf8.RuneCountInString(p1) <= 5 {
+		return true
+	}
+	if len(reg1.FindAllString(pname, -1)) > 1 {
+		return true
+	}
+	for _, reg := range regs {
+		if reg.MatchString(util.ObjToString(tmp["projectname"])) {
+			return true
+		}
+	}
+	return false
+}
+
+func combArr(arr []string) []string {
+	var nArr []string
+	for i := 0; i < len(arr); i++ {
+		if i == len(arr)-1 {
+			if utf8.RuneCountInString(arr[i]) == 1 {
+				nArr[len(nArr)-1] += arr[i]
+			} else {
+				nArr = append(nArr, arr[i])
+			}
+		} else {
+			if utf8.RuneCountInString(arr[i]) == 1 {
+				if i == 0 {
+					nArr = append(nArr, arr[i])
+				} else {
+					nArr[len(nArr)-1] += arr[i]
+					//if utf8.RuneCountInString(arr[i+1]) == 1 {
+					//	if utf8.RuneCountInString(nArr[len(nArr)-1]) == 1 {
+					//		nArr[len(nArr)-1] += arr[i]
+					//	} else {
+					//		nArr = append(nArr, arr[i])
+					//	}
+					//} else {
+					//	nArr[len(nArr)-1] += arr[i]
+					//}
+				}
+			} else {
+				if len(nArr) > 0 && utf8.RuneCountInString(nArr[len(nArr)-1]) == 1 {
+					nArr[len(nArr)-1] += arr[i]
+				} else {
+					nArr = append(nArr, arr[i])
+				}
+			}
+		}
+	}
+
+	return nArr
+}
+
+func SavePpMethod() {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-savePpPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				savePpSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-savePpSp
+					}()
+					MgoPro.SaveBulk(config.Conf.DB.MongoP.CombColl, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				savePpSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-savePpSp
+					}()
+					MgoPro.SaveBulk(config.Conf.DB.MongoP.CombColl, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+var StageCode []TagMatching
+
+func initStage() {
+	info, _ := MgoBid.Find(config.Conf.Serve.TagRule, bson.M{"label_name": "project_stage"}, `{"_id": 1}`, nil, false, -1, -1)
+	for _, m := range *info {
+		tag := TagMatching{}
+		tag.tagName = util.ObjToString(m["label_name"])
+		tag.tagCode = util.ObjToString(m["code"])
+		// 关键词
+		tag.matchField = []string{"title", "project"}
+		if v := util.ObjToString(m["keyword"]); v != "" {
+			tag.matchKey = util.ObjToString(m["keyword"])
+			tag.matchKeyReg = GetRegex(util.ObjToString(m["keyword"]))
+		}
+		// 附件词
+		if f := util.ObjToString(m["match_fjword"]); f != "" {
+			tag.addField = strings.Split(f, ",")
+			for _, s := range tag.addField {
+				SelectF[s] = 1
+			}
+			if v := util.ObjToString(m["fjword"]); v != "" {
+				tag.addKey = util.ObjToString(m["fjword"])
+				tag.addKeyReg = GetRegex(util.ObjToString(m["fjword"]))
+			}
+		}
+		// 排除词
+		if f := util.ObjToString(m["match_pcword"]); f != "" {
+			tag.excludeField = strings.Split(f, ",")
+			for _, s := range tag.excludeField {
+				SelectF[s] = 1
+			}
+			if v := util.ObjToString(m["pcword"]); v != "" {
+				tag.excludeKey = util.ObjToString(m["pcword"])
+				tag.excludeKeyReg = GetRegex(util.ObjToString(m["pcword"]))
+			}
+		}
+		// 清理词
+		if v := util.ObjToString(m["qlword"]); v != "" {
+			tag.clearKey = strings.Split(util.ObjToString(m["qlword"]), ",")
+		}
+		StageCode = append(StageCode, tag)
+	}
+	log.Info("initStage", zap.Int("StageCode", len(StageCode)))
+}
+
+func taskD() {
+	sess := MgoPro.GetMgoConn()
+	defer MgoPro.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	//q := bson.M{"_id": mongodb.StringTOBsonId("60a2995b8a2adb30a57172ec")}
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.CombColl).Find(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%2000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+
+			if ids, ok := tmp["ids"].([]interface{}); ok {
+				id := mongodb.BsonIdToSId(tmp["_id"])
+
+				for _, p := range ids {
+					p1 := p.(map[string]interface{})
+					info, _ := MgoPro.FindById(config.Conf.DB.MongoP.ProjectColl, util.ObjToString(p1["pid"]), nil)
+					if list, ok1 := (*info)["list"].([]interface{}); ok1 {
+						for _, l := range list {
+							l1 := l.(map[string]interface{})
+							m := make(map[string]interface{})
+							m["project_stage_code"] = tagFunc(l1)
+							m["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+							m["title"] = util.ObjToString(l1["title"])
+							if t := util.Int64All(l1["publishtime"]); t > 0 {
+								m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+							}
+							m["infoid"] = util.ObjToString(l1["infoid"])
+							m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(l1["infoid"])))
+							m["createtime"] = time.Now().Format(util.Date_Full_Layout)
+							//MgoPro.Save("projectset_comb_temp1", m)
+							MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+						}
+					}
+					if buyer := util.ObjToString((*info)["buyer"]); buyer != "" {
+						s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": buyer})
+						if s <= 0 {
+							saveEnt := make(map[string]interface{})
+							saveEnt["proposed_id"] = id
+							saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+							saveEnt["name"] = buyer
+							if eid := redis.GetStr("ent_id", buyer); eid != "" {
+								arr := strings.Split(eid, "_")
+								saveEnt["name_id"] = arr[0]
+								if len(arr) == 2 {
+									saveEnt["area_code"] = arr[1]
+								} else if len(arr) == 3 {
+									saveEnt["city_code"] = arr[2]
+								}
+								info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
+								if info != nil && len(*info) > 0 {
+									saveEnt["address"] = (*info)[0]["address"]
+								}
+							}
+							saveEnt["identity_type"] = 2
+							saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+							saveEntPool1 <- saveEnt
+						}
+					}
+					if winner := util.ObjToString((*info)["buyer"]); winner != "" {
+						for _, w := range strings.Split(winner, ",") {
+							s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": w})
+							if s <= 0 {
+								saveEnt := make(map[string]interface{})
+								saveEnt["proposed_id"] = id
+								saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+								saveEnt["name"] = w
+								if eid := redis.GetStr("ent_id", w); eid != "" {
+									arr := strings.Split(eid, "_")
+									saveEnt["name_id"] = arr[0]
+									if len(arr) == 2 {
+										saveEnt["area_code"] = arr[1]
+									} else if len(arr) == 3 {
+										saveEnt["city_code"] = arr[2]
+									}
+									info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
+									if info != nil && len(*info) > 0 {
+										saveEnt["address"] = (*info)[0]["address"]
+									}
+								}
+								saveEnt["identity_type"] = 3
+								saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+								saveEntPool1 <- saveEnt
+							}
+						}
+					}
+				}
+
+				size := MysqlTool.Count("dwd_f_nzj_follw_record", bson.M{"proposed_id": id})
+				info := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"proposed_id": id}, "project_stage_code", "publishtime desc")
+				MysqlTool.Update("dwd_f_nzj_baseinfo", bson.M{"proposed_id": id}, bson.M{"follow_num": size, "project_stage_code": (*info)["project_stage_code"], "updatetime": time.Now().Format(util.Date_Full_Layout)})
+			}
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count))
+}
+
+// @Description 施工准备(06)、施工(07)、设计(05)
+// @Author J 2023/4/21 14:45
+func tagFunc(info map[string]interface{}) string {
+	tag := taskFuc1(info)
+	if tag["project_stage"] != "" {
+		return util.ObjToString(tag["project_stage"])
+	}
+	if util.ObjToString(info["toptype"]) == "招标" || util.ObjToString(info["toptype"]) == "预告" {
+		return "06"
+	}
+	return "00"
+}
+
+// @Description 在建项目增量
+// @Author J 2023/4/24 13:58
+func taskAA() {
+	sess := MgoPro.GetMgoConn()
+	defer MgoPro.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	q := bson.M{"pici": bson.M{"$gte": config.Conf.Serve.Pici}}
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProjectColl).Find(q).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%2000 == 0 {
+			log.Info(fmt.Sprintf("project current --- %d", count))
+		}
+		if pc := util.Int64All(tmp["pici"]); pc > config.Conf.Serve.Pici {
+			config.Conf.Serve.Pici = pc
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			bUpdate := false
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			if str := redis.GetStr(config.Conf.DB.Redis.Project, id); str != "" {
+				strs := strings.Split(str, "-")
+				size := len(tmp["list"].([]interface{}))
+				if size != util.IntAll(strs[1]) {
+					list := tmp["list"].([]interface{})
+					for k := range list {
+						info1 := list[size-k-1].(map[string]interface{}) //倒序
+						infoid := util.ObjToString(info1["infoid"])
+						binfo := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"infoid": infoid}, "", "")
+						if binfo != nil && len(*binfo) > 0 {
+							break
+						} else {
+							m := make(map[string]interface{})
+							m["project_stage_code"] = tagFunc(info1)
+							m["proposed_id"] = strs[0]
+							m["title"] = util.ObjToString(info1["title"])
+							if t := util.Int64All(info1["publishtime"]); t > 0 {
+								m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+							}
+							m["infoid"] = util.ObjToString(info1["infoid"])
+							m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(info1["infoid"])))
+							m["createtime"] = time.Now().Format(util.Date_Full_Layout)
+							MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+							bUpdate = true
+						}
+					}
+					redis.PutCKV(config.Conf.DB.Redis.Project, id, fmt.Sprintf("%s-%d", strs[0], size))
+				}
+			} else {
+				// 新项目
+				// 新项目是否都有必要进行关联(拟建项目)
+				bUpdate = Method3(tmp)
+			}
+			// 更新拟在建基本信息表
+			if bUpdate {
+				size := MysqlTool.Count("dwd_f_nzj_follw_record", bson.M{"proposed_id": id})
+				info := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"proposed_id": id}, "project_stage_code", "publishtime desc")
+				if info != nil && len(*info) > 0 {
+					MysqlTool.Update("dwd_f_nzj_baseinfo", bson.M{"proposed_id": id}, bson.M{"follow_num": size, "project_stage_code": (*info)["project_stage_code"], "updatetime": time.Now().Format(util.Date_Full_Layout)})
+				}
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("project over --- %d, pici ---%d", count, config.Conf.Serve.Pici))
+}
+
+// @Description 拟建项目增量
+// @Author J 2023/4/24 13:59
+func taskBB() {
+	sess := MgoPro.GetMgoConn()
+	defer MgoPro.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	q := bson.M{"pici": bson.M{"$gte": config.Conf.Serve.Pici}}
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(q).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%2000 == 0 {
+			log.Info(fmt.Sprintf("proposed current --- %d", count))
+		}
+		if pc := util.Int64All(tmp["pici"]); pc > config.Conf.Serve.Pici {
+			config.Conf.Serve.Pici = pc
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			bUpdate := false
+			id := mongodb.BsonIdToSId(tmp["_id"])
+			if num := redis.GetInt(config.Conf.DB.Redis.Proposed, id); num > 0 {
+				size := len(tmp["list"].([]interface{}))
+				list := tmp["list"].([]interface{})
+				if num != size {
+					for k := range list {
+						info1 := list[size-k-1].(map[string]interface{})
+						infoid := util.ObjToString(info1["infoid"])
+						binfo := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"infoid": infoid}, "", "")
+						if binfo != nil && len(*binfo) > 0 {
+							break
+						} else {
+							m := make(map[string]interface{})
+							m["project_stage_code"] = tagFunc(info1)
+							m["proposed_id"] = id
+							m["title"] = util.ObjToString(info1["title"])
+							if t := util.Int64All(info1["publishtime"]); t > 0 {
+								m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+							}
+							m["infoid"] = util.ObjToString(info1["infoid"])
+							m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(info1["infoid"])))
+							m["createtime"] = time.Now().Format(util.Date_Full_Layout)
+							MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+							bUpdate = true
+						}
+					}
+					redis.PutCKV(config.Conf.DB.Redis.Proposed, id, size)
+				}
+			} else {
+				// 新项目
+				// 1、关联
+				wds, q := Method1(util.ObjToString(tmp["projectname"]))
+				if q != "" {
+					binfo := Es.Get("projectset", q)
+					if binfo != nil && len(*binfo) > 0 {
+						for _, m := range *binfo {
+							if b, _ := redis.Exists(config.Conf.DB.Redis.Project, util.ObjToString(m["_id"])); b {
+								continue
+							}
+							if util.ObjToString(m["bidstatus"]) == "拟建" {
+								continue
+							}
+							if judgeArea(tmp, m, wds) {
+								continue
+							}
+
+							size := len(m["ids"].([]interface{}))
+							redis.PutCKV(config.Conf.DB.Redis.Project, util.ObjToString(m["_id"]), fmt.Sprintf("%s-%d", mongodb.BsonIdToSId(tmp["_id"]), size))
+							redis.PutCKV(config.Conf.DB.Redis.Proposed, mongodb.BsonIdToSId(tmp["_id"]), len(tmp["ids"].([]interface{})))
+
+							bUpdate = true
+							// 2、保存信息到tidb
+							if list, ok1 := m["list"].([]interface{}); ok1 {
+								for _, l := range list {
+									l1 := l.(map[string]interface{})
+									m := make(map[string]interface{})
+									m["project_stage_code"] = tagFunc(l1)
+									m["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+									m["title"] = util.ObjToString(l1["title"])
+									if t := util.Int64All(l1["publishtime"]); t > 0 {
+										m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+									}
+									m["infoid"] = util.ObjToString(l1["infoid"])
+									m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(l1["infoid"])))
+									m["createtime"] = time.Now().Format(util.Date_Full_Layout)
+									MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+								}
+							}
+							if buyer := util.ObjToString(m["buyer"]); buyer != "" {
+								s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": buyer})
+								if s <= 0 {
+									saveEnt := make(map[string]interface{})
+									saveEnt["proposed_id"] = id
+									saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+									saveEnt["name"] = buyer
+									if eid := redis.GetStr("ent_id", buyer); eid != "" {
+										arr := strings.Split(eid, "_")
+										saveEnt["name_id"] = arr[0]
+										if len(arr) == 2 {
+											saveEnt["area_code"] = arr[1]
+										} else if len(arr) == 3 {
+											saveEnt["city_code"] = arr[2]
+										}
+										info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
+										if info != nil && len(*info) > 0 {
+											saveEnt["address"] = (*info)[0]["address"]
+										}
+									}
+									saveEnt["identity_type"] = 2
+									saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+									saveEntPool1 <- saveEnt
+								}
+							}
+							if winner := util.ObjToString(m["buyer"]); winner != "" {
+								for _, w := range strings.Split(winner, ",") {
+									s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": w})
+									if s <= 0 {
+										saveEnt := make(map[string]interface{})
+										saveEnt["proposed_id"] = id
+										saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+										saveEnt["name"] = w
+										if eid := redis.GetStr("ent_id", w); eid != "" {
+											arr := strings.Split(eid, "_")
+											saveEnt["name_id"] = arr[0]
+											if len(arr) == 2 {
+												saveEnt["area_code"] = arr[1]
+											} else if len(arr) == 3 {
+												saveEnt["city_code"] = arr[2]
+											}
+											info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
+											if info != nil && len(*info) > 0 {
+												saveEnt["address"] = (*info)[0]["address"]
+											}
+										}
+										saveEnt["identity_type"] = 3
+										saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+										saveEntPool1 <- saveEnt
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+			// 更新拟在建基本信息表
+			if bUpdate {
+				size := MysqlTool.Count("dwd_f_nzj_follw_record", bson.M{"proposed_id": id})
+				info := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"proposed_id": id}, "project_stage_code", "publishtime desc")
+				MysqlTool.Update("dwd_f_nzj_baseinfo", bson.M{"proposed_id": id}, bson.M{"follow_num": size, "project_stage_code": (*info)["project_stage_code"], "updatetime": time.Now().Format(util.Date_Full_Layout)})
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("proposed over --- %d, pici ---%d", count, config.Conf.Serve.Pici))
+}
+
+func Method3(tmp map[string]interface{}) bool {
+	pname := util.ObjToString(tmp["projectname"])
+	pname = strings.ReplaceAll(pname, "\"", "'")
+	binfo := Es.Get("proposed", fmt.Sprintf(esQ, pname))
+	if binfo != nil && len(*binfo) > 0 {
+		if list, ok1 := (*binfo)[0]["list"].([]interface{}); ok1 {
+			for _, l := range list {
+				l1 := l.(map[string]interface{})
+				m := make(map[string]interface{})
+				m["project_stage_code"] = tagFunc(l1)
+				m["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+				m["title"] = util.ObjToString(l1["title"])
+				if t := util.Int64All(l1["publishtime"]); t > 0 {
+					m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+				}
+				m["infoid"] = util.ObjToString(l1["infoid"])
+				m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(l1["infoid"])))
+				m["createtime"] = time.Now().Format(util.Date_Full_Layout)
+				MysqlTool.Insert("dwd_f_nzj_follw_record", m)
+			}
+			redis.PutCKV(config.Conf.DB.Redis.Project, mongodb.BsonIdToSId(tmp["_id"]), fmt.Sprintf("%s-%d", util.ObjToString((*binfo)[0]["_id"]), len(list)))
+			return true
+		}
+	}
+	return false
+}
+
+var saveEntPool1 = make(chan map[string]interface{}, 5000)
+var saveEntSp1 = make(chan bool, 1)
+
+func SaveEntFunc1(table string, arr []string) {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveEntPool1:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveEntSp1 <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEntSp1
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveEntSp1 <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEntSp1
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

proposed_project/project_tool.go

@@ -0,0 +1,61 @@
+package main
+
+//项目中的字段
+var FIELDS = []string{
+	"area",
+	"city",
+	"district",
+	"projectname",
+	"approvecode",
+	"approvenumber",
+	"approvedept",
+	"total_investment",
+	"funds",
+	"owner",
+	"projectaddr",
+	"projectperiod",
+	"project_startdate",
+	"project_completedate",
+	"approvestatus",
+	"project_scale",
+	"construction_area",
+	"floor_area",
+	"ownerclass_code",
+	"category_code",
+	"nature_code",
+	"project_stage_code",
+	"project_person",
+	"project_phone",
+}
+
+var INFOFIELDS = []string{
+	"area",
+	"city",
+	"district",
+	"title",
+	"site",
+	"href",
+	"projectname",
+	"approvecode",
+	"approvenumber",
+	"total_investment",
+	"funds",
+	"owner",
+	"ownerclass_code",
+	"projectaddr",
+	"projectperiod",
+	"approvestatus",
+	"project_scale",
+	"construction_area",
+	"floor_area",
+	"project_startdate",
+	"project_completedate",
+	"publishtime",
+	"comeintime",
+	"ownerclass_code",
+	"category_code",
+	"nature_code",
+	"project_stage_code",
+	"project_person",
+	"project_phone",
+}

proposed_project/stopwords.txt

@@ -0,0 +1,62 @@
+•
+、
+,
+，
+.
+。
+?
+？
+'
+"
+“
+”
+‘
+’
+·
+~
+！
+@
+#
+￥
+$
+%
+…
+&
+*
+（
+）
+(
+)
+\
+-
+—
++
+=
+【
+】
+[
+]
+｛
+｝
+{
+}
+<
+>
+《
+》
+|
+项目
+工程
+实施
+方案
+获批
+核准
+备案
+生产
+学校
+学院
+及
+仅
+做
+赋码
+用

proposed_project/tagTask.go

@@ -0,0 +1,378 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"fmt"
+	"go.uber.org/zap"
+	"proposed_project/config"
+	"strings"
+	"sync"
+	"time"
+)
+
+var TagField = map[string]string{
+	"owner": "ownerclass",
+	//"projecttype":   "",
+	"top_category":  "category",
+	"sub_category":  "category",
+	"nature":        "nature",
+	"project_stage": "project_stage",
+}
+
+var (
+	//MatchArr []TagMatching
+
+	//不同标签 规则
+	ruleMatch = make(map[string][]TagMatching, 0)
+
+	SelectF = make(map[string]int)
+)
+
+func InitRule() {
+	info, _ := MgoBid.Find(config.Conf.Serve.TagRule, nil, `{"_id": 1}`, nil, false, -1, -1)
+	for _, m := range *info {
+		tag := TagMatching{}
+		tag.tagName = util.ObjToString(m["label_name"])
+		tag.tagCode = util.ObjToString(m["code"])
+		// 关键词
+		if f := util.ObjToString(m["match_keyword"]); f != "" {
+			tag.matchField = strings.Split(f, ",")
+			for _, s := range tag.matchField {
+				SelectF[s] = 1
+			}
+			if v := util.ObjToString(m["keyword"]); v != "" {
+				tag.matchKey = util.ObjToString(m["keyword"])
+				tag.matchKeyReg = GetRegex(util.ObjToString(m["keyword"]))
+			}
+		}
+		// 附件词
+		if f := util.ObjToString(m["match_fjword"]); f != "" {
+			tag.addField = strings.Split(f, ",")
+			for _, s := range tag.addField {
+				SelectF[s] = 1
+			}
+			if v := util.ObjToString(m["fjword"]); v != "" {
+				tag.addKey = util.ObjToString(m["fjword"])
+				tag.addKeyReg = GetRegex(util.ObjToString(m["fjword"]))
+			}
+		}
+		// 排除词
+		if f := util.ObjToString(m["match_pcword"]); f != "" {
+			tag.excludeField = strings.Split(f, ",")
+			for _, s := range tag.excludeField {
+				SelectF[s] = 1
+			}
+			if v := util.ObjToString(m["pcword"]); v != "" {
+				tag.excludeKey = util.ObjToString(m["pcword"])
+				tag.excludeKeyReg = GetRegex(util.ObjToString(m["pcword"]))
+			}
+		}
+		// 清理词
+		if v := util.ObjToString(m["qlword"]); v != "" {
+			tag.clearKey = strings.Split(util.ObjToString(m["qlword"]), ",")
+		}
+
+		rules, _ := ruleMatch[tag.tagName]
+		rules = append(rules, tag)
+		ruleMatch[tag.tagName] = rules
+	}
+	for K, v := range ruleMatch {
+		log.Info("InitRule", zap.Int(K, len(v)))
+	}
+}
+
+func taskRun() {
+	sess := MgoPro.GetMgoConn()
+	defer MgoPro.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(nil).Select(SelectF).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%20000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			tag := taskFuc(tmp)
+			update := make(map[string]interface{})
+			//项目性质
+			if tag["nature"] != "" {
+				update["nature_code"] = tag["nature"]
+				tmp["nature_code"] = tag["nature"]
+			} else {
+				update["nature_code"] = "00"
+				tmp["nature_code"] = "00"
+			}
+			//项目阶段
+			if tag["project_stage"] != "" {
+				update["project_stage_code"] = tag["project_stage"]
+				tmp["project_stage_code"] = tag["project_stage"]
+			} else {
+				update["project_stage_code"] = "00"
+				tmp["project_stage_code"] = "00"
+			}
+			//业主类型
+			if tag["owner"] != "" {
+				update["ownerclass_code"] = tag["owner"]
+				tmp["ownerclass_code"] = tag["owner"]
+			} else {
+				update["ownerclass_code"] = "00"
+				tmp["ownerclass_code"] = "00"
+			}
+			//项目类别
+			if tag["sub_category"] != "" {
+				update["category_code"] = tag["sub_category"]
+				tmp["category_code"] = tag["sub_category"]
+			} else {
+				if tag["top_category"] != "" {
+					update["category_code"] = tag["top_category"]
+					tmp["category_code"] = tag["top_category"]
+				}
+			}
+			if util.ObjToString(update["category_code"]) == "" {
+				update["category_code"] = "04"
+				tmp["category_code"] = "04"
+			}
+			//updatePool <- []map[string]interface{}{
+			//	{"_id": tmp["_id"]},
+			//	{"$set": update},
+			//}
+			savePool <- tmp
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count))
+}
+
+func taskFuc(tmp map[string]interface{}) map[string]string {
+	tag := make(map[string]string) // 打上的标签
+
+	for _, MatchArr := range ruleMatch {
+		for _, v := range MatchArr {
+			// 同个类型的标签如果存在，就不需要再打
+			if tag[v.tagName] != "" {
+				continue
+			}
+			// 排除词
+			if len(v.excludeField) > 0 && len(v.excludeKeyReg) > 0 {
+				for _, f := range v.excludeField {
+					if val := util.ObjToString(tmp[f]); val != "" {
+						for _, e1 := range v.excludeKeyReg {
+							flag := false
+							if e1.regs != nil && e1.regs.MatchString(val) {
+								flag = true
+							} else {
+								// && 特殊处理
+								if strings.Contains(e1.keyStr, "&&") {
+									for _, s := range strings.Split(e1.keyStr, "&&") {
+										if strings.Contains(val, s) {
+											flag = true
+											break
+										}
+									}
+								}
+							}
+							if flag {
+								goto L
+							}
+						}
+					}
+				}
+			}
+			// 清理词
+			if len(v.clearKey) > 0 && len(v.matchField) > 0 {
+				for _, s := range v.clearKey {
+					for _, f := range v.matchField {
+						if val := util.ObjToString(tmp[f]); val != "" {
+							tmp[f] = strings.ReplaceAll(val, s, "")
+						}
+					}
+				}
+			}
+			// 关键词
+			if len(v.matchField) > 0 && len(v.matchKeyReg) > 0 {
+				for _, f := range v.matchField {
+					if val := util.ObjToString(tmp[f]); val != "" {
+						for _, r1 := range v.matchKeyReg {
+							if r1.regs.MatchString(val) {
+								if len(v.addField) > 0 && len(v.addKeyReg) > 0 {
+									// 匹配附加词
+									isCt := false
+									for _, f1 := range v.addField {
+										if v1 := util.ObjToString(tmp[f1]); v1 != "" {
+											for _, r2 := range v.addKeyReg {
+												if r2.regs != nil && r2.regs.MatchString(v1) {
+													isCt = true
+												} else {
+													// && 特殊处理
+													if strings.Contains(r2.keyStr, "&&") {
+														flag := true
+														for _, s := range strings.Split(r2.keyStr, "&&") {
+															if !strings.Contains(v1, s) {
+																flag = false
+																break
+															}
+														}
+														if flag {
+															isCt = true
+														}
+													}
+												}
+											}
+										}
+									}
+									if isCt {
+										tag[v.tagName] = v.tagCode
+									}
+								} else {
+									tag[v.tagName] = v.tagCode
+								}
+							}
+						}
+					}
+				}
+			}
+		L:
+		}
+	}
+	return tag
+}
+
+func taskFuc1(tmp map[string]interface{}) map[string]string {
+	tag := make(map[string]string) // 打上的标签
+
+	for _, v := range StageCode {
+		// 同个类型的标签如果存在，就不需要再打
+		if tag[v.tagName] != "" {
+			continue
+		}
+		// 排除词
+		if len(v.excludeField) > 0 && len(v.excludeKeyReg) > 0 {
+			for _, f := range v.excludeField {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					for _, e1 := range v.excludeKeyReg {
+						flag := false
+						if e1.regs != nil && e1.regs.MatchString(val) {
+							flag = true
+						} else {
+							// && 特殊处理
+							if strings.Contains(e1.keyStr, "&&") {
+								for _, s := range strings.Split(e1.keyStr, "&&") {
+									if strings.Contains(val, s) {
+										flag = true
+										break
+									}
+								}
+							}
+						}
+						if flag {
+							goto L
+						}
+					}
+				}
+			}
+		}
+		// 清理词
+		if len(v.clearKey) > 0 && len(v.matchField) > 0 {
+			for _, s := range v.clearKey {
+				for _, f := range v.matchField {
+					if val := util.ObjToString(tmp[f]); val != "" {
+						tmp[f] = strings.ReplaceAll(val, s, "")
+					}
+				}
+			}
+		}
+		// 关键词
+		if len(v.matchField) > 0 && len(v.matchKeyReg) > 0 {
+			for _, f := range v.matchField {
+				if val := util.ObjToString(tmp[f]); val != "" {
+					for _, r1 := range v.matchKeyReg {
+						if r1.regs.MatchString(val) {
+							if len(v.addField) > 0 && len(v.addKeyReg) > 0 {
+								// 匹配附加词
+								isCt := false
+								for _, f1 := range v.addField {
+									if v1 := util.ObjToString(tmp[f1]); v1 != "" {
+										for _, r2 := range v.addKeyReg {
+											if r2.regs != nil && r2.regs.MatchString(v1) {
+												isCt = true
+											} else {
+												// && 特殊处理
+												if strings.Contains(r2.keyStr, "&&") {
+													flag := true
+													for _, s := range strings.Split(r2.keyStr, "&&") {
+														if !strings.Contains(v1, s) {
+															flag = false
+															break
+														}
+													}
+													if flag {
+														isCt = true
+													}
+												}
+											}
+										}
+									}
+								}
+								if isCt {
+									tag[v.tagName] = v.tagCode
+								}
+							} else {
+								tag[v.tagName] = v.tagCode
+							}
+						}
+					}
+				}
+			}
+		}
+	L:
+	}
+	return tag
+}
+
+func UpdateMethod() {
+	arru := make([][]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MgoPro.UpdateBulk(config.Conf.DB.MongoP.ProposedColl, arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MgoPro.UpdateBulk(config.Conf.DB.MongoP.ProposedColl, arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

proposed_project/tidbTask.go

@@ -0,0 +1,642 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"app.yhyue.com/data_processing/common_utils/log"
+	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"app.yhyue.com/data_processing/common_utils/redis"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.uber.org/zap"
+	"math/rand"
+	"proposed_project/config"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+	"unicode/utf8"
+)
+
+var (
+	saveBasePool = make(chan map[string]interface{}, 5000)
+	saveBaseSp   = make(chan bool, 1)
+	saveRcPool   = make(chan map[string]interface{}, 5000)
+	saveRcSp     = make(chan bool, 1)
+	saveCtPool   = make(chan map[string]interface{}, 5000)
+	saveCtSp     = make(chan bool, 1)
+	saveCyPool   = make(chan map[string]interface{}, 5000)
+	saveCySp     = make(chan bool, 1)
+	saveEntPool  = make(chan map[string]interface{}, 5000)
+	saveEntSp    = make(chan bool, 1)
+
+	BaseField = []string{"lasttime", "firsttime", "proposed_number", "proposed_id", "follow_num", "title", "projectname", "approvecode", "approvedept",
+		"approvenumber", "project_stage_code", "total_investment", "funds", "owner", "name_id", "ownerclass_code", "projecttype_code", "projectaddr",
+		"projectperiod", "project_startdate", "project_completedate", "industry_code", "approvestatus", "project_scale",
+		"category_code", "nature_code", "construction_area", "floor_area", "area_code", "city_code", "createtime"}
+	RecordField   = []string{"proposed_id", "infoid", "follow_num", "project_stage_code", "title", "project_scale", "publishtime", "jybxhref", "createtime"}
+	ContactField  = []string{"proposed_id", "infoid", "follow_num", "name_id", "name", "contact_name", "contact_tel", "contact_addr", "createtime"}
+	CategoryField = []string{"proposed_id", "labelcode", "labelvalues", "labelweight", "createtime"}
+	EntField      = []string{"proposed_id", "name_id", "name", "area_code", "city_code", "address", "createtime", "identity_type"}
+
+	AreaCode = make(map[string]string, 5000)
+	TagCode  = make(map[string]interface{}, 100)
+)
+
+func InitArea() {
+	info := MysqlTool.Find("d_area_code_back", nil, "", "", -1, -1)
+	for _, m := range *info {
+		var key string
+		for i, v := range []string{"area", "city", "district"} {
+			if i == 0 && util.ObjToString(m[v]) != "" {
+				key = util.ObjToString(m[v])
+			} else if util.ObjToString(m[v]) != "" {
+				key += "," + util.ObjToString(m[v])
+			}
+		}
+		AreaCode[key] = util.ObjToString(m["code"])
+	}
+	log.Info("InitField", zap.Int("AreaCode", len(AreaCode)))
+}
+
+func InitTagCode() {
+	info, _ := MgoBid.Find("nzj_rule", nil, nil, bson.M{"label_name": 1, "label": 1, "code": 1}, false, -1, -1)
+	for _, m := range *info {
+		lname := util.ObjToString(m["label_name"])
+		lb := util.ObjToString(m["label"])
+		code := util.ObjToString(m["code"])
+		if lname == "sub_category" || lname == "top_category" {
+			lname = "category"
+		}
+		if TagCode[lname] != nil {
+			m1 := TagCode[lname].(map[string]interface{})
+			m1[code] = lb
+			TagCode[lname] = m1
+		} else {
+			m1 := make(map[string]interface{})
+			m1[code] = lb
+			TagCode[lname] = m1
+		}
+	}
+	TagCode["nature"].(map[string]interface{})["00"] = "其它"
+	TagCode["project_stage"].(map[string]interface{})["00"] = "其它"
+	TagCode["category"].(map[string]interface{})["04"] = "其它工程"
+	log.Info("InitTagCode", zap.Any("TagCode", TagCode))
+}
+
+func taskTidb(q map[string]interface{}) {
+	sess := MgoPro.GetMgoConn()
+	defer MgoPro.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+	var query *mongodb.MgoIter
+	if q != nil && len(q) > 0 {
+		query = sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(q).Select(SelectF).Iter()
+	} else {
+		query = sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(nil).Select(SelectF).Iter()
+	}
+
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%20000 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		if t := util.Int64All(tmp["pici"]); t > pici {
+			pici = t
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			saveM := make(map[string]interface{})
+			for _, f := range BaseField {
+				if f == "lasttime" || f == "firsttime" {
+					if t := util.Int64All(tmp[f]); t > 0 {
+						saveM[f] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+					}
+				} else if f == "proposed_id" {
+					saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
+				} else if f == "area_code" {
+					if tmp["area"] != nil {
+						saveM[f] = AreaCode[util.ObjToString(tmp["area"])]
+					}
+				} else if f == "city_code" {
+					if tmp["area"] != nil && tmp["city"] != nil {
+						c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"])
+						saveM[f] = AreaCode[c]
+					}
+				} else if f == "owner" {
+					if v := util.ObjToString(tmp[f]); v != "" {
+						if utf8.RuneCountInString(v) < 100 {
+							saveM[f] = v
+						}
+					}
+				} else if f == "name_id" {
+					if b := util.ObjToString(tmp["owner"]); b != "" {
+						if eid := redis.GetStr("ent_id", b); eid != "" {
+							saveM["name_id"] = strings.Split(eid, "_")[0]
+						}
+					}
+				} else if f == "lasttime" || f == "firsttime" || f == "project_startdate" || f == "project_completedate" {
+					if tmp[f] != nil && util.IntAll(tmp[f]) > 0 {
+						t := util.Int64All(tmp[f])
+						saveM[f] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+					}
+				} else if f == "createtime" {
+					saveM[f] = time.Now().Format(util.Date_Full_Layout)
+				} else if f == "total_investment" {
+					text := util.ObjToString(tmp[f])
+					capital := ObjToMoney(text)
+					capital = capital / 10000
+					if capital != 0 {
+						capital, _ = util.FormatFloat(capital, 6)
+						saveM[f] = capital
+					}
+				} else if f == "approvestatus" {
+					if util.ObjToString(tmp[f]) != "" && utf8.RuneCountInString(util.ObjToString(tmp[f])) < 8 {
+						saveM[f] = tmp[f]
+					}
+				} else if f == "proposed_number" {
+					if tmp[f] == nil {
+						now := time.Now().Unix()
+						st := util.FormatDateByInt64(&now, util.Date_yyyyMMdd)
+						parseSt := strconv.FormatInt(util.Int64All(st), 8)                                       // 转8进制
+						rd := fmt.Sprintf("%04v", rand.New(rand.NewSource(time.Now().UnixNano())).Int63n(10000)) // 4位随机数
+						saveM[f] = fmt.Sprintf("NZJ%s%s", parseSt, rd)
+					} else {
+						saveM[f] = tmp[f]
+					}
+				} else if f == "approvecode" {
+					if util.ObjToString(tmp[f]) != "" && utf8.RuneCountInString(util.ObjToString(tmp[f])) < 200 {
+						saveM[f] = tmp[f]
+					}
+				} else if f == "floor_area" {
+					if util.ObjToString(tmp[f]) != "" && utf8.RuneCountInString(util.ObjToString(tmp[f])) < 255 {
+						saveM[f] = tmp[f]
+					}
+				} else {
+					if tmp[f] != nil {
+						saveM[f] = tmp[f]
+					}
+				}
+			}
+			saveBasePool <- saveM
+			saveCy := make(map[string]interface{})
+			saveCy["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+			saveCy["labelcode"] = "category_code"
+			saveCy["labelvalues"] = util.ObjToString(tmp["category_code"])
+			saveCy["createtime"] = time.Now().Format(util.Date_Full_Layout)
+			saveCyPool <- saveCy
+			if ow := util.ObjToString(tmp["owner"]); ow != "" {
+				saveEnt := make(map[string]interface{})
+				saveEnt["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+				saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+				saveEnt["name"] = ow
+				if eid := redis.GetStr("ent_id", ow); eid != "" {
+					arr := strings.Split(eid, "_")
+					saveEnt["name_id"] = arr[0]
+					if len(arr) == 2 {
+						saveEnt["area_code"] = arr[1]
+					} else if len(arr) == 3 {
+						saveEnt["city_code"] = arr[2]
+					}
+					info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
+					if info != nil && len(*info) > 0 {
+						saveEnt["address"] = (*info)[0]["address"]
+					}
+				}
+				saveEnt["identity_type"] = 1
+				saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+				saveEntPool <- saveEnt
+			}
+
+			for _, v := range tmp["list"].([]interface{}) {
+				saveRc := make(map[string]interface{})
+				v1 := v.(map[string]interface{})
+				saveRc["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+				infoid := util.ObjToString(v1["infoid"])
+				saveRc["infoid"] = infoid
+				saveRc["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", infoid))
+				saveRc["follow_num"] = v1["follow_num"]
+				saveRc["project_scale"] = util.ObjToString(v1["project_scale"])
+				saveRc["project_stage_code"] = util.ObjToString(v1["project_stage_code"])
+				saveRc["title"] = util.ObjToString(v1["title"])
+				if t := util.Int64All(v1["publishtime"]); t > 0 {
+					saveRc["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+				}
+				saveRc["createtime"] = time.Now().Format(util.Date_Full_Layout)
+				saveRcPool <- saveRc
+
+				if util.ObjToString(v1["project_person"]) != "" || util.ObjToString(v1["project_phone"]) != "" {
+					saveCt := make(map[string]interface{})
+					saveCt["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+					saveCt["infoid"] = infoid
+					saveCt["follow_num"] = tmp["follow_num"]
+					if b := util.ObjToString(tmp["owner"]); b != "" {
+						saveCt["name"] = util.ObjToString(tmp["owner"])
+						if eid := redis.GetStr("ent_id", b); eid != "" {
+							saveCt["name_id"] = strings.Split(eid, "_")[0]
+						}
+					}
+					if p := util.ObjToString(v1["project_person"]); p != "" {
+						saveCt["contact_name"] = p
+					}
+					if p := util.ObjToString(v1["project_phone"]); p != "" {
+						saveCt["contact_tel"] = p
+					}
+					if p := util.ObjToString(v1["projectaddr"]); p != "" {
+						saveCt["contact_addr"] = p
+					}
+					saveCt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+					saveCtPool <- saveCt
+				}
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Info(fmt.Sprintf("over --- %d", count), zap.Int64("pici", pici))
+}
+
+func taskTidb_add(q map[string]interface{}) {
+	sess := MgoPro.GetMgoConn()
+	defer MgoPro.DestoryMongoConn(sess)
+
+	ch := make(chan bool, config.Conf.Serve.Thread)
+	wg := &sync.WaitGroup{}
+
+	log.Info("taskTidb_add", zap.Any("q: ", q))
+	query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(q).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%200 == 0 {
+			log.Info(fmt.Sprintf("current --- %d", count))
+		}
+		if t := util.Int64All(tmp["pici"]); t > pici {
+			pici = t
+		}
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			go func() {
+				<-ch
+				wg.Done()
+			}()
+			taskB(tmp)
+			taskE(tmp)
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	TaskSingle = true
+	log.Info(fmt.Sprintf("over --- %d", count), zap.Int64("pici", pici))
+}
+
+func taskB(tmp map[string]interface{}) {
+	saveM := make(map[string]interface{})
+	for _, f := range BaseField {
+		if f == "lasttime" || f == "firsttime" {
+			if t := util.Int64All(tmp[f]); t > 0 {
+				saveM[f] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+			}
+		} else if f == "proposed_id" {
+			saveM[f] = mongodb.BsonIdToSId(tmp["_id"])
+		} else if f == "area_code" {
+			if tmp["area"] != nil {
+				saveM[f] = AreaCode[util.ObjToString(tmp["area"])]
+			}
+		} else if f == "city_code" {
+			if tmp["area"] != nil && tmp["city"] != nil {
+				c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"])
+				saveM[f] = AreaCode[c]
+			}
+		} else if f == "owner" {
+			if v := util.ObjToString(tmp[f]); v != "" {
+				if utf8.RuneCountInString(v) < 100 {
+					saveM[f] = v
+				}
+			}
+		} else if f == "name_id" {
+			if b := util.ObjToString(tmp["owner"]); b != "" {
+				if eid := redis.GetStr("ent_id", b); eid != "" {
+					saveM["name_id"] = strings.Split(eid, "_")[0]
+				}
+			}
+		} else if f == "lasttime" || f == "firsttime" || f == "project_startdate" || f == "project_completedate" {
+			if tmp[f] != nil && util.IntAll(tmp[f]) > 0 {
+				t := util.Int64All(tmp[f])
+				saveM[f] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+			}
+		} else if f == "createtime" {
+			saveM[f] = time.Now().Format(util.Date_Full_Layout)
+		} else if f == "total_investment" {
+			text := util.ObjToString(tmp[f])
+			capital := ObjToMoney(text)
+			capital = capital / 10000
+			if capital != 0 {
+				capital, _ = util.FormatFloat(capital, 6)
+				saveM[f] = capital
+			}
+		} else if f == "approvestatus" {
+			if util.ObjToString(tmp[f]) != "" && utf8.RuneCountInString(util.ObjToString(tmp[f])) < 8 {
+				saveM[f] = tmp[f]
+			}
+		} else if f == "proposed_number" {
+			if tmp[f] == nil {
+				now := time.Now().Unix()
+				st := util.FormatDateByInt64(&now, util.Date_yyyyMMdd)
+				parseSt := strconv.FormatInt(util.Int64All(st), 8)                                       // 转8进制
+				rd := fmt.Sprintf("%04v", rand.New(rand.NewSource(time.Now().UnixNano())).Int63n(10000)) // 4位随机数
+				saveM[f] = fmt.Sprintf("NZJ%s%s", parseSt, rd)
+			} else {
+				saveM[f] = tmp[f]
+			}
+		} else if f == "approvecode" {
+			if util.ObjToString(tmp[f]) != "" && utf8.RuneCountInString(util.ObjToString(tmp[f])) < 200 {
+				saveM[f] = tmp[f]
+			}
+		} else if f == "floor_area" {
+			if util.ObjToString(tmp[f]) != "" && utf8.RuneCountInString(util.ObjToString(tmp[f])) < 255 {
+				saveM[f] = tmp[f]
+			}
+		} else {
+			if tmp[f] != nil {
+				saveM[f] = tmp[f]
+			}
+		}
+	}
+	info := MysqlTool.FindOne("dwd_f_nzj_baseinfo", map[string]interface{}{"proposed_id": mongodb.BsonIdToSId(tmp["_id"])}, "", "")
+	if info != nil && len(*info) > 0 {
+		saveM["updatetime"] = time.Now().Format(util.Date_Full_Layout)
+		MysqlTool.Update("dwd_f_nzj_baseinfo", map[string]interface{}{"proposed_id": mongodb.BsonIdToSId(tmp["_id"])}, saveM)
+	} else {
+		MysqlTool.Insert("dwd_f_nzj_baseinfo", saveM)
+	}
+	info1 := MysqlTool.FindOne("dwd_f_nzj_category_tags", map[string]interface{}{"proposed_id": mongodb.BsonIdToSId(tmp["_id"])}, "", "")
+	if info1 != nil && len(*info1) > 0 {
+
+	} else {
+		saveCy := make(map[string]interface{})
+		saveCy["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+		saveCy["labelcode"] = "category_code"
+		saveCy["labelvalues"] = util.ObjToString(tmp["category_code"])
+		saveCy["createtime"] = time.Now().Format(util.Date_Full_Layout)
+		MysqlTool.Insert("dwd_f_nzj_category_tags", saveCy)
+	}
+	info2 := MysqlTool.FindOne("dwd_f_nzj_ent", map[string]interface{}{"proposed_id": mongodb.BsonIdToSId(tmp["_id"])}, "", "")
+	if info2 != nil && len(*info2) > 0 {
+
+	} else {
+		if ow := util.ObjToString(tmp["owner"]); ow != "" {
+			saveEnt := make(map[string]interface{})
+			saveEnt["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+			saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+			saveEnt["name"] = ow
+			if eid := redis.GetStr("ent_id", ow); eid != "" {
+				arr := strings.Split(eid, "_")
+				saveEnt["name_id"] = arr[0]
+				if len(arr) == 2 {
+					saveEnt["area_code"] = arr[1]
+				} else if len(arr) == 3 {
+					saveEnt["city_code"] = arr[2]
+				}
+				info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
+				if info != nil && len(*info) > 0 {
+					saveEnt["address"] = (*info)[0]["address"]
+				}
+			}
+			saveEnt["identity_type"] = 1
+			saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
+			MysqlTool.Insert("dwd_f_nzj_ent", saveEnt)
+		}
+	}
+}
+
+func taskE(tmp map[string]interface{}) {
+	for _, v := range tmp["list"].([]interface{}) {
+		v1 := v.(map[string]interface{})
+		infoid := util.ObjToString(v1["infoid"])
+		info := MysqlTool.FindOne("dwd_f_nzj_follw_record", map[string]interface{}{"proposed_id": mongodb.BsonIdToSId(tmp["_id"]), "infoid": infoid}, "", "")
+		if info == nil || len(*info) == 0 {
+			saveRc := make(map[string]interface{})
+			saveRc["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+			saveRc["infoid"] = infoid
+			saveRc["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", infoid))
+			saveRc["follow_num"] = v1["follow_num"]
+			saveRc["project_scale"] = util.ObjToString(v1["project_scale"])
+			saveRc["project_stage_code"] = util.ObjToString(v1["project_stage_code"])
+			saveRc["title"] = util.ObjToString(v1["title"])
+			if t := util.Int64All(v1["publishtime"]); t > 0 {
+				saveRc["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+			}
+			saveRc["createtime"] = time.Now().Format(util.Date_Full_Layout)
+			MysqlTool.Insert("dwd_f_nzj_follw_record", saveRc)
+		}
+
+		info1 := MysqlTool.FindOne("dwd_f_nzj_contact", map[string]interface{}{"proposed_id": mongodb.BsonIdToSId(tmp["_id"]), "infoid": infoid}, "", "")
+		if info1 == nil || len(*info1) == 0 {
+			if util.ObjToString(v1["project_person"]) != "" || util.ObjToString(v1["project_phone"]) != "" {
+				saveCt := make(map[string]interface{})
+				saveCt["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
+				saveCt["infoid"] = infoid
+				saveCt["follow_num"] = tmp["follow_num"]
+				if b := util.ObjToString(tmp["owner"]); b != "" {
+					saveCt["name"] = util.ObjToString(tmp["owner"])
+					if eid := redis.GetStr("ent_id", b); eid != "" {
+						saveCt["name_id"] = strings.Split(eid, "_")[0]
+					}
+				}
+				if p := util.ObjToString(v1["project_person"]); p != "" {
+					saveCt["contact_name"] = p
+				}
+				if p := util.ObjToString(v1["project_phone"]); p != "" {
+					saveCt["contact_tel"] = p
+				}
+				if p := util.ObjToString(v1["projectaddr"]); p != "" {
+					saveCt["contact_addr"] = p
+				}
+				saveCt["createtime"] = time.Now().Format(util.Date_Full_Layout)
+				MysqlTool.Insert("dwd_f_nzj_contact", saveCt)
+			}
+		}
+
+	}
+}
+
+func SaveFunc(table string, arr []string) {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveBasePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveBaseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveBaseSp
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveBaseSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveBaseSp
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveRFunc(table string, arr []string) {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveRcPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveRcSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveRcSp
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveRcSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveRcSp
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveCFunc(table string, arr []string) {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveCtPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveCtSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveCtSp
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveCtSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveCtSp
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveCyFunc(table string, arr []string) {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveCyPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveCySp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveCySp
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveCySp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveCySp
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}
+
+func SaveEntFunc(table string, arr []string) {
+	arru := make([]map[string]interface{}, saveSize)
+	indexu := 0
+	for {
+		select {
+		case v := <-saveEntPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == saveSize {
+				saveEntSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEntSp
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru)
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		case <-time.After(1000 * time.Millisecond):
+			if indexu > 0 {
+				saveEntSp <- true
+				go func(arru []map[string]interface{}) {
+					defer func() {
+						<-saveEntSp
+					}()
+					MysqlTool.InsertBulk(table, arr, arru...)
+				}(arru[:indexu])
+				arru = make([]map[string]interface{}, saveSize)
+				indexu = 0
+			}
+		}
+	}
+}

proposed_project/util.go

@@ -0,0 +1,372 @@
+package main
+
+import (
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type TagMatching struct {
+	tagName       string   // 标签名称
+	tagCode       string   // 标签值(保存)
+	matchField    []string //关键词匹配字段
+	matchKey      string   //匹配词
+	matchKeyReg   []*RegexpInfo
+	addField      []string //附加词匹配字段
+	addKey        string   // 附件词匹配词
+	addKeyReg     []*RegexpInfo
+	excludeField  []string //排除词
+	excludeKey    string   //排除词匹配词
+	excludeKeyReg []*RegexpInfo
+	//clearField    []string // 清理词匹配字段
+	clearKey []string //清理词匹配字段跟关键词一样
+}
+
+type RegexpInfo struct {
+	keyStr string
+	regs   *regexp.Regexp
+}
+
+func GetRegex(key string) []*RegexpInfo {
+	var infos []*RegexpInfo
+	for _, s := range strings.Split(key, ",") {
+		if strings.Contains(s, "&&") {
+			info := &RegexpInfo{
+				keyStr: s,
+				regs:   nil,
+			}
+			infos = append(infos, info)
+		} else {
+			if s == "" {
+				continue
+			}
+			info := &RegexpInfo{
+				keyStr: s,
+				regs:   regexp.MustCompile(".*(?i)" + s + ".*"),
+			}
+			infos = append(infos, info)
+		}
+
+	}
+	return infos
+}
+
+var (
+	regNumFloat, _  = regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
+	regStrUnit, _   = regexp.Compile(`[元|万|亿]`)
+	contentUnit, _  = regexp.Compile(`(万元|单位/万)`)
+	numCapitals, _  = regexp.Compile(`([〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]{4,40})`)
+	regStrChar      = `[〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
+	moneyRegChar, _ = regexp.Compile(regStrChar)
+	regQianw, _     = regexp.Compile(`\d{1,2}千万`)
+
+	cutAllSpace, _ = regexp.Compile(`\s*`)
+	spaces         = []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n"}
+
+	moneyChar = map[string]interface{}{ //"〇": "0", "零": "0",
+		"一": float64(1), "壹": float64(1), "二": float64(2), "贰": float64(2), "三": float64(3), "叁": float64(3), "四": float64(4), "肆": float64(4), "五": float64(5), "伍": float64(5),
+		"六": float64(6), "陆": float64(6), "七": float64(7), "柒": float64(7), "八": float64(8), "捌": float64(8), "九": float64(9), "玖": float64(9), "十": float64(10), "拾": float64(10),
+		"百": float64(100), "佰": float64(100), "千": float64(1000), "仟": float64(1000), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000),
+		"零": float64(0), "点": ".", "角": float64(0.1), "分": float64(0.01),
+	}
+	moneyUnit = map[string]float64{
+		"元": float64(1), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000), //单位
+	}
+)
+
+var currencyItem = map[string]string{
+	"人民币": "人民币",
+	"rmb": "人民币",
+	"RMB": "人民币",
+	"$":   "美元",
+	"＄":   "美元",
+	"美元":  "美元",
+	"港元":  "港币",
+	"港币":  "港币",
+	"澳币":  "澳币",
+	"澳元":  "澳币",
+}
+
+//获取币种
+func GetCurrency(text string) (currency string) {
+	if text == "" {
+		return
+	}
+	currency = "人民币"
+	for k, v := range currencyItem {
+		if strings.Contains(text, k) {
+			currency = v
+			return
+		}
+	}
+	return
+}
+
+//金额转换
+func ObjToMoney(text string) float64 {
+	isfindUnit := true
+	ret := capitalMoney(text)
+	if ret < float64(10000) || ret > float64(50000000000) {
+		ret2, b := numMoney(text)
+		isfindUnit = b
+		if ret2 > ret {
+			ret = ret2
+		}
+	}
+	f, _ := strconv.ParseFloat(strconv.FormatFloat(ret, 'f', 4, 64), 64)
+	// if f < 1 {
+	// 	f = 0
+	// }
+	//如果金额小于50，全文检索单位：万
+	if f < 50 && f > 0 && isfindUnit {
+		rep := contentUnit.FindAllStringIndex(text, -1)
+		if len(rep) > 0 {
+			f = f * 10000
+		}
+	}
+	return f
+}
+
+func capitalMoney(text string) float64 {
+	nodes := []float64{}
+	node := float64(0)
+	tmp := float64(0)
+	decimals := 0.0
+	ishaspoint := false //是否含小数点
+	fnum := float64(0)
+	end := false
+	//str := fmt.Sprint(data[0])
+	//提取第一个大写信息
+	strmatch := numCapitals.FindAllStringSubmatch(text, -1)
+	if len(strmatch) > 0 {
+		text = strmatch[0][0]
+	}
+	suffixUnit := float64(1)
+	if strings.HasSuffix(text, "万") || strings.HasSuffix(text, "万元") || strings.HasSuffix(text, "万元整") {
+		index := strings.LastIndex(text, "万")
+		text = text[0:index]
+		suffixUnit = float64(10000)
+	}
+	moneyRegChar.ReplaceAllStringFunc(text, func(key string) string {
+		if key == "元" || key == "圆" || key == "点" {
+			ishaspoint = true
+		}
+		if v, ok := moneyChar[key].(float64); ok && !end {
+			if ishaspoint && v > 10 { //排除后面有其他的单位
+				return ""
+			}
+			//fmt.Println(key, v, fnum)
+			if v < 10 && v >= 0 {
+				if ishaspoint { //小数部分
+					if v >= 1 {
+						fnum = v
+					} else if v < 1 && v > 0 {
+						decimals += fnum * v
+					}
+				} else {
+					if tmp != float64(0) {
+						node += tmp
+					}
+					tmp = float64(v)
+				}
+			} else if v == 10000 || v == 100000000 { //单位万、亿
+				if tmp != float64(0) {
+					node += tmp
+					tmp = float64(0)
+				}
+				nodes = append(nodes, node*float64(v))
+				node = float64(0)
+			} else {
+				if v == 10 && tmp == 0 {
+					tmp = 1
+				}
+				tmp = tmp * float64(v)
+				node += tmp
+				tmp = float64(0)
+			}
+		}
+		if key == "整" || key == "正" || key == "分" {
+			end = true
+		}
+		return ""
+	})
+	nodes = append(nodes, node, tmp)
+	ret := float64(0)
+	for _, v := range nodes {
+		ret += v
+	}
+	return (ret + decimals) * suffixUnit
+}
+
+//数字金额转换
+func numMoney(text string) (moneyFloat float64, flag bool) {
+	//tmp := fmt.Sprintf("%f", data[0])
+	repUnit := float64(1)
+	if regQianw.MatchString(text) {
+		text = strings.Replace(text, "千万", "万", -1)
+		repUnit = float64(1000)
+	}
+	text = replaceSymbol(text, []string{",", "，", "（", "）", "(", ")", "：", "\n"})
+	text = replaceString(text, []string{"万元", "亿元", "．"}, []string{"万", "亿", "."})
+	text = CutAllSpace(text)
+	rets := regNumFloat.FindAllString(text, -1)
+	fnums := []float64{}
+	unitstrs := []string{}
+	if len(rets) > 0 {
+		pindex := 0 //单位前置
+		for k, v := range rets {
+			f, err := strconv.ParseFloat(v, 64)
+			if err == nil {
+				fnums = append(fnums, f)
+				index := strings.Index(text, v)
+				//单位后置
+				start := index + len(v)
+				end := start + 3
+				//log.Println("vvv", tmp, v, pindex, index, start)
+				if k > 0 {
+					if start >= pindex+3 {
+						pstart := pindex + 3
+						if pstart >= index {
+							pstart = index
+						}
+						if len(text) > end {
+							unitstrs = append(unitstrs, text[pstart:index]+text[start:end])
+						} else {
+							unitstrs = append(unitstrs, text[pstart:index]+text[start:])
+						}
+					} else {
+						if len(text) > end {
+							unitstrs = append(unitstrs, text[start:end])
+						} else {
+							unitstrs = append(unitstrs, text[start:])
+						}
+					}
+				} else {
+					if len(text) > end {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, text[index-3:index]+text[start:end])
+						} else {
+							unitstrs = append(unitstrs, text[start:end])
+						}
+					} else {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, text[index-3:index]+text[start:])
+						} else {
+							unitstrs = append(unitstrs, text[start:])
+						}
+					}
+				}
+				pindex = start
+			}
+		}
+	}
+	//log.Println("unitstrs", fnums, unitstrs)
+	unit := float64(0)
+	fnum := float64(0)
+	for k, v := range fnums {
+		fnum = v
+		units := regStrUnit.FindAllString(unitstrs[k], -1)
+		for _, v := range units {
+			if moneyUnit[v] != 0 {
+				unit = moneyUnit[v]
+				break
+			}
+		}
+		if unit != float64(0) { //取第一个
+			break
+		}
+	}
+	fnum = fnum * repUnit
+	if unit == float64(0) {
+		moneyFloat = fnum
+	} else {
+		moneyFloat = fnum * unit
+	}
+	if unit == 10000 {
+		flag = false
+	} else {
+		flag = true
+	}
+	return
+}
+
+//清理所有空白符
+func CutAllSpace(text string) string {
+	tmp := cutAllSpace.ReplaceAllString(text, "")
+	tmp = replaceSymbol(tmp, spaces)
+	return tmp
+}
+
+//符号替换
+func replaceString(con string, ret, rep []string) string {
+	for k, v := range ret {
+		if len(rep) > k {
+			con = strings.Replace(con, v, rep[k], -1)
+		}
+	}
+	return con
+}
+
+//过滤符号
+func replaceSymbol(con string, rep []string) string {
+	for _, v := range rep {
+		con = strings.Replace(con, v, "", -1)
+	}
+	return con
+}
+
+// @Description python in stopWords
+// @Author J 2023/3/7 16:15
+func in(target string) bool {
+	//sort.Strings(str_array)
+	index := sort.SearchStrings(stopWords, target)
+	if index < len(stopWords) && stopWords[index] == target {
+		return true
+	}
+	return false
+}
+
+func diffNatureDays(t1, t2 int64) int {
+	if t1 == t2 {
+		return -1
+	}
+	if t1 > t2 {
+		t1, t2 = t2, t1
+	}
+
+	diffDays := 0
+	secDiff := t2 - t1
+	if secDiff > 86400 {
+		tmpDays := int(secDiff / 86400)
+		t1 += int64(tmpDays) * 86400
+		diffDays += tmpDays
+	}
+
+	st := time.Unix(t1, 0)
+	et := time.Unix(t2, 0)
+	dateFormatTpl := "20060102"
+	if st.Format(dateFormatTpl) != et.Format(dateFormatTpl) {
+		diffDays += 1
+	}
+
+	return diffDays
+}
+
+type FollwRecord struct {
+	Proposed_id        string
+	Infoid             string
+	Title              string
+	Project_stage_code string
+	Jybxhref           string
+	Project_scale      string
+	Publishtime        int64
+	Createtime         int64
+}
+
+type ByPtime []FollwRecord
+
+func (a ByPtime) Len() int           { return len(a) }
+func (a ByPtime) Less(i, j int) bool { return a[i].Publishtime > a[j].Publishtime }
+func (a ByPtime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }

proposed_project/xx_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	util "app.yhyue.com/data_processing/common_utils"
+	"testing"
+)
+
+func TestMethod(t *testing.T) {
+	initSeg()
+	InitEs()
+	//util.Debug(Method("万宁市城南路市政道路高低压电力线路改造工程"))
+
+	wds, query := Method1("1#、2#厂房扩建项目")
+	util.Debug(wds)
+	util.Debug(query)
+	binfo := Es.GetNoLimit("projectset", query)
+	if binfo != nil && len(*binfo) > 0 {
+		util.Debug(len(*binfo))
+	}
+}
+
+func TestMethod2(t *testing.T) {
+
+	util.Debug(regDel.ReplaceAllString("12厂房扩建项目", ""))
+
+	util.Debug(filterMethod(map[string]interface{}{"projectname": "1#、2#厂房扩建项目"}))
+
+	util.Debug(combArr([]string{
+		"崇仁县",
+		"石庄",
+		"乡",
+		"2022",
+		"年",
+		"新",
+		"农村",
+		"建设工程",
+		"石庄村",
+		"程坊",
+		"组"}))
+}