Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

src/jy/extract/extpackage.go

@@ -148,8 +148,10 @@ func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 					sonJobResult["origin"] = pkg.Origin
 					sonJobResult["text"] = pkg.Text
 					sonJobResult["name"] = pkg.Name
-					sonJobResult["winnertel"] = pkg.WinnerTel
-					sonJobResult["winnerperson"] = pkg.WinnerPerson
+					if pkg.WinnerPerson != "" {
+						sonJobResult["winnertel"] = pkg.WinnerTel
+						sonJobResult["winnerperson"] = pkg.WinnerPerson
+					}
 					if pkg.IsTrueBudget{
 						sonJobResult["budget"] = pkg.Budget
 					}

src/jy/extract/extract.go

@@ -934,6 +934,19 @@ func ExtRuleCoreByPkgReg(j *ju.Job, in *RegLuaInfo, e *ExtractTask) {
 				if in.Field == "projectname" && vbpkg.Name != "" {
 					continue
 				}
+				if in.Field == "winnerperson" {
+					if vbpkg.Winner == "" || len(vbpkg.Winner) < 4 {
+						continue
+					}
+					if !strings.Contains(vbpkg.Text, vbpkg.Winner) {
+						continue
+					}
+				}
+				if in.Field == "winnertel" {
+					if vbpkg.WinnerPerson == "" {
+						continue
+					}
+				}
 				//处理正负数修正
 				ptmp := strings.Split(in.RuleText, "#")
 				sign := 0
@@ -1035,6 +1048,16 @@ func ExtRuleCoreByPkgReg(j *ju.Job, in *RegLuaInfo, e *ExtractTask) {
 									j.BlockPackage[k].Name = rep[in.Field+"_"+fmt.Sprint(i)]
 									break
 								}
+							} else if in.Field == "winnerperson" {
+								if j.BlockPackage[k].WinnerPerson == "" {
+									j.BlockPackage[k].WinnerPerson = rep[in.Field+"_"+fmt.Sprint(i)]
+									break
+								}
+							} else if in.Field == "winnertel" {
+								if j.BlockPackage[k].WinnerTel == "" && j.BlockPackage[k].Winner != "" && j.BlockPackage[k].WinnerPerson != "" {
+									j.BlockPackage[k].WinnerTel = rep[in.Field+"_"+fmt.Sprint(i)]
+									break
+								}
 							}
 						}
 					}
@@ -1484,6 +1507,10 @@ func ExtRegBackPkg(j *ju.Job, in *RegLuaInfo) {
 			j.BlockPackage[k].Text = in.RegPreBac.Reg.ReplaceAllString(v.Text, in.RegPreBac.Replace)
 		} else if in.Field == "projectname" {
 			j.BlockPackage[k].Name = in.RegPreBac.Reg.ReplaceAllString(v.Name, in.RegPreBac.Replace)
+		} else if in.Field == "winnerperson" {
+			j.BlockPackage[k].WinnerPerson = in.RegPreBac.Reg.ReplaceAllString(v.WinnerPerson, in.RegPreBac.Replace)
+		} else if in.Field == "winnertel" {
+			j.BlockPackage[k].WinnerTel = in.RegPreBac.Reg.ReplaceAllString(v.WinnerTel, in.RegPreBac.Replace)
 		}
 	}
 }

standardata/src/config.json

@@ -0,0 +1,31 @@
+{
+  "port": "1235",
+  "mgofrom": "172.17.4.187:27083",
+  "mgofromsize":5,
+  "mgofromdb":"qfw",
+  "mgoto": "172.17.145.163:27082",
+  "mgotosize":5,
+  "mgotodb":"extract_v3",
+  "mgoent": "172.17.145.163:27082",
+  "mgoentsize":5,
+  "mgoentdb":"enterprise",
+  "extractcoll":"result_20200116",
+  "standardata":{
+	"winner":{
+		"standarent":"winner_ent",
+		"standarerr":"winner_err",
+		"redisdb":1
+	},
+    "buyer":{
+      "standarent":"buyer_ent",
+      "standarerr":"buyer_err",
+      "redisdb":2
+    },
+    "agency":{
+      "standarent":"agency_data_ent",
+      "standarerr":"agency_data_err",
+      "redisdb":3
+    }
+  },
+  "redis": "winner=172.17.148.44:2679,buyer=172.17.148.44:2679,agency=172.17.148.44:2679"
+}

standardata/src/main.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"dbutil/mongo"
+	"dbutil/redis"
+	"fmt"
+	"log"
+	mu "mfw/util"
+	"os"
+	qu "qfw/util"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	MongoFrom /*抽取原*/, MongoTo /*保存库*/, MongoEnt/*企业库*/ *mongo.MongodbSim
+	sysconfig            map[string]interface{}
+	extractcoll          string
+	winnerent, winnererr string
+	buyerent, buyererr   string
+	agencyent, agencyerr   string
+	winnerbd, buyerbd, agencybd    int
+	Addrs                = make(map[string]interface{}, 0) //省市县
+	winchanbool          = make(chan bool, 3)
+	buyerchanbool        = make(chan bool, 3)
+	agencychanbool        = make(chan bool, 3)
+	gochan               = make(chan bool, 3)
+	udpclient            mu.UdpClient
+	//异常表正则匹配处理
+	WinnerRegOk, WinnerRegErr, AgencyRegOk, AgencyRegErr, BuyerRegOk, BuyerRegErr []regexp.Regexp
+)
+
+func init() {
+	qu.ReadConfig(&sysconfig)
+	extractcoll = qu.ObjToString(sysconfig["extractcoll"])
+	if standardata, ok := sysconfig["standardata"].(map[string]interface{}); ok {
+		winner, _ := standardata["winner"].(map[string]interface{})
+		winnerent = qu.ObjToString(winner["standarent"])
+		winnererr = qu.ObjToString(winner["standarerr"])
+		winnerbd = qu.IntAll(winner["redisdb"])
+		buyer, _ := standardata["buyer"].(map[string]interface{})
+		buyerent = qu.ObjToString(buyer["standarent"])
+		buyererr = qu.ObjToString(buyer["standarerr"])
+		buyerbd = qu.IntAll(buyer["redisdb"])
+		agency, _ := standardata["agency"].(map[string]interface{})
+		agencyent = qu.ObjToString(agency["standarent"])
+		agencyerr = qu.ObjToString(agency["standarerr"])
+		agencybd = qu.IntAll(agency["redisdb"])
+	} else {
+		os.Exit(0)
+	}
+
+	MongoFrom = &mongo.MongodbSim{
+		MongodbAddr: qu.ObjToString(sysconfig["mgofrom"]),
+		Size:        qu.IntAll(sysconfig["mgofromsize"]),
+		DbName:      qu.ObjToString(sysconfig["mgofromdb"]),
+	}
+	MongoFrom.InitPool()
+
+	MongoTo = &mongo.MongodbSim{
+		MongodbAddr: qu.ObjToString(sysconfig["mgoto"]),
+		Size:        qu.IntAll(sysconfig["mgotosize"]),
+		DbName:      qu.ObjToString(sysconfig["mgotodb"]),
+	}
+	MongoTo.InitPool()
+
+	MongoEnt = &mongo.MongodbSim{
+		MongodbAddr: qu.ObjToString(sysconfig["mgoent"]),
+		Size:        qu.IntAll(sysconfig["mgoentsize"]),
+		DbName:      qu.ObjToString(sysconfig["mgoentdb"]),
+	}
+	MongoEnt.InitPool()
+	redis.InitRedis(qu.ObjToString(sysconfig["redis"]))
+
+	//加载省市县代码
+	list, _ := MongoTo.Find("address", map[string]interface{}{}, nil, nil)
+	for _, v := range list {
+		code := v["code"]
+		if code != nil && strings.TrimSpace(code.(string)) != "" {
+			Addrs[fmt.Sprint(code)] = v
+		}
+	}
+	log.Println("加载省市县代码加载完成", len(Addrs))
+	initReg() //初始标记规则
+}
+
+func initReg() {
+	list, _ := MongoTo.Find("rc_rule", map[string]interface{}{"delete": false, "isuse": true}, nil, nil)
+	for _, v := range list {
+		s_field, ok := v["s_field"].(string) //字段
+		s_rule, ok2 := v["s_rule"].(string)  //正则
+		s_type, ok3 := v["s_type"].(string)  //ok or err
+		if !ok || !ok2 || !ok3 || s_field == "" || s_rule == "" || s_type == "" {
+			continue
+		}
+		var pattern string
+		if strings.Contains(s_rule, "\\u") {
+			s_rule = strings.Replace(s_rule, "\\", "\\\\", -1)
+			s_rule = strings.Replace(s_rule, "\\\\u", "\\u", -1)
+			pattern, _ = strconv.Unquote(`"` + s_rule + `"`)
+		} else {
+			pattern = s_rule
+		}
+		regtmp := regexp.MustCompile(pattern)
+		if s_field == "winner" {
+			if s_type == "ok" {
+				WinnerRegOk = append(WinnerRegOk, *regtmp)
+			} else if s_type == "err" {
+				WinnerRegErr = append(WinnerRegErr, *regtmp)
+			}
+		} else if s_field == "buyer" {
+			if s_type == "ok" {
+				BuyerRegOk = append(BuyerRegOk, *regtmp)
+			} else if s_type == "err" {
+				BuyerRegErr = append(BuyerRegErr, *regtmp)
+			}
+		} else if s_field == "agency" {
+			if s_type == "ok" {
+				AgencyRegOk = append(AgencyRegOk, *regtmp)
+			} else if s_type == "err" {
+				AgencyRegErr = append(AgencyRegErr, *regtmp)
+			}
+		}
+	}
+	log.Println(len(WinnerRegOk), len(WinnerRegErr), len(BuyerRegOk), len(BuyerRegErr), len(AgencyRegOk), len(AgencyRegErr))
+}
+
+func main() {
+	//go historywinner(qu.ObjToString(sysconfig["mgofromdb"]), extractcoll)
+	//go historybuyer(qu.ObjToString(sysconfig["mgofromdb"]), extractcoll)
+	go historyagency(qu.ObjToString(sysconfig["mgofromdb"]), extractcoll)
+	//go winStandarHistory(qu.ObjToString(sysconfig["mgotodb"]))
+	//go buyerStandarHistory(qu.ObjToString(sysconfig["mgotodb"]))
+
+
+	//go task_standarData()
+	c := make(chan int, 1)
+	<-c
+}

standardata/src/standaragency.go

@@ -0,0 +1,207 @@
+// standaragency
+package main
+
+import (
+	"dbutil/mongo"
+	"dbutil/redis"
+	"encoding/json"
+	"log"
+	qu "qfw/util"
+	"time"
+	"unicode/utf8"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/mgo.v2/bson"
+)
+
+//增量处理
+func agencyStandarData(db string, query map[string]interface{}) {
+	defer qu.Catch()
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(extractcoll).Find(query).Select(bson.M{"repeat": 1,"agency": 1, "agencytel": 1, "agencyperson": 1, "topscopeclass": 1,
+		"agencyaddr": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0  { //重复数据跳过
+			continue
+		}
+		agency := qu.ObjToString(tmp["agency"])
+		if utf8.RuneCountInString(agency) < 5 {
+			continue
+		}
+		infoid := mongo.BsonTOStringId(tmp["_id"])
+		topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+		entid, _ := redis.GetRedisStr("agency", agencybd, agency)
+		ps := []map[string]interface{}{}
+		agencyperson := qu.ObjToString(tmp["agencyperson"])
+		agencytel := qu.ObjToString(tmp["agencytel"])
+		if entid == "" {//redis 未存
+			savetoerr := true
+			if agencyperson != "" || agencytel != "" {
+				v := map[string]interface{}{
+					"contact_person": agencyperson,
+					"phone":          agencytel,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				ps = append(ps, v)
+				data := comHisMegerNewData(agency, "agency", ps)
+				if data != nil {
+					_id := MongoTo.Save(agencyent, data)
+					redis.PutRedis("agency", agencybd, agency, _id.(primitive.ObjectID).Hex(), -1)
+					savetoerr = false
+				}
+			}
+			if savetoerr {
+				t := MongoTo.FindOne(agencyerr, map[string]interface{}{"name": agency})
+				if len(t) < 1 {
+					MongoTo.Save(agencyerr, map[string]interface{}{
+						"name":       agency,
+						"check":      comMarkdata(agency, "agency"),
+						"updatetime": time.Now().Unix(),
+					})
+				}
+			}
+		} else {
+			if agencyperson != "" || agencytel != "" {
+				v := map[string]interface{}{
+					"contact_person": agencyperson,
+					"phone":          agencytel,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				MongoTo.UpdateById(agencyent, entid,
+					map[string]interface{}{
+						"$set":  v,
+						"$push": map[string]interface{}{"contact": v},
+					},
+				)
+			}
+		}
+		tmp = map[string]interface{}{}
+		if index%100 == 0 {
+			log.Println("agency index", index)
+		}
+	}
+	log.Println("agency ok index", index)
+}
+
+//历史数据处理
+func historyagency(db, fromcoll string) {
+	defer qu.Catch()
+	log.Println("history  start")
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(fromcoll).Find(map[string]interface{}{}).Select(bson.M{"repeat": 1,"agency": 1, "agencytel": 1, "agencyperson": 1, "topscopeclass": 1,
+		"agencyaddr": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
+			continue
+		}
+		_id := mongo.BsonTOStringId(tmp["_id"])
+		agencychanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-agencychanbool
+			}()
+			agency := qu.ObjToString(tmp["agency"])
+			topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+			if agency != "" && utf8.RuneCountInString(agency) > 4 {
+				agencyperson := qu.ObjToString(tmp["agencyperson"])
+				agencytel := qu.ObjToString(tmp["agencytel"])
+				b, _ := redis.ExistRedis("agency", agencybd, agency)
+				if b {//redis 存在
+					if agencyperson != "" || agencytel != "" {
+						strs, _ := redis.GetRedisStr("agency", agencybd, agency)
+						ps := []interface{}{}
+						err := json.Unmarshal([]byte(strs), &ps)
+						if err == nil {
+							v := map[string]interface{}{
+								"contact_person": agencyperson,
+								"phone":          agencytel,
+								"topscopeclass":  comRepTopscopeclass(topscopeclass),
+								"infoid":         _id,
+							}
+							ps = append(ps, v)
+							bs, _ := json.Marshal(ps)
+							redis.PutRedis("agency", agencybd, agency, bs, -1)
+						} else {
+							log.Println("jsonErr", err)
+						}
+					}
+				} else {
+					val := []map[string]interface{}{}
+					if agencyperson != "" || agencytel != "" {
+						tmp := map[string]interface{}{
+							"contact_person": agencyperson,
+							"phone":          agencytel,
+							"topscopeclass":  comRepTopscopeclass(topscopeclass),
+							"infoid":         _id,
+						}
+						val = append(val, tmp)
+					}
+					bs, _ := json.Marshal(val)
+					redis.PutRedis("agency", agencybd, agency, bs, -1)
+					MongoTo.Save(agencyerr, map[string]interface{}{
+						"name":       agency,
+						"updatetime": time.Now().Unix(),
+					})
+				}
+			}
+		}(tmp)
+		tmp = map[string]interface{}{}
+		if index%10000 == 0 {
+			log.Println("index", index, _id)
+		}
+	}
+	log.Println("history  ok  index", index)
+	agencyStandarHistory(qu.ObjToString(sysconfig["mgotodb"]))
+}
+
+//查询agencyerr标准化历史数据
+func agencyStandarHistory(db string) {
+	defer qu.Catch()
+	log.Println("开始标准化数据--agency", db)
+	sessto := MongoTo.GetMgoConn()
+	defer MongoTo.Close()
+	it := sessto.DB(db).C(agencyerr).Find(map[string]interface{}{}).Iter()
+	index := 0
+	entnum := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		err_id := mongo.BsonTOStringId(tmp["_id"])
+		name := qu.ObjToString(tmp["name"])
+		agencychanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-agencychanbool
+			}()
+			strs, err := redis.GetRedisStr("agency", agencybd, name)
+			if err != nil {
+				return
+			}
+			ps := []map[string]interface{}{}
+			err = json.Unmarshal([]byte(strs), &ps)
+			if err == nil {
+				data := comHisMegerNewData(name, "agency", ps)
+				if data != nil {
+					MongoTo.Save(agencyent, data)
+					MongoTo.DeleteById(agencyerr, err_id)
+					entnum++
+				} else { //未查询到企业，打标记并存表
+					num := comMarkdata(name, "agency")
+					tmp["check"] = num
+					MongoTo.UpdateById(agencyerr, err_id, map[string]interface{}{"$set": map[string]interface{}{"check": num}})
+				}
+			} else {
+				log.Println("jsonErr", name, err)
+			}
+		}(tmp)
+		if index%1000 == 0 {
+			log.Println("标准化历史数据--agency", index, err_id, entnum)
+		}
+		tmp = map[string]interface{}{}
+	}
+	log.Println("标准化数据完成--agency", index, entnum)
+}

standardata/src/standarbuyer.go

@@ -0,0 +1,236 @@
+// standarbuyer
+package main
+
+import (
+	"dbutil/mongo"
+	"dbutil/redis"
+	"encoding/json"
+	"log"
+	qu "qfw/util"
+	"time"
+	"unicode/utf8"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/mgo.v2/bson"
+)
+
+//增量处理
+func buyerStandarData(db string, query map[string]interface{}) {
+	defer qu.Catch()
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(extractcoll).Find(query).Select(bson.M{"repeat": 1, "buyer": 1, "buyertel": 1, "buyerperson": 1, "buyerclass": 1, "topscopeclass": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
+			continue
+		}
+		buyer := qu.ObjToString(tmp["buyer"])
+		if utf8.RuneCountInString(buyer) < 5 {
+			continue
+		}
+		infoid := mongo.BsonTOStringId(tmp["_id"])
+		buyerclass := qu.ObjToString(tmp["buyerclass"])
+		topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+		entid, _ := redis.GetRedisStr("buyer", buyerbd, buyer)
+		ps := []map[string]interface{}{}
+		buyerperson := qu.ObjToString(tmp["buyerperson"])
+		buyertel := qu.ObjToString(tmp["buyertel"])
+		if entid == "" {
+			savetoerr := true
+			if buyerperson != "" || buyertel != "" {
+				v := map[string]interface{}{
+					"contact_person": buyerperson,
+					"phone":          buyertel,
+					"buyerclass":     buyerclass,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				ps = append(ps, v)
+				data := comHisMegerNewData(buyer, "buyer", ps)
+				if data != nil {
+					_id := MongoTo.Save(buyerent, data)
+					redis.PutRedis("buyer", buyerbd, buyer, _id.(primitive.ObjectID).Hex(), -1)
+					savetoerr = false
+				}
+			}
+			if savetoerr {
+				t := MongoTo.FindOne(buyererr, map[string]interface{}{"name": buyer})
+				if len(t) < 1 {
+					MongoTo.Save(buyererr, map[string]interface{}{
+						"name":       buyer,
+						"buyerlass":  buyerclass,
+						"check":      comMarkdata(buyer, "buyer"),
+						"updatetime": time.Now().Unix(),
+					})
+				}
+			}
+		} else {
+			if buyerperson != "" || buyertel != "" {
+				v := map[string]interface{}{
+					"contact_person": buyerperson,
+					"phone":          buyertel,
+					"buyerclass":     buyerclass,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				data := buyerMegerBuyerclass(entid, v)
+				MongoTo.UpdateById(buyerent, entid,
+					map[string]interface{}{
+						"$set":  data,
+						"$push": map[string]interface{}{"contact": v},
+					},
+				)
+			}
+		}
+		tmp = map[string]interface{}{}
+		if index%100 == 0 {
+			log.Println("buyer index", index)
+		}
+	}
+	log.Println("buyer ok index", index)
+}
+
+//历史数据处理
+func historybuyer(db, fromcoll string) {
+	defer qu.Catch()
+	log.Println("history  start")
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(fromcoll).Find(map[string]interface{}{}).Select(bson.M{"repeat": 1, "buyer": 1, "buyertel": 1, "buyerperson": 1, "buyerclass": 1, "topscopeclass": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
+			continue
+		}
+		_id := mongo.BsonTOStringId(tmp["_id"])
+		buyerchanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-buyerchanbool
+			}()
+			buyer := qu.ObjToString(tmp["buyer"])
+			buyerclass := qu.ObjToString(tmp["buyerclass"])
+			topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+			if buyer != "" && utf8.RuneCountInString(buyer) > 4 {
+				buyerperson := qu.ObjToString(tmp["buyerperson"])
+				buyertel := qu.ObjToString(tmp["buyertel"])
+				b, _ := redis.ExistRedis("buyer", buyerbd, buyer)
+				if b {
+					if buyerperson != "" || buyertel != "" {
+						strs, _ := redis.GetRedisStr("buyer", buyerbd, buyer)
+						ps := []interface{}{}
+						err := json.Unmarshal([]byte(strs), &ps)
+						if err == nil {
+							v := map[string]interface{}{
+								"contact_person": buyerperson,
+								"phone":          buyertel,
+								"buyerclass":     buyerclass,
+								"topscopeclass":  comRepTopscopeclass(topscopeclass),
+								"infoid":         _id,
+							}
+							ps = append(ps, v)
+							bs, _ := json.Marshal(ps)
+							redis.PutRedis("buyer", buyerbd, buyer, bs, -1)
+						} else {
+							log.Println("jsonErr", err)
+						}
+					}
+				} else {
+					val := []map[string]interface{}{}
+					if buyerperson != "" || buyertel != "" {
+						tmp := map[string]interface{}{
+							"contact_person": buyerperson,
+							"phone":          buyertel,
+							"buyerclass":     buyerclass,
+							"topscopeclass":  comRepTopscopeclass(topscopeclass),
+							"infoid":         _id,
+						}
+						val = append(val, tmp)
+					}
+					bs, _ := json.Marshal(val)
+					redis.PutRedis("buyer", buyerbd, buyer, bs, -1)
+					MongoTo.Save(buyererr, map[string]interface{}{
+						"name":       buyer,
+						"buyerclass": buyerclass,
+						"updatetime": time.Now().Unix(),
+					})
+				}
+			}
+		}(tmp)
+		tmp = map[string]interface{}{}
+		if index%10000 == 0 {
+			log.Println("index", index, _id)
+		}
+	}
+	log.Println("history  ok  index", index)
+	buyerStandarHistory(qu.ObjToString(sysconfig["mgotodb"]))
+}
+
+//查询buyererr标准化历史数据
+func buyerStandarHistory(db string) {
+	defer qu.Catch()
+	log.Println("开始标准化数据--buyer", db)
+	sessto := MongoTo.GetMgoConn()
+	defer MongoTo.Close()
+	it := sessto.DB(db).C(buyererr).Find(map[string]interface{}{}).Iter()
+	index := 0
+	entnum := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		err_id := mongo.BsonTOStringId(tmp["_id"])
+		name := qu.ObjToString(tmp["name"])
+		buyerchanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-buyerchanbool
+			}()
+			strs, err := redis.GetRedisStr("buyer", buyerbd, name)
+			if err != nil {
+				return
+			}
+			ps := []map[string]interface{}{}
+			err = json.Unmarshal([]byte(strs), &ps)
+			if err == nil {
+				data := comHisMegerNewData(name, "buyer", ps)
+				if data != nil {
+					MongoTo.Save(buyerent, data)
+					MongoTo.DeleteById(buyererr, err_id)
+					entnum++
+				} else { //未查询到企业，打标记并存表
+					num := comMarkdata(name, "buyer")
+					tmp["check"] = num
+					MongoTo.UpdateById(buyererr, err_id, map[string]interface{}{"$set": map[string]interface{}{"check": num}})
+				}
+			} else {
+				log.Println("jsonErr", name, err)
+			}
+		}(tmp)
+		if index%1000 == 0 {
+			log.Println("标准化历史数据--buyer", index, err_id, entnum)
+		}
+		tmp = map[string]interface{}{}
+	}
+	log.Println("标准化数据完成--buyer", index, entnum)
+}
+
+//企业数据整合(已有标注信息)
+func buyerMegerBuyerclass(id string, ps map[string]interface{}) map[string]interface{} {
+	tmp := MongoEnt.FindById(buyerent, id, bson.M{"buyerclass": 1})
+	if len(tmp) < 1 {
+		return nil
+	}
+	data := map[string]interface{}{}
+	buyerclass := tmp["buyerclass"].(primitive.A)
+	tmpbuyerclass := map[string]bool{}
+	for _, v := range buyerclass {
+		tt := qu.ObjToString(v)
+		tmpbuyerclass[tt] = true
+	}
+	tmpbuyerclass[qu.ObjToString(ps["buyerclass"])] = true
+	newbuyerclass := []interface{}{}
+	for k, _ := range tmpbuyerclass {
+		newbuyerclass = append(newbuyerclass, k)
+	}
+	data["buyerclass"] = newbuyerclass
+	return data
+}

standardata/src/standarwinner.go

@@ -0,0 +1,500 @@
+// standarwinner
+package main
+
+import (
+	"dbutil/mongo"
+	"dbutil/redis"
+	"encoding/json"
+	"log"
+	qu "qfw/util"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/mgo.v2/bson"
+)
+
+//增量处理
+func winnerStandarData(db string, query map[string]interface{}) {
+	defer qu.Catch()
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(extractcoll).Find(query).Select(bson.M{"repeat": 1, "winner": 1, "winnertel": 1, "winnerperson": 1, "topscopeclass": 1, "package": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
+			continue
+		}
+		winner := qu.ObjToString(tmp["winner"])
+		if utf8.RuneCountInString(winner) < 5 {
+			continue
+		}
+		infoid := mongo.BsonTOStringId(tmp["_id"])
+		topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+		entid, _ := redis.GetRedisStr("winner", winnerbd, winner)
+		winnerperson := qu.ObjToString(tmp["winnerperson"])
+		winnertel := qu.ObjToString(tmp["winnertel"])
+		if entid == "" { //新增标准库
+			savetoerr := true
+			if winnerperson != "" || winnertel != "" {
+				v := map[string]interface{}{
+					"contact_person": winnerperson,
+					"phone":          winnertel,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				data := comHisMegerNewData(winner, "winner", []map[string]interface{}{v})
+				if data != nil {
+					_id := MongoTo.Save(winnerent, data)
+					redis.PutRedis("winner", winnerbd, winner, _id.(primitive.ObjectID).Hex(), -1)
+					savetoerr = false
+				}
+			}
+			if savetoerr {
+				t := MongoTo.FindOne(winnererr, map[string]interface{}{"name": winner})
+				if len(t) < 1 {
+					MongoTo.Save(winnererr, map[string]interface{}{
+						"name":          winner,
+						"topscopeclass": comRepTopscopeclass(topscopeclass),
+						"check":         comMarkdata(winner, "winner"),
+						"updatetime":    time.Now().Unix(),
+					})
+				}
+			}
+		} else { //更新标准库
+			if winnerperson != "" || winnertel != "" {
+				v := map[string]interface{}{
+					"contact_person": winnerperson,
+					"phone":          winnertel,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				data := winMegerIndustry(entid, v)
+				MongoTo.UpdateById(winnerent, entid,
+					map[string]interface{}{
+						"$set":  data,
+						"$push": map[string]interface{}{"contact": v},
+					},
+				)
+			}
+		}
+		//分包处理
+		if packages, ok := tmp["package"].(map[string]interface{}); ok {
+			entpacks := getWinnerPacks(infoid, packages, comRepTopscopeclass(topscopeclass))
+			for name, contact := range entpacks {
+				entid, _ := redis.GetRedisStr("winner", winnerbd, name)
+				if entid == "" {
+					data := comHisMegerNewData(winner, "winner", []map[string]interface{}{contact})
+					if data != nil {
+						_id := MongoTo.Save(winnerent, data)
+						redis.PutRedis("winner", winnerbd, winner, _id.(primitive.ObjectID).Hex(), -1)
+					}
+				} else {
+					data := winMegerIndustry(entid, contact)
+					MongoTo.UpdateById(winnerent, entid,
+						map[string]interface{}{
+							"$set":  data,
+							"$push": map[string]interface{}{"contact": contact},
+						},
+					)
+				}
+			}
+		}
+		tmp = map[string]interface{}{}
+		if index%100 == 0 {
+			log.Println("winner index", index)
+		}
+	}
+	log.Println("winner ok index", index)
+}
+
+//历史数据处理
+func historywinner(db, fromcoll string) {
+	defer qu.Catch()
+	log.Println("history  start")
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(fromcoll).Find(map[string]interface{}{}).Select(bson.M{"repeat": 1, "winner": 1, "winnertel": 1, "winnerperson": 1, "topscopeclass": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
+			continue
+		}
+		_id := mongo.BsonTOStringId(tmp["_id"])
+		winchanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-winchanbool
+			}()
+			winner := qu.ObjToString(tmp["winner"])
+			topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+			if winner != "" && utf8.RuneCountInString(winner) > 4 {
+				winnerperson := qu.ObjToString(tmp["winnerperson"])
+				winnertel := qu.ObjToString(tmp["winnertel"])
+				b, _ := redis.ExistRedis("winner", winnerbd, winner)
+				if b {
+					if winnerperson != "" || winnertel != "" {
+						strs, _ := redis.GetRedisStr("winner", winnerbd, winner)
+						ps := []interface{}{}
+						err := json.Unmarshal([]byte(strs), &ps)
+						if err == nil {
+							v := map[string]interface{}{
+								"contact_person": winnerperson,
+								"phone":          winnertel,
+								"topscopeclass":  comRepTopscopeclass(topscopeclass),
+								"infoid":         _id,
+							}
+							ps = append(ps, v)
+							bs, _ := json.Marshal(ps)
+							redis.PutRedis("winner", winnerbd, winner, bs, -1)
+							//log.Println(_id, index, winner)
+						} else {
+							log.Println("jsonErr", err)
+						}
+					}
+				} else {
+					val := []map[string]interface{}{}
+					if winnerperson != "" || winnertel != "" {
+						tmp := map[string]interface{}{
+							"contact_person": winnerperson,
+							"phone":          winnertel,
+							"topscopeclass":  comRepTopscopeclass(topscopeclass),
+							"infoid":         _id,
+						}
+						val = append(val, tmp)
+					}
+					bs, _ := json.Marshal(val)
+					redis.PutRedis("winner", winnerbd, winner, bs, -1)
+					MongoTo.Save(winnererr, map[string]interface{}{
+						"name":          winner,
+						"topscopeclass": comRepTopscopeclass(topscopeclass),
+						"updatetime":    time.Now().Unix(),
+					})
+				}
+			}
+		}(tmp)
+		tmp = map[string]interface{}{}
+		if index%10000 == 0 {
+			log.Println("index", index, _id)
+		}
+	}
+	log.Println("history  ok  index", index)
+	winStandarHistory(qu.ObjToString(sysconfig["mgotodb"]))
+}
+
+//查询winnererr标准化历史数据
+func winStandarHistory(db string) {
+	defer qu.Catch()
+	log.Println("开始标准化数据--winner", db)
+	sessto := MongoTo.GetMgoConn()
+	defer MongoTo.Close()
+	it := sessto.DB(db).C(winnererr).Find(map[string]interface{}{}).Iter()
+	index := 0
+	entnum := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		err_id := mongo.BsonTOStringId(tmp["_id"])
+		name := qu.ObjToString(tmp["name"])
+		winchanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-winchanbool
+			}()
+			strs, err := redis.GetRedisStr("winner", winnerbd, name)
+			if err != nil {
+				return
+			}
+			ps := []map[string]interface{}{}
+			err = json.Unmarshal([]byte(strs), &ps)
+			if err == nil {
+				data := comHisMegerNewData(name, "winner", ps)
+				if data != nil {
+					MongoTo.Save(winnerent, data)
+					MongoTo.DeleteById(winnererr, err_id)
+					entnum++
+				} else { //未查询到企业，打标记并存表
+					num := comMarkdata(name, "winner")
+					tmp["check"] = num
+					MongoTo.UpdateById(winnererr, err_id, map[string]interface{}{"$set": map[string]interface{}{"check": num}})
+				}
+			} else {
+				log.Println("jsonErr", name, err)
+			}
+		}(tmp)
+		if index%1000 == 0 {
+			log.Println("标准化历史数据--winner", index, err_id, entnum)
+		}
+		tmp = map[string]interface{}{}
+	}
+	log.Println("标准化数据完成--winner", index, entnum)
+}
+
+//企业数据整合(已有标注信息)
+func winMegerIndustry(id string, ps map[string]interface{}) map[string]interface{} {
+	tmp := MongoEnt.FindById(winnerent, id, bson.M{"industry": 1})
+	if len(tmp) < 1 {
+		return nil
+	}
+	data := map[string]interface{}{}
+	industry := tmp["industry"].(primitive.A)
+	tmpindustry := map[string]bool{}
+	for _, v := range industry {
+		tt := qu.ObjToString(v)
+		tmpindustry[tt] = true
+	}
+	if topscopeclass, ok := ps["topscopeclass"].([]interface{}); ok {
+		for _, v := range topscopeclass {
+			tt := qu.ObjToString(v)
+			tmpindustry[tt] = true
+		}
+	}
+	newindustry := []interface{}{}
+	for k, _ := range tmpindustry {
+		newindustry = append(newindustry, k)
+	}
+	data["industry"] = newindustry
+	return data
+}
+
+//中标单位分包提取联系方式
+func getWinnerPacks(infoid string, packs map[string]interface{}, topscopeclass []interface{}) map[string]map[string]interface{} {
+	entmappacks := map[string]map[string]interface{}{}
+	for _, v := range packs {
+		if tmp, ok := v.(map[string]interface{}); ok {
+			winner := qu.ObjToString(tmp["winner"])
+			if utf8.RuneCountInString(winner) < 5 {
+				continue
+			}
+			winnerperson := qu.ObjToString(tmp["winnerperson"])
+			winnertel := qu.ObjToString(tmp["winnertel"])
+			if winnerperson != "" || winnertel != "" {
+				p := map[string]interface{}{
+					"contact_person": winnerperson,
+					"phone":          winnertel,
+					"topscopeclass":  topscopeclass,
+					"infoid":         infoid,
+					"extfrom":        "package",
+				}
+				entmappacks[winner] = p
+			}
+		}
+	}
+	return entmappacks
+}
+
+//数据整合
+func comHisMegerNewData(name, datatype string, ps []map[string]interface{}) map[string]interface{} {
+	tmp := MongoEnt.FindOne("qyxy", map[string]interface{}{"company_name": name})
+	if len(tmp) < 1 {
+		return nil
+	}
+	data := map[string]interface{}{
+		"history_name":    "",
+		"credit_no":       "",
+		"area_code":       qu.ObjToString(tmp["area_code"]),
+		"province":        qu.ObjToString(tmp["province"]),
+		"city":            "",
+		"district":        "",
+		"company_type":    qu.ObjToString(tmp["company_type"]),
+		"legal_person":    qu.ObjToString(tmp["legal_person"]),
+		"company_address": qu.ObjToString(tmp["company_address"]),
+		"business_scope":  qu.ObjToString(tmp["business_scope"]),
+		"wechat_accounts": []interface{}{},
+		"website":         "",
+		"contact":         ps,
+		"comeintime":      time.Now().Unix(),
+		"updatetime":      time.Now().Unix(),
+	}
+	//统一信用代码
+	credit_no := strings.TrimSpace(qu.ObjToString(tmp["credit_no"]))
+	if credit_no != "" {
+		data["credit_no"] = credit_no
+		if len(credit_no) > 8 {
+			dataNo := credit_no[2:8]
+			if Addrs[dataNo] != nil {
+				if v, ok := Addrs[dataNo].(map[string]interface{}); ok {
+					if data["province"] == "" {
+						data["province"] = v["province"]
+					}
+					data["city"] = v["city"]
+					data["district"] = v["district"]
+				}
+			}
+		}
+	}
+
+	//网址
+	annual_reports := tmp["annual_reports"]
+	if annual_reports != nil {
+		report_websitesArr := []string{}
+		if anreports, ok := annual_reports.([]interface{}); ok {
+			for _, report_websites := range anreports {
+				if websites, ok := report_websites.([]interface{}); ok {
+					for _, website := range websites {
+						if rv, ok := website.(map[string]interface{}); ok {
+							web := qu.ObjToString(rv["website_url"])
+							if web != "" {
+								report_websitesArr = append(report_websitesArr, web)
+							}
+						}
+					}
+				}
+			}
+		}
+		if len(report_websitesArr) > 0 {
+			data["website"] = strings.Join(report_websitesArr, ";")
+		}
+	}
+
+	if datatype == "winner" {
+		data["company_name"] = name
+		data["partners"] = tmp["partners"]
+		establish_date := tmp["establish_date"]
+		if establish_date != nil {
+			data["establish_date"] = qu.Int64All(establish_date) / 1000
+		}
+		capital := tmp["capital"]
+		if capital != nil {
+			data["capital"] = ObjToMoney([]interface{}{capital, ""})[0]
+		}
+
+		industry := make([]string, 0)
+		tmpindustry := map[string]bool{}
+		for _, p := range ps {
+			if ts, ok := (p["topscopeclass"]).([]interface{}); ok {
+				for _, v := range ts {
+					tt := qu.ObjToString(v)
+					tmpindustry[tt] = true
+				}
+			}
+		}
+		for k, _ := range tmpindustry {
+			industry = append(industry, k)
+		}
+		data["industry"] = industry
+	} else if datatype == "buyer" {
+		data["buyer_name"] = name
+		tmpbuyerclass := map[string]bool{}
+		for _, p := range ps {
+			tmpbuyerclass[qu.ObjToString(p["buyerclass"])] = true
+		}
+		buyerclass := []interface{}{}
+		for k, _ := range tmpbuyerclass {
+			buyerclass = append(buyerclass, k)
+		}
+		data["buyerclass"] = buyerclass
+		data["ranks"] = ""
+		data["type"] = ""
+		data["address"] = ""
+	} else {
+		data["agency_name"] = name
+		data["ranks"] = ""
+		data["type"] = ""
+		data["address"] = ""
+	}
+	return data
+}
+
+//根据规则数据打标记
+func comMarkdata(name, datatype string) int {
+	tag := 0 //默认错误
+	switch datatype {
+	case "winner":
+		for _, v := range WinnerRegOk {
+			isok := v.MatchString(name)
+			if isok { //匹配ok完,匹配err
+				errflag := true
+				for _, vRegErr := range WinnerRegErr {
+					err := vRegErr.MatchString(name)
+					if err {
+						errflag = true
+						break
+					}
+				}
+				if errflag {
+					tag = 1
+				}
+			}
+		}
+	case "buyer":
+		for _, v := range BuyerRegOk {
+			isok := v.MatchString(name)
+			if isok { //匹配ok完,匹配err
+				errflag := true
+				for _, vRegErr := range BuyerRegErr {
+					err := vRegErr.MatchString(name)
+					if err {
+						errflag = true
+						break
+					}
+				}
+				if errflag {
+					tag = 1
+				}
+			}
+		}
+	case "agency":
+		for _, v := range AgencyRegOk {
+			isok := v.MatchString(name)
+			if isok { //匹配ok完,匹配err
+				errflag := true
+				for _, vRegErr := range AgencyRegErr {
+					err := vRegErr.MatchString(name)
+					if err {
+						errflag = true
+						break
+					}
+				}
+				if errflag {
+					tag = 1
+				}
+			}
+		}
+	default:
+	}
+	return tag
+}
+
+//过滤行业冗余字符
+func comRepTopscopeclass(tops []interface{}) []interface{} {
+	data := []interface{}{}
+	for _, v := range tops {
+		tt := qu.ObjToString(v)
+		if len(tt) > 1 {
+			data = append(data, tt[:len(tt)-1])
+		}
+	}
+	return data
+}
+
+//
+func comUpdateErr(coll, name string, tclass []interface{}) {
+	if len(tclass) < 1 {
+		return
+	}
+	tmp := MongoTo.FindOne(coll, map[string]interface{}{"name": name})
+	topscopeclass := tmp["topscopeclass"].(primitive.A)
+	tmpclass := map[string]bool{}
+	for _, tc := range topscopeclass {
+		tmpclass[qu.ObjToString(tc)] = true
+	}
+	oldlen := len(tmpclass)
+	for _, tc := range tclass {
+		tmpclass[qu.ObjToString(tc)] = true
+	}
+	newlen := len(tmpclass)
+	if oldlen == newlen {
+		return
+	}
+	newclass := []interface{}{}
+	for _, v := range tmpclass {
+		newclass = append(newclass, v)
+	}
+	MongoTo.Update(coll, map[string]interface{}{"name": name}, map[string]interface{}{
+		"$set": map[string]interface{}{
+			"name":          name,
+			"topscopeclass": newclass,
+			"updatetime":    time.Now().Unix(),
+		},
+	})
+}

standardata/src/task.go

@@ -0,0 +1,30 @@
+// task定时执行项目索引
+package main
+
+import (
+	"log"
+	qu "qfw/util"
+	"time"
+
+	"github.com/cron"
+)
+
+func task_standarData() {
+	mgofromdb := qu.ObjToString(sysconfig["mgofromdb"])
+	c := cron.New()
+	_ = c.AddFunc("0/5 * * * * *", func() {
+		t := time.Now()
+		pici := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local).Unix()
+		query := map[string]interface{}{
+			"comeintime": map[string]interface{}{
+				"$gt":  -pici - 86400,
+				"$lte": pici,
+			},
+		}
+		log.Println(mgofromdb, query)
+		//go winnerStandarData(mgofromdb, query)
+		go buyerStandarData(mgofromdb, query)
+		//go agencyStandarData(mgofromdb, query)
+	})
+	c.Start()
+}

standardata/src/util.go

@@ -0,0 +1,247 @@
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	regOperator, _ = regexp.Compile(`[*|+|)*)]`)
+	regNumFloat, _ = regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
+	regStrUnit, _  = regexp.Compile(`[元|万|亿]`)
+
+	regStrChar      = `[〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
+	moneyRegChar, _ = regexp.Compile(regStrChar)
+	contentUnit, _  = regexp.Compile(`(万元|单位/万)`)
+	numCapitals, _  = regexp.Compile(`([〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]{4,40})`)
+
+	regQianw, _ = regexp.Compile(`\d{1,2}千万`)
+	moneyChar   = map[string]interface{}{ //"〇": "0", "零": "0",
+		"一": float64(1), "壹": float64(1), "二": float64(2), "贰": float64(2), "三": float64(3), "叁": float64(3), "四": float64(4), "肆": float64(4), "五": float64(5), "伍": float64(5),
+		"六": float64(6), "陆": float64(6), "七": float64(7), "柒": float64(7), "八": float64(8), "捌": float64(8), "九": float64(9), "玖": float64(9), "十": float64(10), "拾": float64(10),
+		"百": float64(100), "佰": float64(100), "千": float64(1000), "仟": float64(1000), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000),
+		"零": float64(0), "点": ".", "角": float64(0.1), "分": float64(0.01),
+	}
+	moneyUnit = map[string]float64{
+		"元": float64(1), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000), //单位
+	}
+	cutAllSpace, _ = regexp.Compile(`\s*`)
+	spaces         = []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n"}
+)
+//大写数子金额转换
+func capitalMoney(data []interface{}) []interface{} {
+	nodes := []float64{}
+	node := float64(0)
+	tmp := float64(0)
+	decimals := 0.0
+	ishaspoint := false //是否含小数点
+	fnum := float64(0)
+	end := false
+	str := fmt.Sprint(data[0])
+	//提取第一个大写信息
+	strmatch := numCapitals.FindAllStringSubmatch(str, -1)
+	if len(strmatch) > 0 {
+		str = strmatch[0][0]
+	}
+	suffixUnit := float64(1)
+	if strings.HasSuffix(str, "万") || strings.HasSuffix(str, "万元") || strings.HasSuffix(str, "万元整") {
+		index := strings.LastIndex(str, "万")
+		str = str[0:index]
+		suffixUnit = float64(10000)
+	}
+	moneyRegChar.ReplaceAllStringFunc(str, func(key string) string {
+		if key == "元" || key == "圆" || key == "点" {
+			ishaspoint = true
+		}
+		if v, ok := moneyChar[key].(float64); ok && !end {
+			if ishaspoint && v > 10 { //排除后面有其他的单位
+				return ""
+			}
+			//fmt.Println(key, v, fnum)
+			if v < 10 && v >= 0 {
+				if ishaspoint { //小数部分
+					if v >= 1 {
+						fnum = v
+					} else if v < 1 && v > 0 {
+						decimals += fnum * v
+					}
+				} else {
+					if tmp != float64(0) {
+						node += tmp
+					}
+					tmp = float64(v)
+				}
+			} else if v == 10000 || v == 100000000 { //单位万、亿
+				if tmp != float64(0) {
+					node += tmp
+					tmp = float64(0)
+				}
+				nodes = append(nodes, node*float64(v))
+				node = float64(0)
+			} else {
+				if v == 10 && tmp == 0 {
+					tmp = 1
+				}
+				tmp = tmp * float64(v)
+				node += tmp
+				tmp = float64(0)
+			}
+		}
+		if key == "整" || key == "正" || key == "分" {
+			end = true
+		}
+		return ""
+	})
+	nodes = append(nodes, node, tmp)
+	ret := float64(0)
+	for _, v := range nodes {
+		ret += v
+	}
+	return []interface{}{(ret + decimals) * suffixUnit, data[1]}
+}
+
+//金额转换
+func ObjToMoney(data []interface{}) []interface{} {
+	isfindUnit := true
+	ret := capitalMoney(data)[0]
+	if ret.(float64) < float64(10000) || ret.(float64) > float64(50000000000) {
+		ret2, b := numMoney(data)
+		isfindUnit = b
+		if ret2[0].(float64) > ret.(float64) {
+			ret = ret2[0]
+		}
+	}
+	f, _ := strconv.ParseFloat(strconv.FormatFloat(ret.(float64), 'f', 4, 64), 64)
+	if f < 1 {
+		f = 0
+	}
+	//若果金额小于50，全文检索单位：万
+	if f < 50 && f > 0 && isfindUnit {
+		rep := contentUnit.FindAllStringIndex(fmt.Sprint(data[1]), -1)
+		if len(rep) > 0 {
+			f = f * 10000
+		}
+	}
+	data[0] = f
+	return data
+}
+
+//符号替换
+func replaceString(con string, ret, rep []string) string {
+	for k, v := range ret {
+		if len(rep) > k {
+			con = strings.Replace(con, v, rep[k], -1)
+		}
+	}
+	return con
+}
+
+//清理所有空白符
+func CutAllSpace(data []interface{}) []interface{} {
+	tmp := cutAllSpace.ReplaceAllString(fmt.Sprint(data[0]), "")
+	tmp = replaceSymbol(tmp, spaces)
+	data[0] = tmp
+	return data
+}
+
+//数字金额转换
+func numMoney(data []interface{}) ([]interface{}, bool) {
+	tmp := fmt.Sprintf("%f", data[0])
+	repUnit := float64(1)
+	if regQianw.MatchString(tmp) {
+		tmp = strings.Replace(tmp, "千万", "万", -1)
+		repUnit = float64(1000)
+	}
+	tmp = replaceSymbol(tmp, []string{",", "，", "（", "）", "(", ")", "：", "\n"})
+	tmp = replaceString(tmp, []string{"万元", "亿元", "．"}, []string{"万", "亿", "."})
+	tmp = fmt.Sprint(CutAllSpace([]interface{}{tmp, data[1]})[0])
+	rets := regNumFloat.FindAllString(tmp, -1)
+	fnums := []float64{}
+	unitstrs := []string{}
+	if len(rets) > 0 {
+		pindex := 0 //单位前置
+		for k, v := range rets {
+			f, err := strconv.ParseFloat(v, 64)
+			if err == nil {
+				fnums = append(fnums, f)
+				index := strings.Index(tmp, v)
+				//单位后置
+				start := index + len(v)
+				end := start + 3
+				//log.Println("vvv", tmp, v, pindex, index, start)
+				if k > 0 {
+					if start >= pindex+3 {
+						pstart := pindex + 3
+						if pstart >= index {
+							pstart = index
+						}
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:])
+						}
+					} else {
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				} else {
+					if len(tmp) > end {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:end])
+						}
+					} else {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				}
+				pindex = start
+			}
+		}
+	}
+	//log.Println("unitstrs", fnums, unitstrs)
+	unit := float64(0)
+	fnum := float64(0)
+	for k, v := range fnums {
+		fnum = v
+		units := regStrUnit.FindAllString(unitstrs[k], -1)
+		for _, v := range units {
+			if moneyUnit[v] != 0 {
+				unit = moneyUnit[v]
+				break
+			}
+		}
+		if unit != float64(0) { //取第一个
+			break
+		}
+	}
+	fnum = fnum * repUnit
+	if unit == float64(0) {
+		data[0] = fnum
+	} else {
+		data[0] = fnum * unit
+	}
+	if unit == 10000 {
+		return data, false
+	} else {
+		return data, true
+	}
+}
+
+//过滤符号
+func replaceSymbol(con string, rep []string) string {
+	for _, v := range rep {
+		con = strings.Replace(con, v, "", -1)
+	}
+	return con
+}
+

util/src/dbutil/mongo/mgo.go

@@ -146,7 +146,7 @@ func (m *MongodbSim) InitPool() {
 	opts := options.Client()
 	opts.SetConnectTimeout(3 * time.Second)
 	opts.ApplyURI("mongodb://" + m.MongodbAddr)
-	opts.SetMaxPoolSize(uint64(m.Size))
+	opts.SetMaxPoolSize(uint16(m.Size))
 	m.pool = make(chan bool, m.Size)
 	opts.SetMaxConnIdleTime(2 * time.Hour)
 	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)
@@ -237,6 +237,24 @@ func (m *MongodbSim) UpdateById(c, id string, doc map[string]interface{}) bool {
 	return true
 }
 
+//更新Update
+func (m *MongodbSim) Update(c string, query map[string]interface{}, up map[string]interface{}) (map[int64]interface{}, bool) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	write := mongo.NewUpdateOneModel()
+	write.SetFilter(query)
+	write.SetUpdate(up)
+	write.SetUpsert(true)
+	writes = append(writes, write)
+	r, err := coll.BulkWrite(m.Ctx, writes)
+	if err != nil {
+		return nil, false
+	}
+	return r.UpsertedIDs, true
+}
+
 //删除by id
 func (m *MongodbSim) DeleteById(c, id string) int64 {
 	m.Open()
@@ -262,11 +280,12 @@ func (m *MongodbSim) Delete(c string, query map[string]interface{}) int64 {
 }
 
 //findbyid
-func (m *MongodbSim) FindById(c, id string) map[string]interface{} {
+func (m *MongodbSim) FindById(c, id string, fields interface{}) map[string]interface{} {
 	m.Open()
 	defer m.Close()
 	coll := m.C.Database(m.DbName).Collection(c)
-	r := coll.FindOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
+	op := options.FindOne()
+	r := coll.FindOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)}, op.SetProjection(fields))
 	v := map[string]interface{}{}
 	r.Decode(&v)
 	return v
@@ -277,6 +296,8 @@ func (m *MongodbSim) FindOne(c string, query map[string]interface{}) map[string]
 	m.Open()
 	defer m.Close()
 	coll := m.C.Database(m.DbName).Collection(c)
+	//op := options.FindOne()
+	//r := coll.FindOne(m.Ctx, query, op.SetProjection(fields))
 	r := coll.FindOne(m.Ctx, query)
 	v := map[string]interface{}{}
 	r.Decode(&v)