Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

* 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract:
  table key
  分包
  修改
  修改
  修改
  xg
  代理机构 增存量，联系人相同过滤修改
  数据审核、title抽buyer打分
  代理机构
  标准库
  标准库
  config

src/config.json

@@ -28,7 +28,7 @@
     "filelength": 100000,
     "iscltlog": false,
     "brandgoods": false,
-    "udptaskid": "5cdd3021698414032c8322b1",
+    "udptaskid": "5cdd3025698414032c8322b1",
     "udpport": "1484",
     "nextNode": [
         {

src/jy/admin/audit/dataaudit.go

@@ -10,6 +10,7 @@ import (
 	qu "qfw/util"
 	"qfw/util/elastic"
 	redis "qfw/util/redis"
+	"strconv"
 	"strings"
 	"time"
 
@@ -70,16 +71,17 @@ func AuditOneField(c *gin.Context) {
 	start := c.GetInt("start")
 	limit := c.GetInt("length")
 	auditattr, _ := c.GetPostForm("auditattr")
+	check, _ := strconv.Atoi(auditattr)
 	query := map[string]interface{}{}
-	if auditattr != "-1" {
+	if check != -1 {
 		query = map[string]interface{}{
-			field + "_" + auditattr: 1,
+			"check": check,
 		}
 	} else {
 		query = map[string]interface{}{
 			"$or": []interface{}{
-				map[string]interface{}{field + "_ok": 1},
-				map[string]interface{}{field + "_err": 1},
+				map[string]interface{}{"check": 1},
+				map[string]interface{}{"check": 0},
 			},
 		}
 	}
@@ -107,24 +109,24 @@ func AllAudit(c *gin.Context) {
 	} else { //批量审核
 		SaveDb := ""
 		FieldBd := 0
-		ElasticClientIndex := ""
-		ElasticClientType := ""
+		// ElasticClientIndex := ""
+		// ElasticClientType := ""
 		RedisName := util.QYK_RedisName
 		if field == "winner" {
 			SaveDb = util.ElasticClientDB
 			FieldBd = util.WinnerDB
-			ElasticClientIndex = util.ElasticClientIndex
-			ElasticClientType = util.ElasticClientType
+			// ElasticClientIndex = util.ElasticClientIndex
+			// ElasticClientType = util.ElasticClientType
 		} else if field == "buyer" {
 			SaveDb = util.ElasticClientBuyerDB
 			FieldBd = util.BuyerDB
-			ElasticClientIndex = util.ElasticClientBuyerIndex
-			ElasticClientType = util.ElasticClientBuyerType
+			// ElasticClientIndex = util.ElasticClientBuyerIndex
+			// ElasticClientType = util.ElasticClientBuyerType
 		} else {
 			SaveDb = util.ElasticClientAgencyDB
 			FieldBd = util.AgencyDB
-			ElasticClientIndex = util.ElasticClientAgencyIndex
-			ElasticClientType = util.ElasticClientAgencyType
+			// ElasticClientIndex = util.ElasticClientAgencyIndex
+			// ElasticClientType = util.ElasticClientAgencyType
 		}
 		//redis
 		qykredis := redis.RedisPool[RedisName].Get()
@@ -156,11 +158,11 @@ func AllAudit(c *gin.Context) {
 						return
 					}
 				}
-				_, err := escon.Index().Index(ElasticClientIndex).Type(ElasticClientType).Id(sid).BodyJson(e).Refresh(true).Do()
-				if err != nil {
-					c.JSON(200, gin.H{"rep": false, "msg": "更新es错误"})
-					return
-				}
+				// _, err := escon.Index().Index(ElasticClientIndex).Type(ElasticClientType).Id(sid).BodyJson(e).Refresh(true).Do()
+				// if err != nil {
+				// 	c.JSON(200, gin.H{"rep": false, "msg": "更新es错误"})
+				// 	return
+				// }
 			}
 			//删除标记数据
 			query := map[string]interface{}{
@@ -177,8 +179,8 @@ func AllAudit(c *gin.Context) {
 func DataSave(c *gin.Context) {
 	SaveDb := ""
 	FieldBd := 0
-	ElasticClientIndex := ""
-	ElasticClientType := ""
+	// ElasticClientIndex := ""
+	// ElasticClientType := ""
 	RedisName := util.QYK_RedisName
 	//企业名称
 	e := make(map[string]interface{})
@@ -188,8 +190,8 @@ func DataSave(c *gin.Context) {
 	if field == "winner" {
 		SaveDb = util.ElasticClientDB
 		FieldBd = util.WinnerDB
-		ElasticClientIndex = util.ElasticClientIndex
-		ElasticClientType = util.ElasticClientType
+		// ElasticClientIndex = util.ElasticClientIndex
+		// ElasticClientType = util.ElasticClientType
 		capital, _ := c.GetPostForm("capital")
 		capitalfloat := clear.ObjToMoney([]interface{}{capital, ""})[0]
 		business_scope, _ := c.GetPostForm("business_scope")
@@ -200,8 +202,8 @@ func DataSave(c *gin.Context) {
 	} else if field == "buyer" {
 		SaveDb = util.ElasticClientBuyerDB
 		FieldBd = util.BuyerDB
-		ElasticClientIndex = util.ElasticClientBuyerIndex
-		ElasticClientType = util.ElasticClientBuyerType
+		// ElasticClientIndex = util.ElasticClientBuyerIndex
+		// ElasticClientType = util.ElasticClientBuyerType
 		buyerclass, _ := c.GetPostForm("buyerclass")
 		ranks, _ := c.GetPostForm("ranks")
 		buyer_type, _ := c.GetPostForm("type")
@@ -213,8 +215,8 @@ func DataSave(c *gin.Context) {
 	} else {
 		SaveDb = util.ElasticClientAgencyDB
 		FieldBd = util.AgencyDB
-		ElasticClientIndex = util.ElasticClientAgencyIndex
-		ElasticClientType = util.ElasticClientAgencyType
+		// ElasticClientIndex = util.ElasticClientAgencyIndex
+		// ElasticClientType = util.ElasticClientAgencyType
 		ranks, _ := c.GetPostForm("ranks")
 		agency_type, _ := c.GetPostForm("type")
 		e["ranks"] = ranks
@@ -279,13 +281,13 @@ func DataSave(c *gin.Context) {
 				return
 			}
 		}
-		escon := elastic.GetEsConn()
-		defer elastic.DestoryEsConn(escon)
-		_, err := escon.Index().Index(ElasticClientIndex).Type(ElasticClientType).Id(sid).BodyJson(e).Refresh(true).Do()
-		if err != nil {
-			c.JSON(200, gin.H{"rep": false, "msg": "更新es错误"})
-			return
-		}
+		// escon := elastic.GetEsConn()
+		// defer elastic.DestoryEsConn(escon)
+		// _, err := escon.Index().Index(ElasticClientIndex).Type(ElasticClientType).Id(sid).BodyJson(e).Refresh(true).Do()
+		// if err != nil {
+		// 	c.JSON(200, gin.H{"rep": false, "msg": "更新es错误"})
+		// 	return
+		// }
 	}
 	//删除标记数据
 	coll, _ := c.GetPostForm("coll")

src/jy/extract/extpackage.go

@@ -148,6 +148,9 @@ func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 					sonJobResult["origin"] = pkg.Origin
 					sonJobResult["text"] = pkg.Text
 					sonJobResult["name"] = pkg.Name
+					if pkg.Winner!= ""{
+						sonJobResult["winner"] = pkg.Winner
+					}
 					if pkg.WinnerPerson != "" {
 						sonJobResult["winnertel"] = pkg.WinnerTel
 						sonJobResult["winnerperson"] = pkg.WinnerPerson
@@ -177,7 +180,7 @@ func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 							}
 						}
 					} else {
-						if len(j.Winnerorder) > 0 {
+						if sonJobResult["winner"] == "" && len(j.Winnerorder) > 0 {
 							if j.Winnerorder[0]["price"] != nil {
 								sonJobResult["bidamount"] = qu.Float64All(j.Winnerorder[0]["price"])
 							}

src/jy/extract/extract.go

@@ -27,12 +27,12 @@ import (
 var (
 	lock, lockrule, lockclear, locktag, blocktag sync.RWMutex
 
-	cut     = ju.NewCut()                          //获取正文并清理
-	ExtLogs map[*TaskInfo][]map[string]interface{} //抽取日志
-	TaskList      map[string]*ExtractTask          //任务列表
-	ClearTaskList map[string]*ClearTask            //清理任务列表
-	saveLimit     = 100                            //抽取日志批量保存
-	PageSize      = 5000                           //查询分页
+	cut           = ju.NewCut()                          //获取正文并清理
+	ExtLogs       map[*TaskInfo][]map[string]interface{} //抽取日志
+	TaskList      map[string]*ExtractTask                //任务列表
+	ClearTaskList map[string]*ClearTask                  //清理任务列表
+	saveLimit     = 100                                  //抽取日志批量保存
+	PageSize      = 5000                                 //查询分页
 	Fields        = `{"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"bidstatus":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1,"href":1,"infoformat":1}`
 	Fields2       = `{"budget":1,"bidamount":1,"title":1,"projectname":1,"winner":1}`
 )
@@ -583,7 +583,11 @@ func (e *ExtractTask) ExtractDetail(j *ju.Job, isSite bool, codeSite string) {
 		//函数清理
 		for key, val := range j.Result {
 			for i, v := range val {
-				//qu.Debug(key, v.Value)
+				// if v.ExtFrom == "title"&& v.Field == "buyer"{
+				// 	qu.Debug("title---",v.Value)
+				// }else if v.Field == "buyer"{
+				// 	qu.Debug("text---",v.Value)
+				// }
 				lockclear.Lock()
 				var cfn = []string{}
 				if isSite {
@@ -930,6 +934,9 @@ func ExtRuleCoreByPkgReg(j *ju.Job, in *RegLuaInfo, e *ExtractTask) {
 				if in.Field == "projectname" && vbpkg.Name != "" {
 					continue
 				}
+				if in.Field == "winner" && vbpkg.Winner != "" {
+					continue
+				}
 				if in.Field == "winnerperson" {
 					if vbpkg.Winner == "" || len(vbpkg.Winner) < 4 {
 						continue
@@ -1769,6 +1776,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 			if (savewinner == nil || len(savewinner) == 0) && tmp["winner"] != nil {
 				tmp["s_winner"] = tmp["winner"]
 			} else if savewinner != nil {
+				savewinner = RemoveReplicaSliceString(savewinner)
 				tmp["s_winner"] = strings.Join(savewinner, ",")
 			}
 
@@ -1777,6 +1785,11 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 			tmp["s_winner"] = tmp["winner"]
 		}
 		if len(j.Winnerorder) > 0 { //候选人信息
+			for i,v := range j.Winnerorder{
+				if v["price"]!= nil{
+					j.Winnerorder[i]["price"] = clear.ObjToMoney([]interface{}{v["price"],""})[0]
+				}
+			}
 			tmp["winnerorder"] = j.Winnerorder
 		}
 		//处理附件
@@ -1940,7 +1953,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 					tmp["epackage"] = string(bs)
 				}
 			}
-			//tmp["result"] = result
+			tmp["result"] = result
 			tmp["resultf"] = resultf
 			b := db.Mgo.Update(e.TaskInfo.TestColl, `{"_id":"`+_id+`"}`, map[string]interface{}{"$set": tmp}, true, false)
 			if !b {
@@ -2130,7 +2143,7 @@ func (e *ExtractTask) QualityAudit(resulttmp map[string]interface{}) {
 func (e *ExtractTask) RedisMatch(field, fv string, val map[string]interface{}) {
 	defer qu.Catch()
 	i := redis.GetInt(field, field+"_"+fv) //查找redis
-	if i == 0 { //reids未找到，执行规则匹配
+	if i == 0 {                            //reids未找到，执行规则匹配
 		val[field+"_isredis"] = false
 		e.RuleMatch(field, fv, val) //规则匹配
 	} else { //redis找到，打标识存库
@@ -2256,3 +2269,14 @@ func resetWinnerorder(j *ju.Job) {
 	//j.Result["bidamount"] = bidamounts
 
 }
+func RemoveReplicaSliceString(slc []string) []string {
+	result := make([]string, 0)
+	tempMap := make(map[string]bool, len(slc))
+	for _, e := range slc{
+		if tempMap[e] == false{
+			tempMap[e] = true
+			result = append(result, e)
+		}
+	}
+	return result
+}

src/jy/extract/score.go

@@ -30,6 +30,7 @@ func init() {
 	qu.ReadConfig("./res/tagscore.json", &TagConfig)
 	qu.ReadConfig("./res/fieldscore.json", &SoreConfig)
 	if SoreConfig == nil { //配置出错，强退
+		fmt.Println("fieldscore.json配置文件出错，强制退出！")
 		os.Exit(0)
 	}
 	if repeat, ok := SoreConfig["other"]["repeat"].(map[string]interface{}); ok {
@@ -105,6 +106,7 @@ func init() {
 }
 
 var CNreg = regexp.MustCompile("[\u4e00-\u9fa5]")
+
 //结果打分
 func ScoreFields(j *ju.Job, ftag map[string][]*Tag) map[string][]*ju.ExtField {
 	qu.Catch()
@@ -115,9 +117,9 @@ func ScoreFields(j *ju.Job, ftag map[string][]*Tag) map[string][]*ju.ExtField {
 		}
 		if field == "budget" || field == "bidamount" {
 			for tmpsindex, tmpsvalue := range tmps {
-				if strings.Contains(tmpsvalue.RuleText,"总") && tmpsvalue.Type =="colon"{
+				if strings.Contains(tmpsvalue.RuleText, "总") && tmpsvalue.Type == "colon" {
 					tmps[tmpsindex].Score += 1
-					tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: field+`value结果含总字+1`, Code: field, Value: tmpsvalue.Value, Score:1})
+					tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: field + `value结果含总字+1`, Code: field, Value: tmpsvalue.Value, Score: 1})
 				}
 			}
 		}
@@ -126,15 +128,15 @@ func ScoreFields(j *ju.Job, ftag map[string][]*Tag) map[string][]*ju.ExtField {
 		locktag.Unlock()
 		for tmpsindex, tmpsvalue := range tmps {
 
-			if tmpsvalue.Score>0 {
+			if tmpsvalue.Score > 0 {
 				//有初始分先添加进去
 				tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{
-					Des: "正则初始分",
-					Code: tmpsvalue.Code,
-					RuleText: tmpsvalue.RuleText,
+					Des:       "正则初始分",
+					Code:      tmpsvalue.Code,
+					RuleText:  tmpsvalue.RuleText,
 					ScoreFrom: "正则初始分",
-					Value: tmpsvalue.Value,
-					Score: tmpsvalue.Score,
+					Value:     tmpsvalue.Value,
+					Score:     tmpsvalue.Score,
 				})
 			}
 

src/jy/pretreated/analystep.go

@@ -53,6 +53,7 @@ func AnalyStart(job *util.Job, isSite bool, codeSite string) {
 		//log.Println(con)
 		bl := &util.Block{}
 		newCon := con
+		//log.Println(con)
 		if len(tabs) > 0 { //解析表格逻辑
 			job.HasTable = 1 //添加标识:文本中有table
 			newCon = TextAfterRemoveTable(con)
@@ -105,7 +106,7 @@ func AnalyStart(job *util.Job, isSite bool, codeSite string) {
 										job.BlockPackage[k].WinnerOrder = append(job.BlockPackage[k].WinnerOrder, map[string]interface{}{
 											"type":    0,
 											"price":   0.0,
-											"entname": vv.Value,
+											"entname": strings.TrimSpace(vv.Value),
 											"sort":    tmpw,
 										})
 										tmpw++

src/jy/pretreated/analytable.go

@@ -49,7 +49,7 @@ var (
 	FindVal_1  = regexp.MustCompile("[第]?([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)((子|合同|分|施工|监理)?(标段?|包|合同段|标包))|((子|合同|分|施工|监理)?(标|包)(段|号)?)[ 　\u3000\u2003\u00a0]*([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)")
 	FindVal2_1 = regexp.MustCompile("([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+)|^(设计|施工|监理|验收)[分子]?[标包]?[段号]?$")
 	//判断分包前排除
-	excludeKey  = regexp.MustCompile("(标段代码|涉及包号|分包数量|包件号?|项目标号|规格|型号|招标范围|业绩|废标)|(^编号$)|([^包段标]编号)") //编号|划分
+	excludeKey  = regexp.MustCompile("(标段代码|涉及包号|分包数量|包件号?|项目标号|规格|型号|招标范围|业绩|废标|标段选择要求)|(^编号$)|([^包段标]编号)") //编号|划分
 	excludeKey2 = regexp.MustCompile("包/[0-9]{0,4}[|箱|纸|张]")
 	//-------------
 
@@ -2324,7 +2324,7 @@ func (tn *Table) isGoonNext(isSite bool, codeSite string) {
 func foundPacBySortKV(tn *Table, val int, index []string, index_pos []int, keyExistsCount *map[string]int, commonKeyVals *map[string][]string, key_index int, hasPkgTd map[string]bool) (rval int, rindex []string, rindex_pos []int) {
 	keyIsPkg := false
 	for in, k := range tn.SortKV.Keys {
-		if excludeKey.MatchString(BracketsTextReg.ReplaceAllString(k, "")) { //判断分包前排除
+		if excludeKey.MatchString(BracketsTextReg.ReplaceAllString(k, "")) ||regPDFWarap.MatchString(k)||regAZWarap.MatchString(k){ //判断分包前排除
 			continue
 		}
 		v := tn.SortKV.Map[k]

src/jy/pretreated/division.go

@@ -57,10 +57,15 @@ var (
 	regMoreWrap        = regexp.MustCompile("[\r\n]{2,}")
 	regStrWrap         = regexp.MustCompile("分包名称[：:]")
 	regBZJWarap        = regexp.MustCompile("保证金.*")
+	regPDFWarap        = regexp.MustCompile("[a-zA-Z](包|标段).(pdf|PDF)")
+	regAZWarap         = regexp.MustCompile("(标[a-zA-Z]取值|标段划分)")
 	replSerial         = regexp.MustCompile("(\r\n|^)([\\d一二三四五六七八九十][、．.:：，])+\\d")
 	moreColonReg       = regexp.MustCompile("[:：]+")
 	regFilter          = regexp.MustCompile("等$")
 	pkgFilter          = regexp.MustCompile("第[一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ](子|合同|分|施工|监理)?(标段?|包|合同段|标包)|(子|合同|分|施工|监理)?(标|包)(段|号)?")
+	indexTile          = regexp.MustCompile("[0-9.]{2,3}[\\s\u4e00-\u9fa5]{2,8}[:：]+") //小标题
+	indexTile2         = regexp.MustCompile("[\\s\u4e00-\u9fa5]{2,8}")
+	regReplAllSpace2   = regexp.MustCompile("[\u3000\u2003\u00a0\\s0-9.:：、\\(\\)]+")
 	confusion          = map[string]string{
 		"参与": "canyu",
 	}
@@ -566,63 +571,71 @@ func FindPackageFromBlocks(blocks *[]*util.Block, isSite bool, codeSite string)
 	//orderwinner := winnerOrderEntity.Find(content, true, 2, isSite, codeSite)
 
 	for k, v := range blockPackage {
-		if v.ColonKV != nil && v.ColonKV.KvTags != nil {
-			for kc, cv := range v.ColonKV.KvTags {
-				if kc == "预算" && v.Budget <= 0 {
-					moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
-					if len(moneys) > 0 {
-						if vf, ok := moneys[0].(float64); ok {
-							blockPackage[k].Budget = vf
-							blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
-						} else if vi, ok := moneys[0].(int); ok {
-							blockPackage[k].Budget = float64(vi)
-							blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
-						}
+		findWinnerBugetBidmountByKv(v, blockPackage, k)
+	}
+	return
+}
+
+func findWinnerBugetBidmountByKv(v *util.BlockPackage, blockPackage map[string]*util.BlockPackage, k string) {
+	if v.ColonKV != nil && v.ColonKV.KvTags != nil {
+		for kc, cv := range v.ColonKV.KvTags {
+			if kc == "预算" && v.Budget <= 0 {
+				moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
+				if len(moneys) > 0 {
+					if vf, ok := moneys[0].(float64); ok {
+						blockPackage[k].Budget = vf
+						blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
+					} else if vi, ok := moneys[0].(int); ok {
+						blockPackage[k].Budget = float64(vi)
+						blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
 					}
-				} else if kc == "中标金额" && v.Bidamount <= 0 {
-					moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
-					if len(moneys) > 0 {
-						if vf, ok := moneys[0].(float64); ok {
-							blockPackage[k].Bidamount = vf
-							blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
-						} else if vi, ok := moneys[0].(int); ok {
-							blockPackage[k].Bidamount = float64(vi)
-							blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
-						}
+				}
+			} else if kc == "中标金额" && v.Bidamount <= 0 {
+				moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
+				if len(moneys) > 0 {
+					if vf, ok := moneys[0].(float64); ok {
+						blockPackage[k].Bidamount = vf
+						blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
+					} else if vi, ok := moneys[0].(int); ok {
+						blockPackage[k].Bidamount = float64(vi)
+						blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
 					}
 				}
+			} else if kc == "中标单位" && v.Winner == "" {
+				blockPackage[k].Winner = cv[0].Value
 			}
 		}
-		if v.SpaceKV != nil && v.SpaceKV.KvTags != nil {
-			for kc, cv := range v.SpaceKV.KvTags {
-				if kc == "预算" && v.Budget <= 0 {
-					moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
-					if len(moneys) > 0 {
-						if vf, ok := moneys[0].(float64); ok {
-							blockPackage[k].Budget = vf
-							blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
-						} else if vi, ok := moneys[0].(int); ok {
-							blockPackage[k].Budget = float64(vi)
-							blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
-						}
+	}
+	if v.SpaceKV != nil && v.SpaceKV.KvTags != nil {
+		for kc, cv := range v.SpaceKV.KvTags {
+			if kc == "预算" && v.Budget <= 0 {
+				moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
+				if len(moneys) > 0 {
+					if vf, ok := moneys[0].(float64); ok {
+						blockPackage[k].Budget = vf
+						blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
+					} else if vi, ok := moneys[0].(int); ok {
+						blockPackage[k].Budget = float64(vi)
+						blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
 					}
+				}
 
-				} else if kc == "中标金额" && v.Bidamount <= 0 {
-					moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
-					if len(moneys) > 0 {
-						if vf, ok := moneys[0].(float64); ok {
-							blockPackage[k].Bidamount = vf
-							blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
-						} else if vi, ok := moneys[0].(int); ok {
-							blockPackage[k].Bidamount = float64(vi)
-							blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
-						}
+			} else if kc == "中标金额" && v.Bidamount <= 0 {
+				moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
+				if len(moneys) > 0 {
+					if vf, ok := moneys[0].(float64); ok {
+						blockPackage[k].Bidamount = vf
+						blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
+					} else if vi, ok := moneys[0].(int); ok {
+						blockPackage[k].Bidamount = float64(vi)
+						blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
 					}
 				}
+			} else if kc == "中标单位" && v.Winner == "" {
+				blockPackage[k].Winner = cv[0].Value
 			}
 		}
 	}
-	return
 }
 
 //从正文里面找分包
@@ -631,61 +644,7 @@ func FindPackageFromText(title string, content string, isSite bool, codeSite str
 	//从正文里面找分包
 	divisionPackageChild(&blockPackage, content, title, true, false, isSite, codeSite)
 	for k, v := range blockPackage {
-		if v.ColonKV != nil && v.ColonKV.KvTags != nil {
-			for kc, cv := range v.ColonKV.KvTags {
-				if kc == "预算" && v.Budget <= 0 {
-					moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
-					if len(moneys) > 0 {
-						if vf, ok := moneys[0].(float64); ok {
-							blockPackage[k].Budget = vf
-							blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
-						} else if vi, ok := moneys[0].(int); ok {
-							blockPackage[k].Budget = float64(vi)
-							blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
-						}
-					}
-				} else if kc == "中标金额" && v.Bidamount <= 0 {
-					moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
-					if len(moneys) > 0 {
-						if vf, ok := moneys[0].(float64); ok {
-							blockPackage[k].Bidamount = vf
-							blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
-						} else if vi, ok := moneys[0].(int); ok {
-							blockPackage[k].Bidamount = float64(vi)
-							blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
-						}
-					}
-				}
-			}
-		}
-		if v.SpaceKV != nil && v.SpaceKV.KvTags != nil {
-			for kc, cv := range v.SpaceKV.KvTags {
-				if kc == "预算" && v.Budget <= 0 {
-					moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
-					if len(moneys) > 0 {
-						if vf, ok := moneys[0].(float64); ok {
-							blockPackage[k].Budget = vf
-							blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
-						} else if vi, ok := moneys[0].(int); ok {
-							blockPackage[k].Budget = float64(vi)
-							blockPackage[k].IsTrueBudget = moneys[len(moneys)-1].(bool)
-						}
-					}
-
-				} else if kc == "中标金额" && v.Bidamount <= 0 {
-					moneys := clear.ObjToMoney([]interface{}{cv[0].Value, ""})
-					if len(moneys) > 0 {
-						if vf, ok := moneys[0].(float64); ok {
-							blockPackage[k].Bidamount = vf
-							blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
-						} else if vi, ok := moneys[0].(int); ok {
-							blockPackage[k].Bidamount = float64(vi)
-							blockPackage[k].IsTrueBidamount = moneys[len(moneys)-1].(bool)
-						}
-					}
-				}
-			}
-		}
+		findWinnerBugetBidmountByKv(v, blockPackage, k)
 	}
 	//winnerOrderEntity.Find(content, true, 2, isSite, codeSite)
 	return
@@ -694,6 +653,8 @@ func FindPackageFromText(title string, content string, isSite bool, codeSite str
 //分块之后分包
 func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content, title string, isFindWinnerOrder, accuracy bool, isSite bool, codeSite string) (bool, string) {
 	//查找知否有分包
+	content = regPDFWarap.ReplaceAllString(content, "\n")
+	content = regAZWarap.ReplaceAllString(content, "\n")
 	content = regStrWrap.ReplaceAllString(content, "\n")
 	content = regMoreWrap.ReplaceAllString(content, "\n")
 	content = regEndWrap.ReplaceAllString(content, "")
@@ -731,7 +692,6 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 	con = conTemp
 	con = replSerial.ReplaceAllString(con, "\n")
 	con = regMoreWrap.ReplaceAllString(con, "\n")
-	//log.Println(con)
 	//根据分包，找索引位置
 	indexMap := map[int]int{}
 	indexKeyStringMap := map[int]string{}
@@ -740,6 +700,11 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 	startEndMap := map[int]int{}
 	pkgIndexMap := map[string][]int{}
 	indexPkgMap := map[int]string{}
+	//小标题
+	titleindexs := indexTile.FindAllStringIndex(con, -1)
+	if len(titleindexs) == 0 {
+		titleindexs = indexTile2.FindAllStringIndex(con, -1)
+	}
 	//遍历分包，把kv在包前面的移动到包后面
 	for _, v := range pkg {
 		pgflag := v[0] + "[:：]*"
@@ -788,12 +753,12 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 			indexKeyStringMap[iv] = indexKeyStringMap[indexs[ik-1]]
 		}
 	}
-	//
 	//获取截取标识
 	surplusText, maxWarpCount, indexTextMap, indexWarpMap := interceptText(indexs, indexPkgMap, pkgIndexMap, startEndMap, con)
 	//查找分包内容，分kv
 	for _, iv := range indexs {
 		text := indexTextMap[iv]
+		tmptext := text
 		//
 		warpIndex := regSpliteSegment.FindAllStringIndex(text, -1)
 		if len(indexWarpMap) > 0 {
@@ -812,6 +777,20 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 			index := util.PackageNumberConvert(bk)
 			//去掉前缀,空格必须要加，分kv的时候要用
 			text = regexp.MustCompile(bv[0] + "[:：]*").ReplaceAllString(text, "")
+			if strings.TrimLeft(tmptext, bv[0]) == text || strings.TrimLeft(tmptext, bv[0]+":") == text {
+				var tagtitle string
+				for i, v := range titleindexs {
+					if i == 0 {
+						continue
+					}
+					if v[0] > iv {
+						tagtitle = con[titleindexs[i-1][0]:titleindexs[i-1][1]]
+						break
+					}
+				}
+				tagtitle = regReplAllSpace2.ReplaceAllString(tagtitle, "")
+				text = tagtitle + ":" + text
+			}
 			headKey := ""
 			if indexKeyStringMap[iv] != "" {
 				//if !filterPkgTitleKey.MatchString(indexKeyStringMap[iv]) {
@@ -835,7 +814,7 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 				}
 				MergeKvTags((*blockPackage)[index].ColonKV.KvTags, colonJobKv.KvTags)
 				//合并空格kv
-				spaceJobKv := SspacekvEntity.Entrance(text, "", nil, isSite, codeSite)
+				spaceJobKv := SspacekvEntity.Entrance(text, headKey, nil, isSite, codeSite)
 				MergeKvTags((*blockPackage)[index].SpaceKV.KvTags, spaceJobKv.KvTags)
 			} else {
 				newBpkg := &util.BlockPackage{

src/jy/pretreated/winnerorder.go

@@ -235,7 +235,7 @@ func (wo *WinnerOrderEntity) findByReg(content string, blocks []string, reg_2 *r
 				val := wo.clear("中标单位", v)
 				if val != nil {
 					count++
-					object["entname"] = val
+					object["entname"] = strings.TrimSpace(qutil.ObjToString(val))
 					object["sort"] = wo.toNumber(k, count)
 					object["sortstr"] = thisNumberReg.FindString(k)
 					object["type"] = i

src/main_test.go

@@ -7,6 +7,7 @@ import (
 	. "jy/mongodbutil"
 	"log"
 	"os"
+	"qfw/util"
 	"regexp"
 	"strconv"
 	"strings"
@@ -111,3 +112,24 @@ func Test_clear(t *testing.T) {
 	}
 	log.Println("result---", value)
 }
+
+func Test_buyer(t *testing.T) {
+	Mgo = MgoFactory(1, 3, 120, "192.168.3.207:27092", "extract_kf")
+	demo, _ := Mgo.Find("demo_data", nil, `{"_id:1"}`, `{"buyer":1,"title":1}`, false, -1, -1)
+	result, _ := Mgo.Find("mxs_buyer", nil, `{"_id:1"}`, `{"buyer":1}`, false, -1, -1)
+	for _, d := range *demo {
+		id1 := util.BsonIdToSId(d["_id"])
+		buyer1 := util.ObjToString(d["buyer"])
+		title := util.ObjToString(d["title"])
+		for _, r := range *result {
+			id2 := util.BsonIdToSId(r["_id"])
+			buyer2 := util.ObjToString(r["buyer"])
+			if id1 == id2 {
+				if buyer1 != buyer2 {
+					util.Debug(id1, buyer1, buyer2)
+				}
+				break
+			}
+		}
+	}
+}

src/res/ext_v3_dump.sh

@@ -2,8 +2,7 @@
 dbhost="127.0.0.1:27082"
 dbname="extract_v3"
 datapath="/opt/soft/mongodb/mongodb3.4/bin"
-tables=(audit areacode citys classify cleanup memu menusecond fields infoclass infotype province postcode rc_calss rc_field rc.order rc_rule rule_back rule_code rule_logic rule_logicback rule_logicore rule_logicpre rule_pre tag tagdetailinfo version versioninfo block_info block_classify block_classify_info block_classify_tag)
-
+tables=(address areacode audit block_classify block_classify_info block_classify_tag block_info citys classify cleanup fields infoclass infotype memu menusecond pkg_info pkg_logicore postcode province postcode rc_calss rc_field rc_rule rule_back rule_code rule_logic rule_logicback rule_logicore rule_logicpre rule_pre site site_management site_rule_code site_rule_logic site_rule_logicback site_rule_logickv site_rule_logicore site_versioninfo tag tagdetailinfo user version versioninfo)
 
 for i in "${!tables[@]}"; 
 do

src/res/fieldscore.json

@@ -18,6 +18,14 @@
                 "regexp": 1,
                 "kvweight": 1
             },
+            "buyer":{
+            	"title": 0,
+				"table": 5,
+                "colon": 5,
+                "space": 5,
+                "regexp": 3,
+                "kvweight": 3
+            },
             "winner": {
                 "table": 3,
                 "colon": 3,
@@ -147,15 +155,15 @@
         "positivewords": [
             {
                 "describe": "以*结尾",
-                "regstr": ".{2,100}(委员会|中心|分校|办公室|学校|幼儿园|动物园|管理站|图书馆|殡仪馆|博物馆|基地|青年宫|少年宫|艺术宫|电视台|协会|政府|初中|集团|银行|[大中小]学|院|厂|店|段|场|社|室|部|厅|局|处|所|队|公司|监狱|监测站|血站|检查站)$",
-                "score": 3
+                "regstr": ".{2,100}(委员会|管委会|医院|卫计委|机关|社区|中心|中心校|分校|办公室|学校|幼儿园|动物园|管理站|馆|基地|青年宫|少年宫|艺术宫|电视台|协会|政府|[初高]中|集团|银行|[大中小]学|院|厂|店|段|场|社|室|部|厅|局|处|所|队|公司|监狱|监测站|血站|检查站|工作站)$",
+                "score": 10
             }
         ],
         "negativewords": [
             {
                 "describe": "包含负分",
                 "regstr": "(标人|附件|委托|认证|代理|咨询|顾问|管理有限公司|管理顾问|招标失败|交易中心|不足|公告|变更|招标|废标|废止|流标|中标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
-                "score": -5
+                "score": -20
             },
             {
                 "describe": "包含负分不再展示",
@@ -174,7 +182,7 @@
                 "range": [
                     0,
                     3,
-                    -5
+                    -20
                 ]
             },
             {
@@ -190,7 +198,7 @@
                 "range": [
                     25,
                     -1,
-                    -1
+                    -5
                 ]
             }
         ]

src/res/tablev1.json

@@ -1,6 +1,6 @@
 {
 	"normalhead":[
-		"^((.{2,6}(名称|编号|代码|时间|类型|性质|行政区域|原因|项目|意见|须知|程度))|标段(编号)?|招标金额|规模|统一社会信用代码|拟?中标供应商|质量|(质量)?承诺|地址|招标代理|序号|材料|结构|结构层数|评委|单位|数量|排名|标的|标项|开户银行|邮编|账号|电话|传真|网址|得分|名次|包件?号|职务|(建设|招标|采购|中标|成交|甲|乙)(单位|人|供应商|方|规模).{0,2}|.{0,5}(价格?|额|资金|[预概]算|投资|费用|报价|投标价)(万?元?([大小]写)?))$__M",
+		"^((.{2,6}(名称|编号|代码|时间|类型|性质|行政区域|原因|意见|须知|程度))|标段(编号)?|招标金额|规模|统一社会信用代码|拟?中标供应商|质量|(质量)?承诺|地址|招标代理|序号|材料|结构|结构层数|评委|单位|数量|排名|标的|标项|开户银行|邮编|账号|电话|传真|网址|得分|名次|包件?号|职务|(建设|招标|采购|中标|成交|甲|乙)(单位|人|供应商|方|规模).{0,2}|.{0,5}(价格?|额|资金|[预概]算|投资|费用|报价|投标价)(万?元?([大小]写)?))$__M",
 		"^.{0,7}(((单位)?名称|总监|经理|负责人|信息|率|费|期|人|号|码|(价格?|额|资金)(万?元?([大小]写)?)|员|品目|标包|代表|区域|方式|因素|合价|合计|小计|地点|条件|(资质|类别和)等级|类别|状态)|得分|注册专业|方法|家数|全称|简称|邮件|执业或职业资格|证书|部门|事项|来源|划分|长度|规模|保证金|目标)$__",
 		"(名单|证号|名称|要求|时间|日期|地点|单位|条款|机构|范围|情况|概况|品名|规格|参数|标准|指标|型号|限价|数量|方式|等级|依据|明细|概况|内容|次数|产品|性质|地区|地址|币种|主题|详情|说明|代理(公司|机构)|节支率|名单|结果|结果公示)$|^(职称|姓名|级别|职称专业|证书名称|证书编号)$__",
 		"^(联系|评标|单位|公告|采购|商品|附件|质保|用途|公示|机构|评审|品名|规格|参数|指标|型号|数量|证书).{0,10}$__",

src/web/templates/admin/audit_auditone.html

@@ -154,8 +154,8 @@ $(function () {
 	});
 	ttable.on('init.dt', function () {
 		var opt="<option value='-1'>全部</option>"+
-				"<option value='ok'>正确</option>"+
-				"<option value='err'>异常</option>";
+				"<option value='1'>正确</option>"+
+				"<option value='0'>异常</option>";
 		var select="<div class='form-group'><label for='name'>数据类型：</label>"+
 			"<select id='auditattr' onchange='checkclick(this.value)' class='form-control input-sm'>"+
 			opt+

standardata/src/config.json

@@ -0,0 +1,30 @@
+{
+  "mgofrom": "172.17.4.187:27083",
+  "mgofromsize":5,
+  "mgofromdb":"qfw",
+  "mgoto": "172.17.145.163:27082",
+  "mgotosize":5,
+  "mgotodb":"extract_v3",
+  "mgoent": "172.17.145.163:27082",
+  "mgoentsize":5,
+  "mgoentdb":"enterprise",
+  "extractcoll":"result_20200116",
+  "standardata":{
+	"winner":{
+		"standarent":"winner_enterprisenew",
+		"standarerr":"winner_errnew",
+		"redisdb":1
+	},
+    "buyer":{
+      "standarent":"buyer_agency_enterprise",
+      "standarerr":"buyer_err",
+      "redisdb":2
+    },
+    "agency":{
+      "standarent":"agency_enterprise",
+      "standarerr":"agency_err",
+      "redisdb":3
+    }
+  },
+  "redis": "winner=172.17.148.44:2679,buyer=172.17.148.44:2679,agency=172.17.148.44:2679"
+}

standardata/src/historyrepair.go

@@ -0,0 +1,67 @@
+// historyrepair 处理多线程重复数据问题
+package main
+
+import (
+	"dbutil/mongo"
+	"dbutil/redis"
+	"log"
+	qu "qfw/util"
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+func historyrepair(db, coll, datatype string, dbnum int) {
+	sess := MongoTo.GetMgoConn()
+	defer MongoTo.Close()
+	field := ""
+	if datatype == "winner" {
+		field = "company_name"
+	} else if datatype == "buyer" {
+		field = "buyer_name"
+	} else if datatype == "agency" {
+		field = "agency_name"
+	}
+	it := sess.DB(db).C(coll).Find(bson.M{}).Select(bson.M{field: 1}).Iter()
+	index := 0
+	delnum := 0
+	for tmp := map[string]interface{}{}; it.Next(&tmp); index++ {
+		name := qu.ObjToString(tmp[field])
+		id := mongo.BsonTOStringId(tmp["_id"])
+		str, _ := redis.GetRedisStr(datatype, dbnum, name)
+		if str != "" {
+			MongoTo.DeleteById(coll, id)
+			delnum++
+		} else {
+			redis.PutRedis(datatype, dbnum, name, id, -1)
+		}
+		tmp = map[string]interface{}{}
+		if index%100 == 0 {
+			log.Println(index, delnum)
+		}
+	}
+	log.Println(index, delnum)
+}
+
+func historyrepairErr(db, coll, datatype string, dbnum int) {
+	sess := MongoTo.GetMgoConn()
+	defer MongoTo.Close()
+	it := sess.DB(db).C(coll).Find(bson.M{}).Select(bson.M{"name": 1}).Iter()
+	index := 0
+	delnum := 0
+	for tmp := map[string]interface{}{}; it.Next(&tmp); index++ {
+		name := qu.ObjToString(tmp["name"])
+		id := mongo.BsonTOStringId(tmp["_id"])
+		str, _ := redis.GetRedisStr(datatype, dbnum, name)
+		if str != "" {
+			MongoTo.DeleteById(coll, id)
+			delnum++
+		} else {
+			redis.PutRedis(datatype, dbnum, name, id, -1)
+		}
+		tmp = map[string]interface{}{}
+		if index%100 == 0 {
+			log.Println(index, delnum)
+		}
+	}
+	log.Println(index, delnum)
+}

standardata/src/main.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"dbutil/mongo"
+	"dbutil/redis"
+	"fmt"
+	"log"
+	mu "mfw/util"
+	"os"
+	qu "qfw/util"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	MongoFrom /*抽取原*/, MongoTo /*保存库*/, MongoEnt/*企业库*/ *mongo.MongodbSim
+	sysconfig                   map[string]interface{}
+	extractcoll                 string
+	winnerent, winnererr        string
+	buyerent, buyererr          string
+	agencyent, agencyerr        string
+	winnerbd, buyerbd, agencybd int
+	Addrs                       = make(map[string]interface{}, 0) //省市县
+	winchanbool                 = make(chan bool, 3)
+	buyerchanbool               = make(chan bool, 3)
+	agencychanbool              = make(chan bool, 3)
+	gochan                      = make(chan bool, 3)
+	udpclient                   mu.UdpClient
+	//异常表正则匹配处理
+	WinnerRegOk, WinnerRegErr, AgencyRegOk, AgencyRegErr, BuyerRegOk, BuyerRegErr []regexp.Regexp
+)
+
+func init() {
+	qu.ReadConfig(&sysconfig)
+	extractcoll = qu.ObjToString(sysconfig["extractcoll"])
+	if standardata, ok := sysconfig["standardata"].(map[string]interface{}); ok {
+		winner, _ := standardata["winner"].(map[string]interface{})
+		winnerent = qu.ObjToString(winner["standarent"])
+		winnererr = qu.ObjToString(winner["standarerr"])
+		winnerbd = qu.IntAll(winner["redisdb"])
+		buyer, _ := standardata["buyer"].(map[string]interface{})
+		buyerent = qu.ObjToString(buyer["standarent"])
+		buyererr = qu.ObjToString(buyer["standarerr"])
+		buyerbd = qu.IntAll(buyer["redisdb"])
+		agency, _ := standardata["agency"].(map[string]interface{})
+		agencyent = qu.ObjToString(agency["standarent"])
+		agencyerr = qu.ObjToString(agency["standarerr"])
+		agencybd = qu.IntAll(agency["redisdb"])
+	} else {
+		os.Exit(0)
+	}
+
+	MongoFrom = &mongo.MongodbSim{
+		MongodbAddr: qu.ObjToString(sysconfig["mgofrom"]),
+		Size:        qu.IntAll(sysconfig["mgofromsize"]),
+		DbName:      qu.ObjToString(sysconfig["mgofromdb"]),
+	}
+	MongoFrom.InitPool()
+
+	MongoTo = &mongo.MongodbSim{
+		MongodbAddr: qu.ObjToString(sysconfig["mgoto"]),
+		Size:        qu.IntAll(sysconfig["mgotosize"]),
+		DbName:      qu.ObjToString(sysconfig["mgotodb"]),
+	}
+	MongoTo.InitPool()
+
+	MongoEnt = &mongo.MongodbSim{
+		MongodbAddr: qu.ObjToString(sysconfig["mgoent"]),
+		Size:        qu.IntAll(sysconfig["mgoentsize"]),
+		DbName:      qu.ObjToString(sysconfig["mgoentdb"]),
+	}
+	MongoEnt.InitPool()
+	redis.InitRedis(qu.ObjToString(sysconfig["redis"]))
+
+	//加载省市县代码
+	list, _ := MongoTo.Find("address", map[string]interface{}{}, nil, nil)
+	for _, v := range list {
+		code := v["code"]
+		if code != nil && strings.TrimSpace(code.(string)) != "" {
+			Addrs[fmt.Sprint(code)] = v
+		}
+	}
+	log.Println("加载省市县代码加载完成", len(Addrs))
+	initReg() //初始标记规则
+}
+
+func initReg() {
+	list, _ := MongoTo.Find("rc_rule", map[string]interface{}{"delete": false, "isuse": true}, nil, nil)
+	for _, v := range list {
+		s_field, ok := v["s_field"].(string) //字段
+		s_rule, ok2 := v["s_rule"].(string)  //正则
+		s_type, ok3 := v["s_type"].(string)  //ok or err
+		if !ok || !ok2 || !ok3 || s_field == "" || s_rule == "" || s_type == "" {
+			continue
+		}
+		var pattern string
+		if strings.Contains(s_rule, "\\u") {
+			s_rule = strings.Replace(s_rule, "\\", "\\\\", -1)
+			s_rule = strings.Replace(s_rule, "\\\\u", "\\u", -1)
+			pattern, _ = strconv.Unquote(`"` + s_rule + `"`)
+		} else {
+			pattern = s_rule
+		}
+		regtmp := regexp.MustCompile(pattern)
+		if s_field == "winner" {
+			if s_type == "ok" {
+				WinnerRegOk = append(WinnerRegOk, *regtmp)
+			} else if s_type == "err" {
+				WinnerRegErr = append(WinnerRegErr, *regtmp)
+			}
+		} else if s_field == "buyer" {
+			if s_type == "ok" {
+				BuyerRegOk = append(BuyerRegOk, *regtmp)
+			} else if s_type == "err" {
+				BuyerRegErr = append(BuyerRegErr, *regtmp)
+			}
+		} else if s_field == "agency" {
+			if s_type == "ok" {
+				AgencyRegOk = append(AgencyRegOk, *regtmp)
+			} else if s_type == "err" {
+				AgencyRegErr = append(AgencyRegErr, *regtmp)
+			}
+		}
+	}
+	log.Println(len(WinnerRegOk), len(WinnerRegErr), len(BuyerRegOk), len(BuyerRegErr), len(AgencyRegOk), len(AgencyRegErr))
+}
+
+func main() {
+	//go historywinner(qu.ObjToString(sysconfig["mgofromdb"]), extractcoll)
+	//go historybuyer(qu.ObjToString(sysconfig["mgofromdb"]), extractcoll)
+	//go historyagency(qu.ObjToString(sysconfig["mgofromdb"]), extractcoll)
+
+	go task_standarData()
+	c := make(chan int, 1)
+	<-c
+}

standardata/src/standaragency.go

@@ -0,0 +1,238 @@
+// standaragency
+package main
+
+import (
+	"dbutil/mongo"
+	"dbutil/redis"
+	"encoding/json"
+	"log"
+	qu "qfw/util"
+	"time"
+	"unicode/utf8"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/mgo.v2/bson"
+)
+
+//增量处理
+func agencyStandarData(db string, query map[string]interface{}) {
+	defer qu.Catch()
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(extractcoll).Find(query).Select(bson.M{"repeat": 1,"agency": 1, "agencytel": 1, "agencyperson": 1, "topscopeclass": 1,
+		"agencyaddr": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0  { //重复数据跳过
+			continue
+		}
+		agency := qu.ObjToString(tmp["agency"])
+		if utf8.RuneCountInString(agency) < 5 {
+			continue
+		}
+		infoid := mongo.BsonTOStringId(tmp["_id"])
+		topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+		entid, _ := redis.GetRedisStr("agency", agencybd, agency)
+		ps := []map[string]interface{}{}
+		agencyperson := qu.ObjToString(tmp["agencyperson"])
+		agencytel := qu.ObjToString(tmp["agencytel"])
+		if entid == "" {//redis 未存
+			savetoerr := true
+			if agencytel != "" {
+				v := map[string]interface{}{
+					"contact_person": agencyperson,
+					"phone":          agencytel,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				ps = append(ps, v)
+				data := comHisMegerNewData(agency, "agency", ps)
+				if data != nil {
+					_id := MongoTo.Save(agencyent, data)
+					redis.PutRedis("agency", agencybd, agency, _id.(primitive.ObjectID).Hex(), -1)
+					savetoerr = false
+				}
+			}
+			if savetoerr {
+				t := MongoTo.FindOne(agencyerr, map[string]interface{}{"name": agency})
+				if len(t) < 1 {
+					MongoTo.Save(agencyerr, map[string]interface{}{
+						"name":       agency,
+						"check":      comMarkdata(agency, "agency"),
+						"updatetime": time.Now().Unix(),
+					})
+				}
+			}
+		} else {
+			if agencytel != "" {
+				is_exist:=false //电话是否存在
+				for _,v := range ps{
+					if v["phone"]==agencytel{
+						is_exist = true
+						if agencyperson!=""&&v["contact_person"]!=agencyperson {
+							v["contact_person"]=agencyperson
+							v["infoid"] = infoid
+							bs, _ := json.Marshal(ps)//替换数据，更新
+							redis.PutRedis("agency", agencybd, agency, bs, -1)
+						}
+						continue
+					}
+				}
+				if !is_exist {
+					v := map[string]interface{}{
+						"contact_person": agencyperson,
+						"phone":          agencytel,
+						"topscopeclass":  comRepTopscopeclass(topscopeclass),
+						"infoid":         infoid,
+					}
+					MongoTo.UpdateById(agencyent, entid,
+						map[string]interface{}{
+							"$set":  v,
+							"$push": map[string]interface{}{"contact": v},
+						},
+					)
+				}
+			}
+		}
+		tmp = map[string]interface{}{}
+		if index%100 == 0 {
+			log.Println("agency index", index)
+		}
+	}
+	log.Println("agency ok index", index)
+}
+
+//历史数据处理
+func historyagency(db, fromcoll string) {
+	defer qu.Catch()
+	log.Println("history  start")
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(fromcoll).Find(map[string]interface{}{}).Select(bson.M{"repeat": 1,"agency": 1, "agencytel": 1, "agencyperson": 1, "topscopeclass": 1,
+		"agencyaddr": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
+			continue
+		}
+		_id := mongo.BsonTOStringId(tmp["_id"])
+		agencychanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-agencychanbool
+			}()
+			agency := qu.ObjToString(tmp["agency"])
+			topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+			if agency != "" && utf8.RuneCountInString(agency) > 4 {
+				agencyperson := qu.ObjToString(tmp["agencyperson"])
+				agencytel := qu.ObjToString(tmp["agencytel"])
+				b, _ := redis.ExistRedis("agency", agencybd, agency)
+				if b {//redis 存在
+					if  agencytel != "" {
+						strs, _ := redis.GetRedisStr("agency", agencybd, agency)
+						ps := []map[string]interface{}{}
+						err := json.Unmarshal([]byte(strs), &ps)
+						if err == nil {
+							is_exist:=false //电话是否存在
+							for _,v := range ps{
+								if v["phone"]==agencytel{
+									is_exist = true
+									if agencyperson!=""&&v["contact_person"]!=agencyperson {
+										v["contact_person"]=agencyperson
+										v["infoid"] = _id
+										bs, _ := json.Marshal(ps)//替换数据，更新
+										redis.PutRedis("agency", agencybd, agency, bs, -1)
+									}
+									continue
+								}
+							}
+
+							if !is_exist {
+								v := map[string]interface{}{
+									"contact_person": agencyperson,
+									"phone":          agencytel,
+									"topscopeclass":  comRepTopscopeclass(topscopeclass),
+									"infoid":         _id,
+								}
+								ps = append(ps, v)
+								bs, _ := json.Marshal(ps)
+								redis.PutRedis("agency", agencybd, agency, bs, -1)
+							}
+						} else {
+							log.Println("jsonErr", err)
+						}
+					}
+				} else {
+					val := []map[string]interface{}{}
+					if agencytel != "" {
+						tmp := map[string]interface{}{
+							"contact_person": agencyperson,
+							"phone":          agencytel,
+							"topscopeclass":  comRepTopscopeclass(topscopeclass),
+							"infoid":         _id,
+						}
+						val = append(val, tmp)
+					}
+					bs, _ := json.Marshal(val)
+					redis.PutRedis("agency", agencybd, agency, bs, -1)
+					MongoTo.Save(agencyerr, map[string]interface{}{
+						"name":       agency,
+						"updatetime": time.Now().Unix(),
+					})
+				}
+			}
+		}(tmp)
+		tmp = map[string]interface{}{}
+		if index%10000 == 0 {
+			log.Println("index", index, _id)
+		}
+	}
+	log.Println("history  ok  index", index)
+	agencyStandarHistory(qu.ObjToString(sysconfig["mgotodb"]))
+}
+
+//查询agencyerr标准化历史数据
+func agencyStandarHistory(db string) {
+	defer qu.Catch()
+	log.Println("开始标准化数据--agency", db)
+	sessto := MongoTo.GetMgoConn()
+	defer MongoTo.Close()
+	it := sessto.DB(db).C(agencyerr).Find(map[string]interface{}{}).Iter()
+	index := 0
+	entnum := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		err_id := mongo.BsonTOStringId(tmp["_id"])
+		name := qu.ObjToString(tmp["name"])
+		agencychanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-agencychanbool
+			}()
+			strs, err := redis.GetRedisStr("agency", agencybd, name)
+			if err != nil {
+				return
+			}
+			ps := []map[string]interface{}{}
+			err = json.Unmarshal([]byte(strs), &ps)
+			if err == nil {
+				data := comHisMegerNewData(name, "agency", ps)
+				if data != nil {
+					MongoTo.Save(agencyent, data)
+					MongoTo.DeleteById(agencyerr, err_id)
+					entnum++
+				} else { //未查询到企业，打标记并存表
+					num := comMarkdata(name, "agency")
+					tmp["check"] = num
+					MongoTo.UpdateById(agencyerr, err_id, map[string]interface{}{"$set": map[string]interface{}{"check": num}})
+				}
+			} else {
+				log.Println("jsonErr", name, err)
+			}
+		}(tmp)
+		if index%1000 == 0 {
+			log.Println("标准化历史数据--agency", index, err_id, entnum)
+		}
+		tmp = map[string]interface{}{}
+	}
+	log.Println("标准化数据完成--agency", index, entnum)
+}

standardata/src/standarbuyer.go

@@ -0,0 +1,236 @@
+// standarbuyer
+package main
+
+import (
+	"dbutil/mongo"
+	"dbutil/redis"
+	"encoding/json"
+	"log"
+	qu "qfw/util"
+	"time"
+	"unicode/utf8"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/mgo.v2/bson"
+)
+
+//增量处理
+func buyerStandarData(db string, query map[string]interface{}) {
+	defer qu.Catch()
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(extractcoll).Find(query).Select(bson.M{"repeat": 1, "buyer": 1, "buyertel": 1, "buyerperson": 1, "buyerclass": 1, "topscopeclass": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
+			continue
+		}
+		buyer := qu.ObjToString(tmp["buyer"])
+		if utf8.RuneCountInString(buyer) < 5 {
+			continue
+		}
+		infoid := mongo.BsonTOStringId(tmp["_id"])
+		buyerclass := qu.ObjToString(tmp["buyerclass"])
+		topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+		entid, _ := redis.GetRedisStr("buyer", buyerbd, buyer)
+		ps := []map[string]interface{}{}
+		buyerperson := qu.ObjToString(tmp["buyerperson"])
+		buyertel := qu.ObjToString(tmp["buyertel"])
+		if entid == "" {
+			savetoerr := true
+			if buyerperson != "" || buyertel != "" {
+				v := map[string]interface{}{
+					"contact_person": buyerperson,
+					"phone":          buyertel,
+					"buyerclass":     buyerclass,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				ps = append(ps, v)
+				data := comHisMegerNewData(buyer, "buyer", ps)
+				if data != nil {
+					_id := MongoTo.Save(buyerent, data)
+					redis.PutRedis("buyer", buyerbd, buyer, _id.(primitive.ObjectID).Hex(), -1)
+					savetoerr = false
+				}
+			}
+			if savetoerr {
+				t := MongoTo.FindOne(buyererr, map[string]interface{}{"name": buyer})
+				if len(t) < 1 {
+					MongoTo.Save(buyererr, map[string]interface{}{
+						"name":       buyer,
+						"buyerlass":  buyerclass,
+						"check":      comMarkdata(buyer, "buyer"),
+						"updatetime": time.Now().Unix(),
+					})
+				}
+			}
+		} else {
+			if buyerperson != "" || buyertel != "" {
+				v := map[string]interface{}{
+					"contact_person": buyerperson,
+					"phone":          buyertel,
+					"buyerclass":     buyerclass,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				data := buyerMegerBuyerclass(entid, v)
+				MongoTo.UpdateById(buyerent, entid,
+					map[string]interface{}{
+						"$set":  data,
+						"$push": map[string]interface{}{"contact": v},
+					},
+				)
+			}
+		}
+		tmp = map[string]interface{}{}
+		if index%100 == 0 {
+			log.Println("buyer index", index)
+		}
+	}
+	log.Println("buyer ok index", index)
+}
+
+//历史数据处理
+func historybuyer(db, fromcoll string) {
+	defer qu.Catch()
+	log.Println("history  start")
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(fromcoll).Find(map[string]interface{}{}).Select(bson.M{"repeat": 1, "buyer": 1, "buyertel": 1, "buyerperson": 1, "buyerclass": 1, "topscopeclass": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
+			continue
+		}
+		_id := mongo.BsonTOStringId(tmp["_id"])
+		buyerchanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-buyerchanbool
+			}()
+			buyer := qu.ObjToString(tmp["buyer"])
+			buyerclass := qu.ObjToString(tmp["buyerclass"])
+			topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+			if buyer != "" && utf8.RuneCountInString(buyer) > 4 {
+				buyerperson := qu.ObjToString(tmp["buyerperson"])
+				buyertel := qu.ObjToString(tmp["buyertel"])
+				b, _ := redis.ExistRedis("buyer", buyerbd, buyer)
+				if b {
+					if buyerperson != "" || buyertel != "" {
+						strs, _ := redis.GetRedisStr("buyer", buyerbd, buyer)
+						ps := []interface{}{}
+						err := json.Unmarshal([]byte(strs), &ps)
+						if err == nil {
+							v := map[string]interface{}{
+								"contact_person": buyerperson,
+								"phone":          buyertel,
+								"buyerclass":     buyerclass,
+								"topscopeclass":  comRepTopscopeclass(topscopeclass),
+								"infoid":         _id,
+							}
+							ps = append(ps, v)
+							bs, _ := json.Marshal(ps)
+							redis.PutRedis("buyer", buyerbd, buyer, bs, -1)
+						} else {
+							log.Println("jsonErr", err)
+						}
+					}
+				} else {
+					val := []map[string]interface{}{}
+					if buyerperson != "" || buyertel != "" {
+						tmp := map[string]interface{}{
+							"contact_person": buyerperson,
+							"phone":          buyertel,
+							"buyerclass":     buyerclass,
+							"topscopeclass":  comRepTopscopeclass(topscopeclass),
+							"infoid":         _id,
+						}
+						val = append(val, tmp)
+					}
+					bs, _ := json.Marshal(val)
+					redis.PutRedis("buyer", buyerbd, buyer, bs, -1)
+					MongoTo.Save(buyererr, map[string]interface{}{
+						"name":       buyer,
+						"buyerclass": buyerclass,
+						"updatetime": time.Now().Unix(),
+					})
+				}
+			}
+		}(tmp)
+		tmp = map[string]interface{}{}
+		if index%10000 == 0 {
+			log.Println("index", index, _id)
+		}
+	}
+	log.Println("history  ok  index", index)
+	buyerStandarHistory(qu.ObjToString(sysconfig["mgotodb"]))
+}
+
+//查询buyererr标准化历史数据
+func buyerStandarHistory(db string) {
+	defer qu.Catch()
+	log.Println("开始标准化数据--buyer", db)
+	sessto := MongoTo.GetMgoConn()
+	defer MongoTo.Close()
+	it := sessto.DB(db).C(buyererr).Find(map[string]interface{}{}).Iter()
+	index := 0
+	entnum := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		err_id := mongo.BsonTOStringId(tmp["_id"])
+		name := qu.ObjToString(tmp["name"])
+		buyerchanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-buyerchanbool
+			}()
+			strs, err := redis.GetRedisStr("buyer", buyerbd, name)
+			if err != nil {
+				return
+			}
+			ps := []map[string]interface{}{}
+			err = json.Unmarshal([]byte(strs), &ps)
+			if err == nil {
+				data := comHisMegerNewData(name, "buyer", ps)
+				if data != nil {
+					MongoTo.Save(buyerent, data)
+					MongoTo.DeleteById(buyererr, err_id)
+					entnum++
+				} else { //未查询到企业，打标记并存表
+					num := comMarkdata(name, "buyer")
+					tmp["check"] = num
+					MongoTo.UpdateById(buyererr, err_id, map[string]interface{}{"$set": map[string]interface{}{"check": num}})
+				}
+			} else {
+				log.Println("jsonErr", name, err)
+			}
+		}(tmp)
+		if index%1000 == 0 {
+			log.Println("标准化历史数据--buyer", index, err_id, entnum)
+		}
+		tmp = map[string]interface{}{}
+	}
+	log.Println("标准化数据完成--buyer", index, entnum)
+}
+
+//企业数据整合(已有标注信息)
+func buyerMegerBuyerclass(id string, ps map[string]interface{}) map[string]interface{} {
+	tmp := MongoEnt.FindById(buyerent, id, bson.M{"buyerclass": 1})
+	if len(tmp) < 1 {
+		return nil
+	}
+	data := map[string]interface{}{}
+	buyerclass := tmp["buyerclass"].(primitive.A)
+	tmpbuyerclass := map[string]bool{}
+	for _, v := range buyerclass {
+		tt := qu.ObjToString(v)
+		tmpbuyerclass[tt] = true
+	}
+	tmpbuyerclass[qu.ObjToString(ps["buyerclass"])] = true
+	newbuyerclass := []interface{}{}
+	for k, _ := range tmpbuyerclass {
+		newbuyerclass = append(newbuyerclass, k)
+	}
+	data["buyerclass"] = newbuyerclass
+	return data
+}

standardata/src/standarwinner.go

@@ -0,0 +1,468 @@
+// standarwinner
+package main
+
+import (
+	"dbutil/mongo"
+	"dbutil/redis"
+	"encoding/json"
+	"log"
+	qu "qfw/util"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/mgo.v2/bson"
+)
+
+//增量处理
+func winnerStandarData(db string, query map[string]interface{}) {
+	defer qu.Catch()
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(extractcoll).Find(query).Select(bson.M{"repeat": 1, "winner": 1, "winnertel": 1, "winnerperson": 1, "topscopeclass": 1, "package": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
+			continue
+		}
+		winner := qu.ObjToString(tmp["winner"])
+		if utf8.RuneCountInString(winner) < 5 {
+			continue
+		}
+		infoid := mongo.BsonTOStringId(tmp["_id"])
+		topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+		entid, _ := redis.GetRedisStr("winner", winnerbd, winner)
+		winnerperson := qu.ObjToString(tmp["winnerperson"])
+		winnertel := qu.ObjToString(tmp["winnertel"])
+		if entid == "" { //新增标准库
+			savetoerr := true
+			if winnerperson != "" || winnertel != "" {
+				v := map[string]interface{}{
+					"contact_person": winnerperson,
+					"phone":          winnertel,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				data := comHisMegerNewData(winner, "winner", []map[string]interface{}{v})
+				if data != nil {
+					_id := MongoTo.Save(winnerent, data)
+					redis.PutRedis("winner", winnerbd, winner, _id.(primitive.ObjectID).Hex(), -1)
+					savetoerr = false
+				}
+			}
+			if savetoerr {
+				t := MongoTo.FindOne(winnererr, map[string]interface{}{"name": winner})
+				if len(t) < 1 {
+					MongoTo.Save(winnererr, map[string]interface{}{
+						"name":          winner,
+						"topscopeclass": comRepTopscopeclass(topscopeclass),
+						"check":         comMarkdata(winner, "winner"),
+						"updatetime":    time.Now().Unix(),
+					})
+				}
+			}
+		} else { //更新标准库
+			if winnerperson != "" || winnertel != "" {
+				v := map[string]interface{}{
+					"contact_person": winnerperson,
+					"phone":          winnertel,
+					"topscopeclass":  comRepTopscopeclass(topscopeclass),
+					"infoid":         infoid,
+				}
+				data := winMegerIndustry(entid, v)
+				MongoTo.UpdateById(winnerent, entid,
+					map[string]interface{}{
+						"$set":  data,
+						"$push": map[string]interface{}{"contact": v},
+					},
+				)
+			}
+		}
+		//分包处理
+		if packages, ok := tmp["package"].(map[string]interface{}); ok {
+			entpacks := getWinnerPacks(infoid, packages, comRepTopscopeclass(topscopeclass))
+			for name, contact := range entpacks {
+				entid, _ := redis.GetRedisStr("winner", winnerbd, name)
+				if entid == "" {
+					data := comHisMegerNewData(winner, "winner", []map[string]interface{}{contact})
+					if data != nil {
+						_id := MongoTo.Save(winnerent, data)
+						redis.PutRedis("winner", winnerbd, winner, _id.(primitive.ObjectID).Hex(), -1)
+					}
+				} else {
+					data := winMegerIndustry(entid, contact)
+					MongoTo.UpdateById(winnerent, entid,
+						map[string]interface{}{
+							"$set":  data,
+							"$push": map[string]interface{}{"contact": contact},
+						},
+					)
+				}
+			}
+		}
+		tmp = map[string]interface{}{}
+		if index%100 == 0 {
+			log.Println("winner index", index)
+		}
+	}
+	log.Println("winner ok index", index)
+}
+
+//历史数据处理
+func historywinner(db, fromcoll string) {
+	defer qu.Catch()
+	log.Println("history  start")
+	sess := MongoFrom.GetMgoConn()
+	defer MongoFrom.Close()
+	it := sess.DB(db).C(fromcoll).Find(map[string]interface{}{}).Select(bson.M{"repeat": 1, "winner": 1, "winnertel": 1, "winnerperson": 1, "topscopeclass": 1}).Sort("_id").Iter()
+	index := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		if qu.IntAll(tmp["repeat"]) > 0 { //重复数据跳过
+			continue
+		}
+		_id := mongo.BsonTOStringId(tmp["_id"])
+		winchanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-winchanbool
+			}()
+			winner := qu.ObjToString(tmp["winner"])
+			topscopeclass, _ := tmp["topscopeclass"].(primitive.A)
+			if winner != "" && utf8.RuneCountInString(winner) > 4 {
+				winnerperson := qu.ObjToString(tmp["winnerperson"])
+				winnertel := qu.ObjToString(tmp["winnertel"])
+				b, _ := redis.ExistRedis("winner", winnerbd, winner)
+				if b {
+					if winnerperson != "" || winnertel != "" {
+						strs, _ := redis.GetRedisStr("winner", winnerbd, winner)
+						ps := []interface{}{}
+						err := json.Unmarshal([]byte(strs), &ps)
+						if err == nil {
+							v := map[string]interface{}{
+								"contact_person": winnerperson,
+								"phone":          winnertel,
+								"topscopeclass":  comRepTopscopeclass(topscopeclass),
+								"infoid":         _id,
+							}
+							ps = append(ps, v)
+							bs, _ := json.Marshal(ps)
+							redis.PutRedis("winner", winnerbd, winner, bs, -1)
+							//log.Println(_id, index, winner)
+						} else {
+							log.Println("jsonErr", err)
+						}
+					}
+				} else {
+					val := []map[string]interface{}{}
+					if winnerperson != "" || winnertel != "" {
+						tmp := map[string]interface{}{
+							"contact_person": winnerperson,
+							"phone":          winnertel,
+							"topscopeclass":  comRepTopscopeclass(topscopeclass),
+							"infoid":         _id,
+						}
+						val = append(val, tmp)
+					}
+					bs, _ := json.Marshal(val)
+					redis.PutRedis("winner", winnerbd, winner, bs, -1)
+					MongoTo.Save(winnererr, map[string]interface{}{
+						"name":          winner,
+						"topscopeclass": comRepTopscopeclass(topscopeclass),
+						"updatetime":    time.Now().Unix(),
+					})
+				}
+			}
+		}(tmp)
+		tmp = map[string]interface{}{}
+		if index%10000 == 0 {
+			log.Println("index", index, _id)
+		}
+	}
+	log.Println("history  ok  index", index)
+	winStandarHistory(qu.ObjToString(sysconfig["mgotodb"]))
+}
+
+//查询winnererr标准化历史数据
+func winStandarHistory(db string) {
+	defer qu.Catch()
+	log.Println("开始标准化数据--winner", db)
+	sessto := MongoTo.GetMgoConn()
+	defer MongoTo.Close()
+	it := sessto.DB(db).C(winnererr).Find(map[string]interface{}{}).Iter()
+	index := 0
+	entnum := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); index++ {
+		err_id := mongo.BsonTOStringId(tmp["_id"])
+		name := qu.ObjToString(tmp["name"])
+		winchanbool <- true
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-winchanbool
+			}()
+			strs, err := redis.GetRedisStr("winner", winnerbd, name)
+			if err != nil {
+				return
+			}
+			ps := []map[string]interface{}{}
+			err = json.Unmarshal([]byte(strs), &ps)
+			if err == nil {
+				data := comHisMegerNewData(name, "winner", ps)
+				if data != nil {
+					MongoTo.Save(winnerent, data)
+					MongoTo.DeleteById(winnererr, err_id)
+					entnum++
+				} else { //未查询到企业，打标记并存表
+					num := comMarkdata(name, "winner")
+					tmp["check"] = num
+					MongoTo.UpdateById(winnererr, err_id, map[string]interface{}{"$set": map[string]interface{}{"check": num}})
+				}
+			} else {
+				log.Println("jsonErr", name, err)
+			}
+		}(tmp)
+		if index%1000 == 0 {
+			log.Println("标准化历史数据--winner", index, err_id, entnum)
+		}
+		tmp = map[string]interface{}{}
+	}
+	log.Println("标准化数据完成--winner", index, entnum)
+}
+
+//企业数据整合(已有标注信息)
+func winMegerIndustry(id string, ps map[string]interface{}) map[string]interface{} {
+	tmp := MongoEnt.FindById(winnerent, id, bson.M{"industry": 1})
+	if len(tmp) < 1 {
+		return nil
+	}
+	data := map[string]interface{}{}
+	industry := tmp["industry"].(primitive.A)
+	tmpindustry := map[string]bool{}
+	for _, v := range industry {
+		tt := qu.ObjToString(v)
+		tmpindustry[tt] = true
+	}
+	if topscopeclass, ok := ps["topscopeclass"].([]interface{}); ok {
+		for _, v := range topscopeclass {
+			tt := qu.ObjToString(v)
+			tmpindustry[tt] = true
+		}
+	}
+	newindustry := []interface{}{}
+	for k, _ := range tmpindustry {
+		newindustry = append(newindustry, k)
+	}
+	data["industry"] = newindustry
+	return data
+}
+
+//中标单位分包提取联系方式
+func getWinnerPacks(infoid string, packs map[string]interface{}, topscopeclass []interface{}) map[string]map[string]interface{} {
+	entmappacks := map[string]map[string]interface{}{}
+	for _, v := range packs {
+		if tmp, ok := v.(map[string]interface{}); ok {
+			winner := qu.ObjToString(tmp["winner"])
+			if utf8.RuneCountInString(winner) < 5 {
+				continue
+			}
+			winnerperson := qu.ObjToString(tmp["winnerperson"])
+			winnertel := qu.ObjToString(tmp["winnertel"])
+			if winnerperson != "" || winnertel != "" {
+				p := map[string]interface{}{
+					"contact_person": winnerperson,
+					"phone":          winnertel,
+					"topscopeclass":  topscopeclass,
+					"infoid":         infoid,
+					"extfrom":        "package",
+				}
+				entmappacks[winner] = p
+			}
+		}
+	}
+	return entmappacks
+}
+
+//数据整合
+func comHisMegerNewData(name, datatype string, ps []map[string]interface{}) map[string]interface{} {
+	tmp := MongoEnt.FindOne("qyxy", map[string]interface{}{"company_name": name})
+	if len(tmp) < 1 {
+		return nil
+	}
+	data := map[string]interface{}{
+		"history_name":    "",
+		"credit_no":       "",
+		"area_code":       qu.ObjToString(tmp["area_code"]),
+		"province":        qu.ObjToString(tmp["province"]),
+		"city":            "",
+		"district":        "",
+		"company_type":    qu.ObjToString(tmp["company_type"]),
+		"legal_person":    qu.ObjToString(tmp["legal_person"]),
+		"company_address": qu.ObjToString(tmp["company_address"]),
+		"business_scope":  qu.ObjToString(tmp["business_scope"]),
+		"wechat_accounts": []interface{}{},
+		"website":         "",
+		"contact":         ps,
+		"comeintime":      time.Now().Unix(),
+		"updatetime":      time.Now().Unix(),
+	}
+	//统一信用代码
+	credit_no := strings.TrimSpace(qu.ObjToString(tmp["credit_no"]))
+	if credit_no != "" {
+		data["credit_no"] = credit_no
+		if len(credit_no) > 8 {
+			dataNo := credit_no[2:8]
+			if Addrs[dataNo] != nil {
+				if v, ok := Addrs[dataNo].(map[string]interface{}); ok {
+					if data["province"] == "" {
+						data["province"] = v["province"]
+					}
+					data["city"] = v["city"]
+					data["district"] = v["district"]
+				}
+			}
+		}
+	}
+
+	//网址
+	annual_reports := tmp["annual_reports"]
+	if annual_reports != nil {
+		report_websitesArr := []string{}
+		if anreports, ok := annual_reports.([]interface{}); ok {
+			for _, report_websites := range anreports {
+				if websites, ok := report_websites.([]interface{}); ok {
+					for _, website := range websites {
+						if rv, ok := website.(map[string]interface{}); ok {
+							web := qu.ObjToString(rv["website_url"])
+							if web != "" {
+								report_websitesArr = append(report_websitesArr, web)
+							}
+						}
+					}
+				}
+			}
+		}
+		if len(report_websitesArr) > 0 {
+			data["website"] = strings.Join(report_websitesArr, ";")
+		}
+	}
+
+	if datatype == "winner" {
+		data["company_name"] = name
+		data["partners"] = tmp["partners"]
+		establish_date := tmp["establish_date"]
+		if establish_date != nil {
+			data["establish_date"] = qu.Int64All(establish_date) / 1000
+		}
+		capital := tmp["capital"]
+		if capital != nil {
+			data["capital"] = ObjToMoney([]interface{}{capital, ""})[0]
+		}
+
+		industry := make([]string, 0)
+		tmpindustry := map[string]bool{}
+		for _, p := range ps {
+			if ts, ok := (p["topscopeclass"]).([]interface{}); ok {
+				for _, v := range ts {
+					tt := qu.ObjToString(v)
+					tmpindustry[tt] = true
+				}
+			}
+		}
+		for k, _ := range tmpindustry {
+			industry = append(industry, k)
+		}
+		data["industry"] = industry
+	} else if datatype == "buyer" {
+		data["buyer_name"] = name
+		tmpbuyerclass := map[string]bool{}
+		for _, p := range ps {
+			tmpbuyerclass[qu.ObjToString(p["buyerclass"])] = true
+		}
+		buyerclass := []interface{}{}
+		for k, _ := range tmpbuyerclass {
+			buyerclass = append(buyerclass, k)
+		}
+		data["buyerclass"] = buyerclass
+		data["ranks"] = ""
+		data["type"] = ""
+		data["address"] = ""
+	} else {
+		data["agency_name"] = name
+		data["ranks"] = ""
+		data["type"] = ""
+		data["address"] = ""
+	}
+	return data
+}
+
+//根据规则数据打标记
+func comMarkdata(name, datatype string) int {
+	tag := 0 //默认错误
+	switch datatype {
+	case "winner":
+		for _, v := range WinnerRegOk {
+			isok := v.MatchString(name)
+			if isok { //匹配ok完,匹配err
+				errflag := true
+				for _, vRegErr := range WinnerRegErr {
+					err := vRegErr.MatchString(name)
+					if err {
+						errflag = true
+						break
+					}
+				}
+				if errflag {
+					tag = 1
+				}
+			}
+		}
+	case "buyer":
+		for _, v := range BuyerRegOk {
+			isok := v.MatchString(name)
+			if isok { //匹配ok完,匹配err
+				errflag := true
+				for _, vRegErr := range BuyerRegErr {
+					err := vRegErr.MatchString(name)
+					if err {
+						errflag = true
+						break
+					}
+				}
+				if errflag {
+					tag = 1
+				}
+			}
+		}
+	case "agency":
+		for _, v := range AgencyRegOk {
+			isok := v.MatchString(name)
+			if isok { //匹配ok完,匹配err
+				errflag := true
+				for _, vRegErr := range AgencyRegErr {
+					err := vRegErr.MatchString(name)
+					if err {
+						errflag = true
+						break
+					}
+				}
+				if errflag {
+					tag = 1
+				}
+			}
+		}
+	default:
+	}
+	return tag
+}
+
+//过滤行业冗余字符
+func comRepTopscopeclass(tops []interface{}) []interface{} {
+	data := []interface{}{}
+	for _, v := range tops {
+		tt := qu.ObjToString(v)
+		if len(tt) > 1 {
+			data = append(data, tt[:len(tt)-1])
+		}
+	}
+	return data
+}

standardata/src/task.go

@@ -0,0 +1,32 @@
+// task定时执行项目索引
+package main
+
+import (
+	"log"
+	qu "qfw/util"
+	"time"
+
+	"github.com/cron"
+)
+
+func task_standarData() {
+	mgofromdb := qu.ObjToString(sysconfig["mgofromdb"])
+	c := cron.New()
+	_ = c.AddFunc("0 30 4 * * *", func() {
+		t := time.Now()
+		pici := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local).Unix()
+		query := map[string]interface{}{
+			"comeintime": map[string]interface{}{
+				"$gt":  pici - 86400,
+				"$lte": pici,
+			},
+		}
+		log.Println(mgofromdb, query)
+		go winnerStandarData(mgofromdb, query)
+		time.Sleep(1 * time.Minute)
+		go buyerStandarData(mgofromdb, query)
+		time.Sleep(1 * time.Minute)
+		go agencyStandarData(mgofromdb, query)
+	})
+	c.Start()
+}

standardata/src/util.go

@@ -0,0 +1,247 @@
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	regOperator, _ = regexp.Compile(`[*|+|)*)]`)
+	regNumFloat, _ = regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
+	regStrUnit, _  = regexp.Compile(`[元|万|亿]`)
+
+	regStrChar      = `[〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
+	moneyRegChar, _ = regexp.Compile(regStrChar)
+	contentUnit, _  = regexp.Compile(`(万元|单位/万)`)
+	numCapitals, _  = regexp.Compile(`([〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]{4,40})`)
+
+	regQianw, _ = regexp.Compile(`\d{1,2}千万`)
+	moneyChar   = map[string]interface{}{ //"〇": "0", "零": "0",
+		"一": float64(1), "壹": float64(1), "二": float64(2), "贰": float64(2), "三": float64(3), "叁": float64(3), "四": float64(4), "肆": float64(4), "五": float64(5), "伍": float64(5),
+		"六": float64(6), "陆": float64(6), "七": float64(7), "柒": float64(7), "八": float64(8), "捌": float64(8), "九": float64(9), "玖": float64(9), "十": float64(10), "拾": float64(10),
+		"百": float64(100), "佰": float64(100), "千": float64(1000), "仟": float64(1000), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000),
+		"零": float64(0), "点": ".", "角": float64(0.1), "分": float64(0.01),
+	}
+	moneyUnit = map[string]float64{
+		"元": float64(1), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000), //单位
+	}
+	cutAllSpace, _ = regexp.Compile(`\s*`)
+	spaces         = []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n"}
+)
+//大写数子金额转换
+func capitalMoney(data []interface{}) []interface{} {
+	nodes := []float64{}
+	node := float64(0)
+	tmp := float64(0)
+	decimals := 0.0
+	ishaspoint := false //是否含小数点
+	fnum := float64(0)
+	end := false
+	str := fmt.Sprint(data[0])
+	//提取第一个大写信息
+	strmatch := numCapitals.FindAllStringSubmatch(str, -1)
+	if len(strmatch) > 0 {
+		str = strmatch[0][0]
+	}
+	suffixUnit := float64(1)
+	if strings.HasSuffix(str, "万") || strings.HasSuffix(str, "万元") || strings.HasSuffix(str, "万元整") {
+		index := strings.LastIndex(str, "万")
+		str = str[0:index]
+		suffixUnit = float64(10000)
+	}
+	moneyRegChar.ReplaceAllStringFunc(str, func(key string) string {
+		if key == "元" || key == "圆" || key == "点" {
+			ishaspoint = true
+		}
+		if v, ok := moneyChar[key].(float64); ok && !end {
+			if ishaspoint && v > 10 { //排除后面有其他的单位
+				return ""
+			}
+			//fmt.Println(key, v, fnum)
+			if v < 10 && v >= 0 {
+				if ishaspoint { //小数部分
+					if v >= 1 {
+						fnum = v
+					} else if v < 1 && v > 0 {
+						decimals += fnum * v
+					}
+				} else {
+					if tmp != float64(0) {
+						node += tmp
+					}
+					tmp = float64(v)
+				}
+			} else if v == 10000 || v == 100000000 { //单位万、亿
+				if tmp != float64(0) {
+					node += tmp
+					tmp = float64(0)
+				}
+				nodes = append(nodes, node*float64(v))
+				node = float64(0)
+			} else {
+				if v == 10 && tmp == 0 {
+					tmp = 1
+				}
+				tmp = tmp * float64(v)
+				node += tmp
+				tmp = float64(0)
+			}
+		}
+		if key == "整" || key == "正" || key == "分" {
+			end = true
+		}
+		return ""
+	})
+	nodes = append(nodes, node, tmp)
+	ret := float64(0)
+	for _, v := range nodes {
+		ret += v
+	}
+	return []interface{}{(ret + decimals) * suffixUnit, data[1]}
+}
+
+//金额转换
+func ObjToMoney(data []interface{}) []interface{} {
+	isfindUnit := true
+	ret := capitalMoney(data)[0]
+	if ret.(float64) < float64(10000) || ret.(float64) > float64(50000000000) {
+		ret2, b := numMoney(data)
+		isfindUnit = b
+		if ret2[0].(float64) > ret.(float64) {
+			ret = ret2[0]
+		}
+	}
+	f, _ := strconv.ParseFloat(strconv.FormatFloat(ret.(float64), 'f', 4, 64), 64)
+	if f < 1 {
+		f = 0
+	}
+	//若果金额小于50，全文检索单位：万
+	if f < 50 && f > 0 && isfindUnit {
+		rep := contentUnit.FindAllStringIndex(fmt.Sprint(data[1]), -1)
+		if len(rep) > 0 {
+			f = f * 10000
+		}
+	}
+	data[0] = f
+	return data
+}
+
+//符号替换
+func replaceString(con string, ret, rep []string) string {
+	for k, v := range ret {
+		if len(rep) > k {
+			con = strings.Replace(con, v, rep[k], -1)
+		}
+	}
+	return con
+}
+
+//清理所有空白符
+func CutAllSpace(data []interface{}) []interface{} {
+	tmp := cutAllSpace.ReplaceAllString(fmt.Sprint(data[0]), "")
+	tmp = replaceSymbol(tmp, spaces)
+	data[0] = tmp
+	return data
+}
+
+//数字金额转换
+func numMoney(data []interface{}) ([]interface{}, bool) {
+	tmp := fmt.Sprintf("%f", data[0])
+	repUnit := float64(1)
+	if regQianw.MatchString(tmp) {
+		tmp = strings.Replace(tmp, "千万", "万", -1)
+		repUnit = float64(1000)
+	}
+	tmp = replaceSymbol(tmp, []string{",", "，", "（", "）", "(", ")", "：", "\n"})
+	tmp = replaceString(tmp, []string{"万元", "亿元", "．"}, []string{"万", "亿", "."})
+	tmp = fmt.Sprint(CutAllSpace([]interface{}{tmp, data[1]})[0])
+	rets := regNumFloat.FindAllString(tmp, -1)
+	fnums := []float64{}
+	unitstrs := []string{}
+	if len(rets) > 0 {
+		pindex := 0 //单位前置
+		for k, v := range rets {
+			f, err := strconv.ParseFloat(v, 64)
+			if err == nil {
+				fnums = append(fnums, f)
+				index := strings.Index(tmp, v)
+				//单位后置
+				start := index + len(v)
+				end := start + 3
+				//log.Println("vvv", tmp, v, pindex, index, start)
+				if k > 0 {
+					if start >= pindex+3 {
+						pstart := pindex + 3
+						if pstart >= index {
+							pstart = index
+						}
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:])
+						}
+					} else {
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				} else {
+					if len(tmp) > end {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:end])
+						}
+					} else {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				}
+				pindex = start
+			}
+		}
+	}
+	//log.Println("unitstrs", fnums, unitstrs)
+	unit := float64(0)
+	fnum := float64(0)
+	for k, v := range fnums {
+		fnum = v
+		units := regStrUnit.FindAllString(unitstrs[k], -1)
+		for _, v := range units {
+			if moneyUnit[v] != 0 {
+				unit = moneyUnit[v]
+				break
+			}
+		}
+		if unit != float64(0) { //取第一个
+			break
+		}
+	}
+	fnum = fnum * repUnit
+	if unit == float64(0) {
+		data[0] = fnum
+	} else {
+		data[0] = fnum * unit
+	}
+	if unit == 10000 {
+		return data, false
+	} else {
+		return data, true
+	}
+}
+
+//过滤符号
+func replaceSymbol(con string, rep []string) string {
+	for _, v := range rep {
+		con = strings.Replace(con, v, "", -1)
+	}
+	return con
+}
+

util/src/dbutil/mongo/mgo.go

@@ -237,6 +237,24 @@ func (m *MongodbSim) UpdateById(c, id string, doc map[string]interface{}) bool {
 	return true
 }
 
+//更新Update
+func (m *MongodbSim) Update(c string, query map[string]interface{}, up map[string]interface{}) (map[int64]interface{}, bool) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	write := mongo.NewUpdateOneModel()
+	write.SetFilter(query)
+	write.SetUpdate(up)
+	write.SetUpsert(true)
+	writes = append(writes, write)
+	r, err := coll.BulkWrite(m.Ctx, writes)
+	if err != nil {
+		return nil, false
+	}
+	return r.UpsertedIDs, true
+}
+
 //删除by id
 func (m *MongodbSim) DeleteById(c, id string) int64 {
 	m.Open()
@@ -262,11 +280,12 @@ func (m *MongodbSim) Delete(c string, query map[string]interface{}) int64 {
 }
 
 //findbyid
-func (m *MongodbSim) FindById(c, id string) map[string]interface{} {
+func (m *MongodbSim) FindById(c, id string, fields interface{}) map[string]interface{} {
 	m.Open()
 	defer m.Close()
 	coll := m.C.Database(m.DbName).Collection(c)
-	r := coll.FindOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
+	op := options.FindOne()
+	r := coll.FindOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)}, op.SetProjection(fields))
 	v := map[string]interface{}{}
 	r.Decode(&v)
 	return v
@@ -277,6 +296,8 @@ func (m *MongodbSim) FindOne(c string, query map[string]interface{}) map[string]
 	m.Open()
 	defer m.Close()
 	coll := m.C.Database(m.DbName).Collection(c)
+	//op := options.FindOne()
+	//r := coll.FindOne(m.Ctx, query, op.SetProjection(fields))
 	r := coll.FindOne(m.Ctx, query)
 	v := map[string]interface{}{}
 	r.Decode(&v)