Merge branch 'dev3.2' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.2

src/jy/extract/extract.go

@@ -404,6 +404,10 @@ func (e *ExtractTask) ExtractDetail(j *ju.Job) {
 				// log.Debug("抽取-后置规则", tmp)
 			}
 		}
+
+		//for _, vvc := range j.Result["budget"] {
+		//log.Debug("-----", fmt.Sprintf("%+v", vvc))
+		//}
 		//全局后置规则
 		for _, v := range e.RuleBacks {
 			ExtRegBack(j, v, e.TaskInfo)
@@ -678,176 +682,36 @@ func ExtRegCore(extfrom string, doc map[string]interface{}, j *ju.Job, in *RegLu
 
 //lua脚本根据属性设置提取kv值
 func getKvByLuaFields(extfrom string, j *ju.Job, in *RegLuaInfo, t map[string][]*Tag) map[string][]map[string]interface{} {
-	defer qu.Catch()
 	kvmap := map[string][]map[string]interface{}{}
 	for fieldname, field := range in.LFields {
-		lock.Lock()
-		tags := t[field] //获取对应标签库
-		lock.Unlock()
-		if tags == nil {
-			continue
-		}
 		for _, bl := range j.Block {
-			//冒号kv
-			if bl.ColonKV != nil {
-				kvs := bl.ColonKV.Kvs
-				kvs2 := bl.ColonKV.Kvs_2
-				// log.Debug("ColonKV1", kvs)
-				// log.Debug("ColonKV2", kvs2)
-				for _, tag := range tags {
-					for _, kv := range kvs {
-						if tag.Type == "string" {
-							if kv.Key == tag.Key {
-								text := ju.TrimLRSpace(kv.Value, "")
-								if text != "" {
-									kvmap[field] = append(kvmap[field], map[string]interface{}{
-										"field":       field,
-										"code":        in.Code,
-										"ruletext":    tag.Key,
-										"extfrom":     extfrom,
-										"sourcevalue": text,
-										"value":       text,
-										"type":        "colon1",
-										"matchtype":   "tag_string",
-										"blocktag":    bl.Classify,
-									})
-								}
-								break
-							}
-						} else if tag.Type == "regexp" {
-							if tag.Reg.MatchString(kv.Key) {
-								text := ju.TrimLRSpace(kv.Value, "")
-								if text != "" {
-									kvmap[field] = append(kvmap[field], map[string]interface{}{
-										"field":       field,
-										"code":        in.Code,
-										"ruletext":    tag.Key,
-										"extfrom":     extfrom,
-										"sourcevalue": text,
-										"value":       text,
-										"type":        "colon1",
-										"matchtype":   "tag_regexp",
-										"blocktag":    bl.Classify,
-									})
-								}
-								break
-							}
-						}
-					}
-					for _, kv := range kvs2 {
-						if tag.Type == "string" {
-							if kv.Key == tag.Key {
-								text := ju.TrimLRSpace(kv.Value, "")
-								if text != "" {
-									kvmap[field] = append(kvmap[field], map[string]interface{}{
-										"field":       field,
-										"code":        in.Code,
-										"ruletext":    tag.Key,
-										"extfrom":     extfrom,
-										"sourcevalue": text,
-										"value":       text,
-										"type":        "colon2",
-										"matchtype":   "tag_string",
-										"blocktag":    bl.Classify,
-									})
-								}
-								break
-							}
-						} else if tag.Type == "regexp" {
-							if tag.Reg.MatchString(kv.Key) {
-								text := ju.TrimLRSpace(kv.Value, "")
-								if text != "" {
-									kvmap[field] = append(kvmap[field], map[string]interface{}{
-										"field":       field,
-										"code":        in.Code,
-										"ruletext":    tag.Key,
-										"extfrom":     extfrom,
-										"sourcevalue": text,
-										"value":       text,
-										"type":        "colon2",
-										"matchtype":   "tag_regexp",
-										"blocktag":    bl.Classify,
-									})
-								}
-								break
-							}
-						}
-					}
+			tp := ""
+			for k, v := range []*ju.JobKv{bl.ColonKV, bl.SpaceKV, bl.TableKV} {
+				if k == 0 {
+					tp = "colon"
+				} else if k == 1 {
+					tp = "space"
+				} else if k == 2 {
+					tp = "table"
 				}
-			}
-			//空格kv
-			if bl.SpaceKV != nil {
-				kvs := bl.SpaceKV.Kvs
-				// log.Debug("SpaceKV", kvs)
-				for _, tag := range tags {
-					for _, kv := range kvs {
-						if tag.Type == "string" {
-							if kv.Key == tag.Key {
-								text := ju.TrimLRSpace(kv.Value, "")
-								if text != "" {
-									kvmap[field] = append(kvmap[field], map[string]interface{}{
-										"field":       field,
-										"code":        in.Code,
-										"ruletext":    tag.Key,
-										"extfrom":     extfrom,
-										"sourcevalue": text,
-										"value":       text,
-										"type":        "space",
-										"matchtype":   "tag_string",
-										"blocktag":    bl.Classify,
-									})
-								}
-								break
-							}
-						} else if tag.Type == "regexp" {
-							if tag.Reg.MatchString(kv.Key) {
-								text := ju.TrimLRSpace(kv.Value, "")
-								if text != "" {
-									kvmap[field] = append(kvmap[field], map[string]interface{}{
-										"field":       field,
-										"code":        in.Code,
-										"ruletext":    tag.Key,
-										"extfrom":     extfrom,
-										"sourcevalue": text,
-										"value":       text,
-										"type":        "space",
-										"matchtype":   "tag_regexp",
-										"blocktag":    bl.Classify,
-									})
-								}
-								break
-							}
-						}
-					}
+				if v == nil || v.KvTags == nil {
+					continue
 				}
-			}
-			//表格kv
-			if bl.TableKV != nil {
-				tkv := bl.TableKV
-				// log.Debug("tkv", tkv)
-				for k, v := range tkv.Kv {
-					if k == fieldname {
-						if len(tags) > -tkv.KvIndex[fieldname] {
-							ruletext := ""
-							if fieldname == "项目名称" && -tkv.KvIndex[fieldname] == -100 {
-								ruletext = "项目名称"
-							} else {
-								ruletext = tags[-tkv.KvIndex[fieldname]].Key
-							}
-							kvmap[field] = append(kvmap[field], map[string]interface{}{
-								"field":       field,
-								"code":        in.Code,
-								"ruletext":    ruletext,
-								"extfrom":     "table",
-								"sourcevalue": v,
-								"value":       v,
-								"type":        "table",
-								"matchtype":   "tag_string",
-								"blocktag":    bl.Classify,
-							})
-						} else { //涉及其他待处理
-							// log.Debug(tags)
-						}
+				for _, vv := range v.KvTags[fieldname] {
+					text := ju.TrimLRSpace(vv.Value, "")
+					if text != "" {
+						kvmap[field] = append(kvmap[field], map[string]interface{}{
+							"field":       field,
+							"code":        in.Code,
+							"ruletext":    vv.Key,
+							"extfrom":     extfrom,
+							"sourcevalue": text,
+							"value":       text,
+							"type":        tp,
+							"matchtype":   "tag_string",
+							"blocktag":    bl.Tag,
+							"weight":      vv.Weight,
+						})
 					}
 				}
 			}
@@ -1286,28 +1150,17 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 				blocks = append(blocks, ju.BlockAndTag{v.Tag, tmpblock})
 			}
 			//把所有kv组装成一个字符串，存库
-			if v.ColonKV != nil {
-				for ck, cv := range v.ColonKV.Kv {
-					kvtext.WriteString(ck)
-					kvtext.WriteString(":")
-					kvtext.WriteString(cv)
-					kvtext.WriteString(" ")
-				}
-			}
-			if v.SpaceKV != nil {
-				for sk, sv := range v.SpaceKV.Kv {
-					kvtext.WriteString(sk)
-					kvtext.WriteString(":")
-					kvtext.WriteString(sv)
-					kvtext.WriteString(" ")
+			for _, jv := range []*ju.JobKv{v.ColonKV, v.SpaceKV, v.TableKV} {
+				if jv == nil {
+					continue
 				}
-			}
-			if v.TableKV != nil {
-				for tk, tv := range v.TableKV.Kv {
-					kvtext.WriteString(tk)
-					kvtext.WriteString(":")
-					kvtext.WriteString(tv)
-					kvtext.WriteString(" ")
+				for jv_k, jv_v := range jv.KvTags {
+					for _, jv_vv := range jv_v {
+						kvtext.WriteString(jv_k)
+						kvtext.WriteString(":")
+						kvtext.WriteString(jv_vv.Value)
+						kvtext.WriteString(" ")
+					}
 				}
 			}
 		}
@@ -1382,12 +1235,15 @@ func otherNeedSave(j *ju.Job, result map[string][]*ju.ExtField, e *ExtractTask)
 	for _, v := range j.Block {
 		//
 		for _, vv := range []*ju.JobKv{v.ColonKV, v.TableKV, v.SpaceKV} {
-			if vv == nil || vv.KvTag == nil {
+			if vv == nil || vv.KvTags == nil {
 				continue
 			}
-			for kkk, vvv := range vv.KvTag {
-				if vvv.Weight == ju.RetainKvWeight {
-					kv[kkk] = kv[kkk] + 1
+			for kkk, vvv := range vv.KvTags {
+				for _, vvvv := range vvv {
+					if vvvv.IsInvalid {
+						kv[kkk] = kv[kkk] + 1
+						break
+					}
 				}
 			}
 		}

src/jy/extract/extractInit.go

@@ -734,7 +734,7 @@ func (e *ExtractTask) InitTag() {
 			for k, key := range tmp {
 				tag := &Tag{Type: "string", Key: key.(string)}
 				e.Tag[field] = append(e.Tag[field], tag)
-				tab.Items[k] = &ju.Tag{key.(string), 0 - k, nil}
+				tab.Items[k] = &ju.Tag{"", key.(string), 0 - k, nil, false}
 			}
 			sort.Sort(tab.Items)
 			ju.TagdbTable[fname] = &tab
@@ -751,7 +751,7 @@ func (e *ExtractTask) InitTag() {
 			for k, key := range tmp {
 				tag := &Tag{Type: "regexp", Key: key.(string), Reg: regexp.MustCompile(key.(string))}
 				e.Tag[field] = append(e.Tag[field], tag)
-				tab.Items[k] = &ju.Tag{key.(string), 0 - k, regexp.MustCompile(key.(string))}
+				tab.Items[k] = &ju.Tag{"", key.(string), 0 - k, regexp.MustCompile(key.(string)), false}
 			}
 			sort.Sort(tab.Items)
 			ju.TagdbTable[fname+"_reg"] = &tab

src/jy/extract/score.go

@@ -17,20 +17,43 @@ var (
 	TagConfig     map[string]map[string]float64
 	TagConfigDesc map[string]string
 
-	TitleScore, RepeatScore, BlockScore float64
+	RepeatScore, BlockScore float64
+	CommonScore             map[string]float64
+	FieldsScore             map[string]map[string]float64
 )
 
 func init() {
 	qu.ReadConfig("./res/tagscoredesc.json", &TagConfigDesc)
 	qu.ReadConfig("./res/tagscore.json", &TagConfig)
 	qu.ReadConfig("./res/fieldscore.json", &SoreConfig)
-	TitleScore = qu.Float64All(SoreConfig["extractype"]["title"])
 	if repeat, ok := SoreConfig["other"]["repeat"].(map[string]interface{}); ok {
 		RepeatScore = qu.Float64All(repeat["score"])
 	}
 	if block, ok := SoreConfig["other"]["block"].(map[string]interface{}); ok {
 		BlockScore = qu.Float64All(block["score"])
 	}
+	//通用抽取属性打分配置
+	if tmp, ok := SoreConfig["extractype"]["common"].(map[string]interface{}); ok {
+		CommonScore = map[string]float64{}
+		for k, v := range tmp {
+			CommonScore[k] = qu.Float64All(v)
+		}
+	}
+	log.Println(CommonScore)
+	//指定抽取属性打分配置
+	if tmp, ok := SoreConfig["extractype"]["fields"].(map[string]interface{}); ok {
+		FieldsScore = map[string]map[string]float64{}
+		for key, fieldmap := range tmp {
+			fieldscore := map[string]float64{}
+			if field, ok := fieldmap.(map[string]interface{}); ok {
+				for k, score := range field {
+					fieldscore[k] = qu.Float64All(score)
+				}
+			}
+			FieldsScore[key] = fieldscore
+		}
+	}
+	log.Println(FieldsScore)
 	//实例化正则
 	for _, tmp := range SoreConfig {
 		//log.Println(tmp)
@@ -102,9 +125,10 @@ func ScoreFields(j *ju.Job) map[string][]*ju.ExtField {
 	result := j.Result
 	for field, tmps := range result {
 		for tmpsindex, tmpsvalue := range tmps {
+			describe := qu.ObjToString(SoreConfig["extractype"]["describe"])
 			if tmpsvalue.ExtFrom == "title" { //标题打分初始化
-				tmps[tmpsindex].Score += TitleScore
-				tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "title初始化", Code: "fieldscore.title", RuleText: qu.ObjToString(SoreConfig["extractype"]["describe"]), ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: TitleScore})
+				tmps[tmpsindex].Score += CommonScore["title"]
+				tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "title初始化", Code: "fieldscore.title", RuleText: describe, ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: CommonScore["title"]})
 			}
 			//是否有段标签
 			if len(tmpsvalue.BlockTag) > 0 {
@@ -125,23 +149,42 @@ func ScoreFields(j *ju.Job) map[string][]*ju.ExtField {
 			}
 
 			//抽取类型打分
-			if strings.Contains(tmpsvalue.Type, "colon") {
-				tmps[tmpsindex].Score += qu.Float64All(SoreConfig["extractype"]["colon"])
-				tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "colonkv", Code: "fieldscore.colon", RuleText: qu.ObjToString(SoreConfig["extractype"]["describe"]), ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: qu.Float64All(SoreConfig["extractype"]["colon"])})
-			} else if strings.Contains(tmpsvalue.Type, "space") {
-				tmps[tmpsindex].Score += qu.Float64All(SoreConfig["extractype"]["space"])
-				tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "spacekv", Code: "fieldscore.space", RuleText: qu.ObjToString(SoreConfig["extractype"]["describe"]), ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: qu.Float64All(SoreConfig["extractype"]["space"])})
-			} else if strings.Contains(tmpsvalue.Type, "table") {
-				tmps[tmpsindex].Score += qu.Float64All(SoreConfig["extractype"]["table"])
-				tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "tablekv", Code: "fieldscore.table", RuleText: qu.ObjToString(SoreConfig["extractype"]["describe"]), ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: qu.Float64All(SoreConfig["extractype"]["table"])})
-			} else if strings.Contains(tmpsvalue.Type, "regexp") {
-				tmps[tmpsindex].Score += qu.Float64All(SoreConfig["extractype"]["regexp"])
-				tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "regexp", Code: "fieldscore.regexp", RuleText: qu.ObjToString(SoreConfig["extractype"]["describe"]), ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: qu.Float64All(SoreConfig["extractype"]["regexp"])})
+			if FieldsScore[field] != nil { //指定抽取属性打分配置
+				fieldscore := FieldsScore[field]
+				if strings.Contains(tmpsvalue.Type, "colon") {
+					tmps[tmpsindex].Score += fieldscore["colon"]
+					tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "colonkv", Code: "fieldscore.colon", RuleText: describe, ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: fieldscore["colon"]})
+				} else if strings.Contains(tmpsvalue.Type, "space") {
+					tmps[tmpsindex].Score += fieldscore["space"]
+					tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "spacekv", Code: "fieldscore.space", RuleText: describe, ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: fieldscore["space"]})
+				} else if strings.Contains(tmpsvalue.Type, "table") {
+					tmps[tmpsindex].Score += fieldscore["table"]
+					tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "tablekv", Code: "fieldscore.table", RuleText: describe, ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: fieldscore["table"]})
+				} else if strings.Contains(tmpsvalue.Type, "regexp") {
+					tmps[tmpsindex].Score += fieldscore["regexp"]
+					tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "regexp", Code: "fieldscore.regexp", RuleText: describe, ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: fieldscore["regexp"]})
+				}
+			} else { //通用抽取属性打分配置
+				if strings.Contains(tmpsvalue.Type, "colon") {
+					tmps[tmpsindex].Score += CommonScore["colon"]
+					tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "colonkv", Code: "fieldscore.colon", RuleText: describe, ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: CommonScore["colon"]})
+				} else if strings.Contains(tmpsvalue.Type, "space") {
+					tmps[tmpsindex].Score += CommonScore["space"]
+					tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "spacekv", Code: "fieldscore.space", RuleText: describe, ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: CommonScore["space"]})
+				} else if strings.Contains(tmpsvalue.Type, "table") {
+					tmps[tmpsindex].Score += CommonScore["table"]
+					tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "tablekv", Code: "fieldscore.table", RuleText: describe, ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: CommonScore["table"]})
+				} else if strings.Contains(tmpsvalue.Type, "regexp") {
+					tmps[tmpsindex].Score += CommonScore["regexp"]
+					tmps[tmpsindex].ScoreItem = append(tmps[tmpsindex].ScoreItem, &ju.ScoreItem{Des: "regexp", Code: "fieldscore.regexp", RuleText: describe, ScoreFrom: "fieldscore.json", Value: tmpsvalue.Value, Score: CommonScore["regexp"]})
+				}
 			}
+
 			scoreRule := SoreConfig[field]
 			if scoreRule == nil {
 				continue
 			}
+			//配置打分
 			if scoreRule["type"] == "string" {
 				//1.长度打分
 				valueLen := utf8.RuneCountInString(fmt.Sprint(tmpsvalue.Value))
@@ -207,7 +250,7 @@ func ScoreFields(j *ju.Job) map[string][]*ju.ExtField {
 						}
 					}
 				}
-				//4.位置打分
+				//4.中标候选人打分
 				if winnerorders, ok := scoreRule["winnerorder"].([]interface{}); ok {
 					for _, winnerorder := range winnerorders {
 						if p, ok := winnerorder.(map[string]interface{}); ok {

src/jy/pretreated/analykv.go

@@ -23,7 +23,7 @@ var matchkh = map[string]string{
 	"〖": "〗",
 }
 
-func GetKvFromtxt(con, tag string, from int) ([]*u.Kv, map[string]string, map[string]*u.Tag) {
+func GetKvFromtxt(con, tag string, from int) ([]*u.Kv, map[string][]*u.Tag) {
 	res := FindKv(TextAfterRemoveTable(con), tag, from)
 	kvs := []*u.Kv{}
 	for _, k := range res.Keys {
@@ -35,8 +35,8 @@ func GetKvFromtxt(con, tag string, from int) ([]*u.Kv, map[string]string, map[st
 			})
 		}
 	}
-	kv, tagKv := KvTagsToKV(kvs, tag, nil, from)
-	return kvs, kv, tagKv
+	kvTags := GetKvTags(kvs, tag, nil)
+	return kvs, kvTags
 }
 
 type Line struct {

src/jy/pretreated/analystep.go

@@ -6,7 +6,6 @@ package pretreated
 import (
 	"encoding/json"
 	"jy/util"
-	qutil "qfw/util"
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
@@ -30,37 +29,20 @@ func AnalyStart(job *util.Job) {
 		}
 	}
 	blockArrays, _ := DivideBlock(job.CategorySecond, con, 1, job.RuleBlock) //分块
-	if len(blockArrays) > 0 { //有分块
+	if len(blockArrays) > 0 {                                                //有分块
 		//从块里面找分包
 		job.BlockPackage = FindPackageFromBlocks(&blockArrays, job.Title) //从块里面找分包
 		for _, bl := range blockArrays {
 			if len([]rune(bl.Text)) > 80 {
 				bl.Block, _ = DivideBlock(job.CategorySecond, bl.Text, 1, job.RuleBlock)
-			}
-			//块中再查找表格(块,处理完把值赋到块)
-			t1, _ := ComputeConRatio(bl.Text, 2)
-			if len(t1) > 0 {
-				job.HasTable = 1
-				for i := 0; i < len(tabs); i++ {
-					bl := &util.Block{}
-					//添加标识:文本中有table
-					tabres := AnalyTableV2(t1[0], job.Category, bl.Title, bl.Text, 2, job.SourceMid, job.RuleBlock) //解析表格入口 返回：汇总表格对象
-					processTableResult(tabres, bl, job)                                                             //分析table解析结果
-					if bl.Title == "" && tabres.BlockTag != "" {
-						bl.Title = tabres.BlockTag
-					}
-					if len(bl.TableKV.Kv) > 0 {
-						bl.Text = tabs[i].Text()
-						job.Block = append(job.Block, bl)
-					}
+				for _, bl_bl := range bl.Block {
+					processTableInBlock(bl_bl, job)
 				}
-				//				for k, v := range bl.TableKV.Kv {
-				//					log.Println("bl.TableKV.Kv", k, v)
-				//				}
 			}
+			processTableInBlock(bl, job)
+			//新加 未分块table中未能解析到中标候选人，从正文中解析
 			if job.Winnerorder == nil || len(job.Winnerorder) == 0 {
-				//新加table未找到winnerorder， 从分块文本中找中标候选人
-				job.Winnerorder = winnerOrderEntity.Find(bl.Text, true, 1)
+				bl.Winnerorder = winnerOrderEntity.Find(bl.Text, true, 1)
 			}
 			job.Block = append(job.Block, bl)
 		}
@@ -72,17 +54,9 @@ func AnalyStart(job *util.Job) {
 			newCon = TextAfterRemoveTable(con)
 			job.BlockPackage = FindPackageFromText(job.Title, newCon)
 			for i := 0; i < len(tabs); i++ {
-				bl := &util.Block{}
 				//添加标识:文本中有table
 				tabres := AnalyTableV2(tabs[i], job.Category, "", con, 1, job.SourceMid, job.RuleBlock) //解析表格入口 返回：汇总表格对象
 				processTableResult(tabres, bl, job)                                                     //分析table解析结果
-				if bl.Title == "" && tabres.BlockTag != "" {
-					bl.Title = tabres.BlockTag
-				}
-				if len(bl.TableKV.Kv) > 0 {
-					bl.Text = tabs[i].Text()
-					job.Block = append(job.Block, bl)
-				}
 			}
 			//			for k, v := range bl.TableKV.Kv {
 			//				log.Println("bl.TableKV.Kv", k, v)
@@ -91,19 +65,33 @@ func AnalyStart(job *util.Job) {
 			//从正文里面找分包
 			job.BlockPackage = FindPackageFromText(job.Title, newCon)
 		}
-		//新加 未分块table中未能解析到中标候选人，从正文中解析
-		if job.Winnerorder == nil || len(job.Winnerorder) == 0 {
-			bl.Winnerorder = winnerOrderEntity.Find(bl.Text, true, 1)
-		}
 		FindProjectCode(newCon, job) //匹配项目编号
 		bl.Text = newCon
 		//调用kv解析
 		bl.ColonKV = GetKVAll(newCon, "", nil, 1)
 		bl.SpaceKV = SspacekvEntity.Entrance(newCon, "", nil)
+		//新加 未分块table中未能解析到中标候选人，从正文中解析
+		if job.Winnerorder == nil || len(job.Winnerorder) == 0 {
+			bl.Winnerorder = winnerOrderEntity.Find(bl.Text, true, 1)
+		}
 		job.Block = append(job.Block, bl)
 	}
 }
 
+func processTableInBlock(bl *util.Block, job *util.Job) {
+	//块中再查找表格(块,处理完把值赋到块)
+	tabs, _ := ComputeConRatio(bl.Text, 2)
+	for _, tab := range tabs {
+		job.HasTable = 1
+		//添加标识:文本中有table
+		tabres := AnalyTableV2(tab, job.Category, bl.Title, bl.Text, 2, job.SourceMid, job.RuleBlock) //解析表格入口 返回：汇总表格对象
+		processTableResult(tabres, bl, job)                                                           //分析table解析结果
+		if bl.Title == "" && tabres.BlockTag != "" {
+			bl.Title = tabres.BlockTag
+		}
+	}
+}
+
 //匹配项目编号
 func FindProjectCode(newCon string, job *util.Job) {
 	newCon = TextAfterRemoveTable(newCon)
@@ -133,14 +121,16 @@ func FindProjectCode(newCon string, job *util.Job) {
 		jsonMap := make(map[string]string)
 		json.Unmarshal([]byte(proCode), &jsonMap)
 		jobKv := util.NewJobKv()
+		kvTags := map[string][]*util.Tag{}
 		for k, v := range jsonMap {
+			kvTags[k] = append(kvTags[k], &util.Tag{Key: k, Value: v})
 			tmpkv := new(util.Kv)
 			tmpkv.Line = k + v
 			tmpkv.Key = k
 			tmpkv.Value = v
 			jobKv.Kvs = append(jobKv.Kvs, tmpkv)
 		}
-		jobKv.Kv = jsonMap
+		jobKv.KvTags = kvTags
 		blCode.ColonKV = jobKv
 		job.Block = append(job.Block, blCode)
 	}
@@ -149,20 +139,7 @@ func FindProjectCode(newCon string, job *util.Job) {
 //分析table解析结果
 func processTableResult(tabres *TableResult, block *util.Block, job *util.Job) {
 	//解析结果中的kv
-	kv := map[string]string{}
-	for k, v := range tabres.SortKV.Map {
-		kv[k] = qutil.ObjToString(v)
-	}
-	kvIndex := map[string]int{}
-	for k, v := range tabres.SortKVWeight {
-		kvIndex[k] = v
-	}
-	KvTag := map[string]*util.Tag{}
-	for k, _ := range tabres.SortKV.NotTagKey {
-		KvTag[k] = &util.Tag{Weight: util.RetainKvWeight}
-	}
-	block.TableKV = &util.JobKv{Kv: kv, KvIndex: kvIndex, KvTag: KvTag}
-
+	block.TableKV = &util.JobKv{KvTags: tabres.KvTags}
 	//分包
 	tablePackage := map[string]*util.BlockPackage{}
 	if tabres.IsMultiPackage {
@@ -175,23 +152,19 @@ func processTableResult(tabres *TableResult, block *util.Block, job *util.Job) {
 			//解析kv
 			//找到key是“包1中标单位”这种的key，过滤掉包1，再次到标签库中匹配
 			labelKVs := []*util.Kv{}
-			if blockPackage.TableKV != nil && blockPackage.TableKV.Kv != nil {
-				for tk, tv := range blockPackage.TableKV.Kv {
-					if regReplKey.MatchString(tk) || regSplit.MatchString(tk) {
-						labelKVs = append(labelKVs, &util.Kv{
-							Key:   tk,
-							Value: tv,
-						})
+			if blockPackage.TableKV != nil && len(blockPackage.TableKV.KvTags) > 0 {
+				for tk, tv := range blockPackage.TableKV.KvTags {
+					for _, tvv := range tv {
+						if regReplKey.MatchString(tk) || regSplit.MatchString(tk) {
+							labelKVs = append(labelKVs, &util.Kv{
+								Key:   tk,
+								Value: tvv.Value,
+							})
+						}
 					}
 				}
 			}
-			labelKV, _ := KvTagsToKV(labelKVs, "", nil, 2)
-			for lk, lv := range labelKV {
-				if blockPackage.TableKV.Kv[lk] != "" {
-					continue
-				}
-				blockPackage.TableKV.Kv[lk] = lv
-			}
+			blockPackage.TableKV.KvTags = GetKvTags(labelKVs, "", nil)
 			tablePackage[k] = blockPackage
 		}
 	}
@@ -225,24 +198,14 @@ func processTableResult(tabres *TableResult, block *util.Block, job *util.Job) {
 				if bv.ColonKV == nil {
 					bv.ColonKV = util.NewJobKv()
 				}
-				for k, v := range tv.ColonKV.Kv {
-					if bv.ColonKV.Kv[k] != "" {
-						continue
-					}
-					bv.ColonKV.Kv[k] = v
-				}
+				MergeKvTags(bv.ColonKV.KvTags, tv.ColonKV.KvTags)
 			}
 			//
 			if tv.TableKV != nil {
 				if bv.TableKV == nil {
 					bv.TableKV = util.NewJobKv()
 				}
-				for k, v := range tv.TableKV.Kv {
-					if bv.TableKV.Kv[k] != "" {
-						continue
-					}
-					bv.TableKV.Kv[k] = v
-				}
+				MergeKvTags(bv.TableKV.KvTags, tv.TableKV.KvTags)
 			}
 			//
 			if tv.Origin != "" {

src/jy/pretreated/analytable.go

@@ -3,7 +3,6 @@ package pretreated
 import (
 	"fmt"
 	u "jy/util"
-	"log"
 	qutil "qfw/util"
 	"regexp"
 	"strings"
@@ -126,10 +125,9 @@ func IsHide(g *goquery.Selection) (b bool) {
 
 //对表格的key进行标准化处理，多个k相同时，出现覆盖问题
 //待扩展，暂不支持正则标签库
-func CommonDataAnaly(k, tabletag, tabledesc string, v interface{}) (k1, k2 []string, weight []int, v1, returntag string, b bool) {
-	k1, k2 = []string{}, []string{}
-	weight = []int{}
-	tk := k
+func CommonDataAnaly(k, tabletag, tabledesc string, v interface{}) (kvTags map[string][]*u.Tag, returntag string) {
+	kvTags = map[string][]*u.Tag{}
+	v1 := ""
 	if sv, sok := v.(string); sok { //取KV
 		v1 = sv
 	} else if sv, sok := v.([]string); sok { //是数组先默认取第一个
@@ -138,17 +136,18 @@ func CommonDataAnaly(k, tabletag, tabledesc string, v interface{}) (k1, k2 []str
 		}
 	}
 	//对值单位的处理   (预算|费|价|额|规模|投资)
-	if moneyreg.MatchString(tk) {
-		v1 += GetMoneyUnit(tk, v1)
+	if moneyreg.MatchString(k) {
+		v1 += GetMoneyUnit(k, v1)
 	}
 	//先清理key
 	//u.Debug(1, k, v1)
-	k = ClearKey(k, 2)
+	k1 := ClearKey(k, 2)
 	//u.Debug(2, k)
 	//取标准key
-	res := u.GetTags(k)
-	if len(res) == 0 && tk != k {
-		res = u.GetTags(tk)
+	res := u.GetTags(k1)
+	if len(res) == 0 && k1 != k {
+		res = u.GetTags(k)
+		k1 = k
 	}
 	//log.Println(k, res)
 	//	if len(res) == 0 {
@@ -156,50 +155,44 @@ func CommonDataAnaly(k, tabletag, tabledesc string, v interface{}) (k1, k2 []str
 	//	}
 	//当取到标准化值时，放入数组
 	if len(res) > 0 {
-		b = true
 		for _, t1 := range res {
-			k1 = append(k1, t1.Value)
-			weight = append(weight, t1.Weight)
+			//降低冒号值的权重
+			if MhSpilt.MatchString(v1) {
+				t1.Weight -= 50
+			}
+			kvTags[t1.Value] = append(kvTags[t1.Value], &u.Tag{Key: k1, Value: v1, Weight: t1.Weight})
 		}
 		//k1 = res[0].Value
-	}
-	//没有取到标准化key时，对中标金额和中标单位的逻辑处理
-	if !b {
+	} else {
+		kvTags[k] = append(kvTags[k], &u.Tag{Key: k, Value: v1, IsInvalid: true})
+		//没有取到标准化key时，对中标金额和中标单位的逻辑处理
 		if filter_zbje_k.MatchString(k) && !filter_zbje_kn.MatchString(k) && filter_zbje_v.MatchString(v1) {
 			if tabletag == "" {
 				returntag = "中标情况"
 			}
-			k1 = append(k1, "中标金额")
-			weight = append(weight, -100)
-			b = true
+			kvTags["中标金额"] = append(kvTags["中标金额"], &u.Tag{Key: "中标金额", Value: v1, Weight: -100})
 		} else if filter_zbdw_ky.MatchString(k) && !filter_zbdw_kn.MatchString(k) &&
 			filter_zbdw_v.MatchString(v1) {
-			k1 = append(k1, "中标单位")
-			weight = append(weight, -100)
+			kvTags["中标单位"] = append(kvTags["中标单位"], &u.Tag{Key: "中标单位", Value: v1, Weight: -100})
 			if tabletag == "" {
 				returntag = "中标情况"
 			}
-			b = true
 		} else {
-			k2 = append(k2, k)
-		}
-	}
-	//对上一步没有取到标准化key的进一步处理
-	if !b {
-		if tabletag == "" {
+			//对上一步没有取到标准化key的进一步处理
+			if tabletag == "" {
 
-		}
-		if filter_tag_zb.MatchString(tabletag) || filter_tag_zb.MatchString(tabledesc) {
-			//u.Debug(v1, k, "-----", filter_zbdw_jd.MatchString(k), filter_zbdw_v.MatchString(v1))
-			if filter_zbje_jd.MatchString(k) && !filter_zbje_kn.MatchString(k) && filter_zbje_v.MatchString(v1) {
-				k1 = append(k1, "中标金额")
-				weight = append(weight, -100)
-				b = true
-			} /*else if filter_zbdw_jd.MatchString(k) && filter_zbdw_v.MatchString(v1) {
-				k1 = append(k1, "中标单位")
-				weight = append(weight, -100)
-				b = true
-			}*/
+			}
+			if filter_tag_zb.MatchString(tabletag) || filter_tag_zb.MatchString(tabledesc) {
+				//u.Debug(v1, k, "-----", filter_zbdw_jd.MatchString(k), filter_zbdw_v.MatchString(v1))
+				if filter_zbje_jd.MatchString(k) && !filter_zbje_kn.MatchString(k) && filter_zbje_v.MatchString(v1) {
+					kvTags["中标金额"] = append(kvTags["中标金额"], &u.Tag{Key: "中标金额", Value: v1, Weight: -100})
+
+				} /*else if filter_zbdw_jd.MatchString(k) && filter_zbdw_v.MatchString(v1) {
+					k1 = append(k1, "中标单位")
+					weight = append(weight, -100)
+					b = true
+				}*/
+			}
 		}
 	}
 	return
@@ -230,45 +223,14 @@ func (table *Table) KVFilter() {
 			continue
 		}
 		v := table.SortKV.Map[k]
-		if table.SortKVWeight[k] == -99 { //td格式化kv降低权重
-			as.AddKey(k, v)
-			continue
-		}
 		if _, ok := v.(string); ok { //table.SortKV.Value为字符串，匹配抽取关键词table.SortKV.Key，匹配到添加k,v到table.StandKV,table.StandKVWeight
 			k = regSpliteSegment.ReplaceAllString(regReplAllSpace.ReplaceAllString(k, ""), "")
-			k1, n_k1, w1, v1, tag, b := CommonDataAnaly(k, table.Tag, table.Desc, v) //对key标准化处理，没有找到会走中标
+			kvTags, tag := CommonDataAnaly(k, table.Tag, table.Desc, v) //对key标准化处理，没有找到会走中标
 			//qutil.Debug(k, v, k1, w1, v1, tag, b)
-			if b {
-				//降低冒号值的权重
-				if MhSpilt.MatchString(v1) {
-					for pos, _ := range k1 {
-						w1[pos] -= 50
-					}
-				}
-				if tag != "" && table.Tag == "" {
-					table.Tag = tag
-				}
-				for pos, k2 := range k1 { //根据关键词,过滤table.SortKV到table.StandKV和table.StandKVWeight
-					if table.StandKV[k2] == "" || w1[pos] > table.StandKVWeight[k2] {
-						table.StandKV[k2] = v1 //本节点
-						table.StandKVWeight[k2] = w1[pos]
-					}
-					//					 else if k2 == "中标金额" {
-					//						//						u.Debug(qutil.Float64All(v1), qutil.Float64All(table.StandKV[k2]))
-					//						if qutil.Float64All(v1) > qutil.Float64All(table.StandKV[k2]) {
-					//							table.StandKV[k2] = v1
-					//						}
-					//					}
-				}
-			} else {
-				if table.StandKV[k] == "" && qutil.ObjToString(v) != "" {
-					table.StandKV[k] = qutil.ObjToString(v)
-					table.StandKVWeight[k] = -99
-				}
-			}
-			for _, n_k2 := range n_k1 {
-				table.SortKV.NotTagKey[n_k2] = true
+			if tag != "" && table.Tag == "" {
+				table.Tag = tag
 			}
+			MergeKvTags(table.StandKV, kvTags)
 		} else {
 			//u.Debug(k, v, "---------")
 			as.AddKey(k, v)
@@ -321,11 +283,10 @@ func (table *Table) KVFilter() {
 		if len(table.WinnerOrder) > 0 {
 			//中标候选人合并
 			winnerOrderEntity.Merge(table.WinnerOrder, winnerOrder)
-			if table.StandKV["中标单位"] == "" {
+			if len(table.StandKV["中标单位"]) == 0 {
 				ent := table.WinnerOrder[0]["entname"]
 				if ent != nil {
-					table.StandKV["中标单位"], _ = ent.(string)
-					table.StandKVWeight["中标单位"] = -25
+					table.StandKV["中标单位"] = append(table.StandKV["中标单位"], &u.Tag{Key: "中标单位", Value: qutil.ObjToString(ent), Weight: -25})
 				}
 			}
 		} else if !table.BPackage { //没有table.WinnerOrder也没有分包 将td中的WinnerOrder赋值给table.WinnerOrder
@@ -353,7 +314,7 @@ func (table *Table) KVFilter() {
 func (table *Table) sortKVArr(as *SortMap, winnertag bool) {
 	checkKey := map[int]bool{}
 	for kn, k := range as.Keys { //遍历table.SortKV.value为数组的key
-		if strings.TrimSpace(table.StandKV[k]) != "" {
+		if len(table.StandKV[k]) == 0 || strings.TrimSpace(table.StandKV[k][0].Value) != "" {
 			continue
 		}
 		v := as.Map[k]
@@ -488,26 +449,17 @@ func (table *Table) sortKVArr(as *SortMap, winnertag bool) {
 					}
 				}
 			}
-			k1, n_k1, w1, v1, tag, b := CommonDataAnaly(k, table.Tag, table.Desc, v)
-			if b {
-				if tag != "" && table.Tag == "" {
-					table.Tag = tag
-				}
-				for pos, k2 := range k1 {
-					if table.StandKV[k2] == "" || w1[pos] > table.StandKVWeight[k2] {
-						table.StandKV[k2] = v1 //本节点
-						table.StandKVWeight[k2] = w1[pos]
-					}
-					//				else if k2 == "中标金额" {
-					//					if qutil.Float64All(v1) > qutil.Float64All(table.StandKV[k2]) {
-					//						table.StandKV[k2] = v1
-					//					}
-					//				}
-				}
-			} else {
-				for _, n_k2 := range n_k1 {
-					table.SortKV.NotTagKey[n_k2] = true
-				}
+			kvTags, tag := CommonDataAnaly(k, table.Tag, table.Desc, v)
+			if tag != "" && table.Tag == "" {
+				table.Tag = tag
+			}
+			for kk, vv := range kvTags {
+				table.StandKV[kk] = append(table.StandKV[kk], vv...)
+				//				else if k2 == "中标金额" {
+				//					if qutil.Float64All(v1) > qutil.Float64All(table.StandKV[k2]) {
+				//						table.StandKV[k2] = v1
+				//					}
+				//				}
 			}
 		}
 	}
@@ -547,11 +499,8 @@ func (table *Table) analyTdKV() {
 			//u.Debug(td.BH, td.Val, td.SonTableResult)
 			if td.SonTableResult != nil {
 				//u.Debug(td.SonTableResult.SortKV.Map, "-------", td.SonTableResult.Tabs)
-				for _, k3 := range td.SonTableResult.SortKV.Keys {
-					if table.StandKV[k3] == "" || td.SonTableResult.SortKVWeight[k3] > table.StandKVWeight[k3] {
-						table.StandKV[k3] = qutil.ObjToString(td.SonTableResult.SortKV.Map[k3])
-						table.StandKVWeight[k3] = td.SonTableResult.SortKVWeight[k3]
-					}
+				for k3, v3 := range td.SonTableResult.KvTags {
+					table.StandKV[k3] = append(table.StandKV[k3], v3...)
 				}
 				//中标候选人排序
 				if table.WinnerOrder == nil || len(table.WinnerOrder) == 0 {
@@ -579,15 +528,23 @@ func (table *Table) MergerToTableresult() {
 					bp.TableKV = u.NewJobKv()
 				}
 				v1 := v.(*u.BlockPackage)
-				if v1.TableKV != nil && v1.TableKV.Kv != nil {
-					for k2, v2 := range v1.TableKV.Kv {
+				if v1.TableKV != nil && len(v1.TableKV.KvTags) > 0 {
+					for k2, v2 := range v1.TableKV.KvTags {
 						if bp.TableKV == nil {
 							bp.TableKV = u.NewJobKv()
 						}
-						if bp.TableKV.Kv[k2] == "" || (v1.TableKV.KvTag[k2] != nil && bp.TableKV.KvTag[k2] != nil && v1.TableKV.KvTag[k2].Weight > bp.TableKV.KvTag[k2].Weight) {
-							//可能会报错 assignment to entry in nil map
-							bp.TableKV.Kv[k2] = v2
-							bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
+						isExists := false
+						for _, v2v := range v2 {
+							for _, v2vv := range bp.TableKV.KvTags[k2] {
+								if v2v.Value == v2vv.Value {
+									isExists = true
+									break
+								}
+							}
+							if !isExists {
+								bp.TableKV.KvTags[k2] = append(bp.TableKV.KvTags[k2], v2v)
+								bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
+							}
 						}
 					}
 				}
@@ -603,25 +560,12 @@ func (table *Table) MergerToTableresult() {
 		//		u.Debug(table, table.TableResult, str)
 	}
 	//遍历标准key到tableresult.sortkv中
-	for k, v := range table.StandKV {
-		if table.TableResult.SortKV.Map[k] == nil || table.StandKVWeight[k] > table.TableResult.SortKVWeight[k] || strings.Contains(table.Tag, "变更") {
-			v = strings.Replace(v, "__", "", -1)
-			if table.TableResult.SortKV.Map[k] == nil {
-				table.TableResult.SortKV.AddKey(k, v) //父集
-			} else {
-				if k == "项目编号" { //项目编号存在，又匹配到全为中文跳过
-
-					if regHz.MatchString(v) {
-						continue
-					}
-				}
-				table.TableResult.SortKV.ReplaceKey(k, v, k)
-			}
-			table.TableResult.SortKVWeight[k] = table.StandKVWeight[k]
-		} else if table.TableResult.SortKV.Map[k] != nil {
-			//u.Debug(k, v, table.TableResult.SortKV.Map[k], "..............")
+	for _, v := range table.StandKV {
+		for _, vv := range v {
+			vv.Value = strings.Replace(vv.Value, "__", "", -1)
 		}
 	}
+	MergeKvTags(table.TableResult.KvTags, table.StandKV)
 	//表格的块标签
 	if table.TableResult.BlockTag == "" && table.Tag != "" {
 		table.TableResult.BlockTag = table.Tag
@@ -690,10 +634,13 @@ func (ts *TableResult) Analy() {
 		pns := map[string]string{}
 		pnarr := []string{}
 		for _, table := range tabs {
-			pn := table.StandKV["项目名称"]
-			if pn != "" && TitleReg.MatchString(pn) {
-				pnarr = append(pnarr, pn)
-				matchres := TitleReg.FindAllStringSubmatch(pn, -1)
+			if len(table.StandKV["项目名称"]) == 0 {
+				continue
+			}
+			pn := table.StandKV["项目名称"][0]
+			if pn != nil && pn.Value != "" && TitleReg.MatchString(pn.Value) {
+				pnarr = append(pnarr, pn.Value)
+				matchres := TitleReg.FindAllStringSubmatch(pn.Value, -1)
 				if len(matchres) == 1 && len(matchres[0]) > 0 {
 					v1 := u.PackageNumberConvert(matchres[0][0])
 					pns[v1] = matchres[0][0]
@@ -702,7 +649,9 @@ func (ts *TableResult) Analy() {
 					bp.Origin = matchres[0][0]
 					bp.TableKV = u.NewJobKv()
 					for _, k := range []string{"中标金额", "中标单位", "预算", "成交状态", "项目名称", "项目编号", "采购范围"} {
-						bp.TableKV.Kv[k] = table.StandKV[k]
+						if len(table.StandKV[k]) > 0 {
+							bp.TableKV.KvTags[k] = append(bp.TableKV.KvTags[k], &u.Tag{Key: k, Value: table.StandKV[k][0].Value})
+						}
 					}
 					bp.WinnerOrder = table.WinnerOrder
 					if table.BlockPackage.Map[v1] == nil {
@@ -727,8 +676,7 @@ func (ts *TableResult) Analy() {
 				}
 			}
 			if btrue {
-				ts.SortKV.AddKey("项目名称", pname)
-				ts.SortKVWeight["项目名称"] = 100
+				ts.KvTags["项目名称"] = append(ts.KvTags["项目名称"], &u.Tag{Key: "项目名称", Value: pname, Weight: 100})
 				for _, table := range tabs {
 					table.BPackage = true
 					//预算、中标金额、NullTxtBid成交供应商排名 中标单位 成交状态
@@ -742,7 +690,9 @@ func (ts *TableResult) Analy() {
 								bp.TableKV = u.NewJobKv()
 							}
 							for nk, k := range []string{"中标金额", "中标单位", "预算", "成交状态", "项目名称", "项目编号", "采购范围"} {
-								bp.TableKV.Kv[k] = table.StandKV[k]
+								if len(table.StandKV[k]) > 0 {
+									bp.TableKV.KvTags[k] = append(bp.TableKV.KvTags[k], &u.Tag{Key: k, Value: table.StandKV[k][0].Value})
+								}
 								if nk < 4 {
 									delete(table.StandKV, k)
 								}
@@ -832,8 +782,7 @@ func (tn *Table) AnalyTables(contactFormat *u.ContactFormat) []*Table {
 			if table.Tag != "" {
 				_, _, b := CheckMultiPackage(table.Tag, "")
 				if b {
-					table.StandKV["项目名称"] = table.Tag
-					table.StandKVWeight["项目名称"] = -100
+					table.StandKV["项目名称"] = append(table.StandKV["项目名称"], &u.Tag{Key: "项目名称", Value: table.Tag, Weight: -100})
 				}
 			}
 			table.TdContactFormat(contactFormat) //contactFormat，处理采购单位，代理机构
@@ -845,35 +794,12 @@ func (tn *Table) AnalyTables(contactFormat *u.ContactFormat) []*Table {
 			}
 			//判断是否是多包，并处理分包的//遍历td分块
 			table.CheckMultiPackageByTable()
-			//str := "\n"
-			//for k, v := range table.StandKV {
-			//	str += fmt.Sprintf("_==___%s:%v\n", k, v)
-			//	if table.TableResult.SortKV.Map[k] == nil {
-			//		table.TableResult.SortKV.AddKey(k, v)
-			//		table.TableResult.SortKVWeight[k] = table.StandKVWeight[k]
-			//	}
-			//}
 			res, _, _, _, _ := CheckCommon(table.Tag, "abandontable")
 			if !res {
 				//过滤、标准化、合并kv,table.StandKV,table.StandKVWeight
 				table.KVFilter()
 			}
-			if len(table.StandKV) == 0{
-				for k,v := range table.SortKV.Map{
-					table.StandKV[k] = fmt.Sprint(v)
-					table.StandKVWeight[k] = -999
-				}
-			}
-			for k, v := range table.StandKV { //过滤后的标准化kv
-				if table.TableResult.SortKV.Map[k] == nil || table.StandKVWeight[k] > table.TableResult.SortKVWeight[k] {
-					table.TableResult.SortKV.AddKey(k, v)
-					table.TableResult.SortKVWeight[k] = table.StandKVWeight[k]
-				}
-			}
-			for k, v := range table.SortKV.NotTagKey {
-				table.TableResult.SortKV.NotTagKey[k] = v
-			}
-			//u.Debug(str)
+			//MergeKvTags(table.TableResult.KvTags, table.StandKV)
 		}
 	}
 	return ts
@@ -917,8 +843,7 @@ func (table *Table) tableSubDemolitionTable() []*Table {
 				tab1 = NewTable("", table.TableResult, table.Goquery)
 				tab1.BSplit = true
 				if tmn[rownum] != nil {
-					tab1.StandKV["项目名称"] = tmn[rownum]["tag"].(string)
-					tab1.StandKVWeight["项目名称"] = -100
+					tab1.StandKV["项目名称"] = append(tab1.StandKV["项目名称"], &u.Tag{Key: "项目名称", Value: tmn[rownum]["tag"].(string), Weight: -100})
 				}
 				ts = append(ts, tab1)
 			}
@@ -1494,7 +1419,6 @@ func (table *Table) FindKV() {
 										continue
 									}
 									table.SortKV.AddKey(tdk, tdv)
-									table.SortKVWeight[tdk] = -99
 								}
 							}
 						}
@@ -2103,16 +2027,27 @@ func (tn *Table) manyPackageProcessByIndex(index []string, standIndex_pos []int)
 			//}
 			for _, vcgdw := range k1tags {
 				if vcgdw.Value == "采购单位" {
-					tn.SortKV.RemoveKey(k1)
 				}
 			}
 		} else if val, bvs := v1.(string); bvs && len(index) == 1 {
 			//删除子包的kv
-			k1tags, _, _, _, _, _ := CommonDataAnaly(k1, "", "", val)
-			if !(len(k1tags) > 0 && regexp.MustCompile("^(项目|开标|采购单位|招标机构)").MatchString(k1tags[0]))  {
-				//log.Println("remove", k1, val)
-				tn.SortKV.RemoveKey(k1)
-				tn.assemblePackage(k1, val, index[0])
+			kvTags, _ := CommonDataAnaly(k1, "", "", val)
+			for kvTag_k, kvTag_v := range kvTags {
+				hasValid := false
+				for _, kvTag_vv := range kvTag_v {
+					if kvTag_vv.IsInvalid {
+						continue
+					}
+					hasValid = true
+				}
+				if !hasValid {
+					continue
+				}
+				if !(len(kvTags) > 0 && regexp.MustCompile("^(项目|开标|采购单位|招标机构)").MatchString(kvTag_k)) {
+					tn.SortKV.RemoveKey(k1)
+					tn.assemblePackage(k1, val, index[0])
+					//log.Println("remove", k1, val)
+				}
 			}
 			//u.Debug("----==2==-------", k1)
 		}
@@ -2147,18 +2082,34 @@ func (tn *Table) isGoonNext() {
 					if bp.TableKV == nil {
 						bp.TableKV = u.NewJobKv()
 					}
-					for k2, v2 := range mv.ColonKV.Kv {
-						if bp.TableKV.Kv[k2] == "" {
-							bp.TableKV.Kv[k2] = v2
-							bp.TableKV.KvTag[k2] = mv.ColonKV.KvTag[k2]
-							bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
+					for k2, v2 := range mv.ColonKV.KvTags {
+						for _, v2v := range v2 {
+							isExists := false
+							for _, v2vv := range bp.TableKV.KvTags[k2] {
+								if v2v.Value == v2vv.Value {
+									isExists = true
+									break
+								}
+							}
+							if !isExists {
+								bp.TableKV.KvTags[k2] = append(bp.TableKV.KvTags[k2], v2v)
+								bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
+							}
 						}
 					}
-					for k2, v2 := range mv.SpaceKV.Kv {
-						if bp.TableKV.Kv[k2] == "" {
-							bp.TableKV.Kv[k2] = v2
-							bp.TableKV.KvTag[k2] = mv.SpaceKV.KvTag[k2]
-							bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
+					for k2, v2 := range mv.SpaceKV.KvTags {
+						for _, v2v := range v2 {
+							isExists := false
+							for _, v2vv := range bp.SpaceKV.KvTags[k2] {
+								if v2v.Value == v2vv.Value {
+									isExists = true
+									break
+								}
+							}
+							if !isExists {
+								bp.SpaceKV.KvTags[k2] = append(bp.SpaceKV.KvTags[k2], v2v)
+								bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
+							}
 						}
 					}
 				}
@@ -2330,22 +2281,9 @@ func (tn *Table) assemblePackage(k1, v1, key string) {
 		bp.TableKV = u.NewJobKv()
 	}
 	if v1 != "" {
-		k2, _, w1, v2, _, bf := CommonDataAnaly(k1, "中标情况", "", v1) //匹配抽取关键词
-		if bf {
-			for pos, k3 := range k2 {
-				if bp.TableKV.Kv != nil && bp.TableKV.KvTag[k3] != nil && (bp.TableKV.Kv[k3] == "" || w1[pos] > bp.TableKV.KvTag[k3].Weight) {
-					bp.TableKV.Kv[k3] = v2
-					bp.TableKV.KvTag[k3] = &u.Tag{Value: v2, Weight: w1[pos]}
-				} else {
-					bp.TableKV.Kv[k1] = qutil.ObjToString(v1)
-					//if tn.SortKV.Map[k3] == nil {
-					//	tn.SortKV.AddKey(k3, v2) //添加匹配到抽取关键词的key，value
-					//	tn.StandKVWeight[k3]=w1[pos]
-					//}
-				}
-			}
-		} else {
-			bp.TableKV.Kv[k1] = qutil.ObjToString(v1)
+		kvTags, _ := CommonDataAnaly(k1, "中标情况", "", v1) //匹配抽取关键词
+		for k3, v3 := range kvTags {
+			bp.TableKV.KvTags[k3] = append(bp.TableKV.KvTags[k3], v3...)
 		}
 	}
 	k1 = regReplAllSpace.ReplaceAllString(k1, "")
@@ -2712,9 +2650,9 @@ func modle(thisTdKvs []*u.Kv, td *TD, myContactType, td_k, td_v string, contactT
 	} else {
 		//
 		if !strings.HasSuffix(td_k, "方式") {
-			_, kTag := KvTagsToKV([]*u.Kv{&u.Kv{Key: myContactType + td_k, Value: td_v}}, "", BuyerContacts, 3)
-			if len(kTag) == 1 {
-				tagVal, _ := u.FirstKeyValueInMap(kTag)
+			kvTags := GetKvTags([]*u.Kv{&u.Kv{Key: myContactType + td_k, Value: td_v}}, "", BuyerContacts)
+			if len(kvTags) == 1 {
+				tagVal, _ := u.FirstKeyValueInMap(kvTags)
 				if tagVal == "采购单位联系人" && ContactBuyerPersonFilterReg.MatchString(td_v) {
 					return
 				}
@@ -2735,8 +2673,6 @@ func modle(thisTdKvs []*u.Kv, td *TD, myContactType, td_k, td_v string, contactT
 			}
 		}
 		td.SortKV.AddKey(myContactType+td_k, td_v)
-		log.Println(myContactType, td_k, td_v)
-		delete(td.SortKV.NotTagKey, td_k)
 	}
 }
 
@@ -2754,8 +2690,8 @@ func (tn *Table) asdFind(td_k string, matchCount int, weightMap map[string]map[s
 		if ContactInfoVagueReg.MatchString(td_k) {
 			thisTrHasMatch = tn.matchContactType(&matchMap, k, td_k, td_kv.Value, td, &weightMap, thisTrHasMatch)
 		} else if k == "采购单位" { //打标签，权重高的重新覆盖
-			_, kTag := KvTagsToKV([]*u.Kv{td_kv}, "", []string{"采购单位"}, 3)
-			tagVal, weightVal := u.FirstKeyValueInMap(kTag)
+			kvTags := GetKvTags([]*u.Kv{td_kv}, "", []string{"采购单位"})
+			tagVal, weightVal := u.FirstKeyValueInMap(kvTags)
 			if tagVal == k {
 				if weightMap[k][k] == nil || (weightVal != nil && weightVal.(int) >= weightMap[k][k].(int)) || len(matchMap[k]) == 0 {
 					weightMap[k][k] = weightVal.(int)
@@ -2808,9 +2744,9 @@ func (tn *Table) matchContactType(matchMap *map[string]map[string]bool, k string
 	}
 	isAddToMatchMap := true
 	if !strings.HasSuffix(td_k, "方式") {
-		_, kTag := KvTagsToKV([]*u.Kv{&u.Kv{Key: td_k, Value: td_v}}, "", BuyerContacts, 3)
-		if len(kTag) == 1 {
-			tagVal, weightVal := u.FirstKeyValueInMap(kTag)
+		kvTags := GetKvTags([]*u.Kv{&u.Kv{Key: td_k, Value: td_v}}, "", BuyerContacts)
+		if len(kvTags) == 1 {
+			tagVal, weightVal := u.FirstKeyValueInMap(kvTags)
 			if tagVal == "采购单位联系人" && ContactBuyerPersonFilterReg.MatchString(td_v) {
 				isAddToMatchMap = false
 			}
@@ -3201,7 +3137,7 @@ func initLineMapLineMapArr(table *Table) (lineMapArr map[string]*SortMap, lineMa
 	for _, key := range table.SortKV.Keys { //遍历table.SortKV.Keys而不是直接遍历table.SortKV.Map是为了得到table头的顺序
 		val := table.SortKV.Map[key]
 		key = regReplAllSpace.ReplaceAllString(key, "")
-		key = strings.Replace(key, "", "", -1) //处理一个特殊的采购量 经上层处理空格后未处理掉
+		key = strings.Replace(key, "", "", -1)    //处理一个特殊的采购量 经上层处理空格后未处理掉
 		if realTypeVal, ok := val.([]string); ok { //val为数组 {"数量":["1","2","3"]}
 			/*
 				{

src/jy/pretreated/colonkv.go

@@ -4,7 +4,7 @@ package pretreated
 import (
 	"jy/clear"
 	. "jy/util"
-	jutil "jy/util"
+	"log"
 	qutil "qfw/util"
 	"regexp"
 	"sort"
@@ -398,9 +398,9 @@ func FormatContactKv(kvs *[]*Kv, title string, buyers []string, contactFormat *C
 						matchMap[ct_k] = map[string]bool{}
 					}
 					if !strings.HasSuffix(k, "方式") {
-						_, kTag := KvTagsToKV([]*Kv{&Kv{Key: k, Value: v}}, "", BuyerContacts, 1)
-						if len(kTag) == 1 {
-							tagVal, weightVal := FirstKeyValueInMap(kTag)
+						kvTags := GetKvTags([]*Kv{&Kv{Key: k, Value: v}}, "", BuyerContacts)
+						if len(kvTags) == 1 {
+							tagVal, weightVal := FirstKeyValueInMap(kvTags)
 							if tagVal == "采购单位联系人" && ContactBuyerPersonFilterReg.MatchString(v) {
 								isAddToMatchMap = false
 							}
@@ -430,8 +430,8 @@ func FormatContactKv(kvs *[]*Kv, title string, buyers []string, contactFormat *C
 					}
 				}
 				if ct_k == "采购单位" { //打标签，权重高的重新覆盖
-					_, kTag := KvTagsToKV([]*Kv{&Kv{Key: k, Value: v}}, "", []string{"采购单位"}, 1)
-					tagVal, weightVal := FirstKeyValueInMap(kTag)
+					kvTags := GetKvTags([]*Kv{&Kv{Key: k, Value: v}}, "", []string{"采购单位"})
+					tagVal, weightVal := FirstKeyValueInMap(kvTags)
 					if tagVal == ct_k {
 						if weightMap[ct_k][ct_k] == nil || (weightVal != nil && weightVal.(int) > weightMap[ct_k][ct_k].(int)) {
 							weightMap[ct_k][ct_k] = weightVal.(int)
@@ -538,9 +538,9 @@ func FormatContactKv(kvs *[]*Kv, title string, buyers []string, contactFormat *C
 		allMatchCount++
 		delete(totalIndexMap, myContactType)
 		if !strings.HasSuffix(k, "方式") {
-			_, kTag := KvTagsToKV([]*Kv{&Kv{Key: myContactType + k, Value: v}}, "", BuyerContacts, 1)
-			if len(kTag) == 1 {
-				tagVal, _ := FirstKeyValueInMap(kTag)
+			kvTags := GetKvTags([]*Kv{&Kv{Key: myContactType + k, Value: v}}, "", BuyerContacts)
+			if len(kvTags) == 1 {
+				tagVal, _ := FirstKeyValueInMap(kvTags)
 				if tagVal == "采购单位联系人" && ContactBuyerPersonFilterReg.MatchString(v) {
 					continue
 				}
@@ -633,33 +633,53 @@ func HasOrderContactType(text string) []string {
 func GetKVAll(content, title string, contactFormat *ContactFormat, from int) *JobKv {
 	content = formatText(content, "kv")
 	m1Kvs, _ := colonkvEntity.entrance(content, title, contactFormat, from)
-	m1, m1Weight := KvTagsToKV(m1Kvs, title, nil, from)
-	if m1 == nil {
-		m1 = map[string]string{}
+	kvTags := GetKvTags(m1Kvs, title, nil)
+	m2Kvs, m2KvTags := GetKvFromtxt(content, title, from)
+	MergeKvTags(kvTags, m2KvTags)
+	return &JobKv{
+		Kvs:    m1Kvs,
+		Kvs_2:  m2Kvs,
+		KvTags: kvTags,
 	}
-	m2Kvs, m2, m2Weight := GetKvFromtxt(content, title, from)
-	for k, v := range m2 {
-		if m1[k] == "" {
-			m1[k] = v
-			m1Weight[k] = m2Weight[k]
+}
+
+//合并kv标签,把kvTags_2合并到kvTags_1
+func MergeKvTags(kvTags_1, kvTags_2 map[string][]*Tag) {
+	for k, v := range kvTags_2 {
+		for _, vv := range v {
+			if strings.TrimSpace(vv.Value) == "" {
+				continue
+			}
+			isExists := false
+			for _, vvv := range kvTags_1[k] {
+				if vvv.Value == vv.Value && vvv.Weight == vv.Weight {
+					isExists = true
+					break
+				}
+			}
+			if !isExists {
+				kvTags_1[k] = append(kvTags_1[k], vv)
+			}
 		}
 	}
-	return &JobKv{
-		Kvs:   m1Kvs,
-		Kvs_2: m2Kvs,
-		Kv:    m1,
-		KvTag: m1Weight,
+}
+
+//控制台输出kv的值
+func PrintKvTags(kvTags map[string][]*Tag) {
+	for k, v := range kvTags {
+		for _, vv := range v {
+			log.Println("kvTags===", k, "---", vv.Key, vv.Value, vv.Weight, vv.IsInvalid)
+		}
 	}
 }
 
 //KVTags转kv
-func KvTagsToKV(findkvs []*Kv, title string, tagdbs []string, from int) (map[string]string, map[string]*Tag) {
-	kvTags := map[string]*Tag{}
+func GetKvTags(findkvs []*Kv, title string, tagdbs []string) map[string][]*Tag {
+	kvTags := map[string][]*Tag{}
 	if title != "" && BlockTagMap[title] {
-		kvTags[title] = &Tag{title, 0, nil}
+		kvTags[title] = append(kvTags[title], &Tag{title, title, 0, nil, false})
 	}
 	for _, findkv := range findkvs {
-		kvMap := map[string]string{}
 		k, val := findkv.Key, findkv.Value
 		//val是空的话，不打标签
 		if filterValue.MatchString(val) {
@@ -688,43 +708,18 @@ func KvTagsToKV(findkvs []*Kv, title string, tagdbs []string, from int) (map[str
 				}
 			}
 		}
-		if len(tags) == 0 {
-			//go AddtoNoMatchMap(key)
-			//Debug(key)
-			//continue
-			//由跳过修改为保留
-			tags = []*Tag{&Tag{k, jutil.RetainKvWeight, nil}}
-		}
-		for _, tk := range tags {
-			//分包过来给kv打标签的时候，只取第一个，后面的不覆盖
-			if kvTags[tk.Value] == nil || (kvTags[tk.Value].Weight < tk.Weight && from != 4) {
-				//				fc := StandardNameMap[tk.Value]
-				//				if (fc != nil && fc.CheckNum) || (moneyreg.MatchString(tk.Value)) {
-				//					val += GetMoneyUnit(k, val)
-				//				}
+		if len(tags) > 0 {
+			for _, tk := range tags {
 				if moneyreg.MatchString(tk.Value) {
 					val += GetMoneyUnit(k, val)
 				}
-				//Debug("KV-key", tk, val)
-				kvTags[tk.Value] = &Tag{val, tk.Weight, nil}
-				kvMap[tk.Value] = val
-				//Debug("KV-key", tk.Value, val, key, tk.Weight)
-			}
-		}
-	}
-	//
-	kv := map[string]string{}
-	kvWeight := map[string]*Tag{}
-	if len(kvTags) > 0 {
-		for k, v := range kvTags {
-			if kv[k] != "" {
-				continue
+				kvTags[tk.Value] = append(kvTags[tk.Value], &Tag{Key: k, Value: val, Weight: tk.Weight})
 			}
-			kv[k] = v.Value
-			kvWeight[k] = v
+		} else {
+			kvTags[key] = append(kvTags[key], &Tag{Key: k, Value: val, IsInvalid: true})
 		}
 	}
-	return kv, kvWeight
+	return kvTags
 }
 
 func FilterContactKey(key string) string {

src/jy/pretreated/division.go

@@ -645,33 +645,12 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 				colonJobKv := GetKVAll(strings.TrimLeft(text, headKey), "", nil, 1)
 				if headKey != "" {
 					kvAgain := GetKVAll(text, "", nil, 4)
-					for kv_k, kv_v := range kvAgain.Kv {
-						if colonJobKv.Kv[kv_k] == "" {
-							colonJobKv.Kv[kv_k] = kv_v
-							colonJobKv.KvTag[kv_k] = kvAgain.KvTag[kv_k]
-						}
-					}
-				}
-				for kv_k, kv_v := range colonJobKv.Kv {
-					if kv_v == "" {
-						continue
-					}
-					if (*blockPackage)[index].ColonKV.Kv[kv_k] != "" {
-						continue
-					}
-					(*blockPackage)[index].ColonKV.Kv[kv_k] = kv_v
+					MergeKvTags(colonJobKv.KvTags, kvAgain.KvTags)
 				}
+				MergeKvTags((*blockPackage)[index].ColonKV.KvTags, colonJobKv.KvTags)
 				//合并空格kv
 				spaceJobKv := SspacekvEntity.Entrance(text, "", nil)
-				for kv_k, kv_v := range spaceJobKv.Kv {
-					if kv_v == "" {
-						continue
-					}
-					if (*blockPackage)[index].SpaceKV.Kv[kv_k] != "" {
-						continue
-					}
-					(*blockPackage)[index].SpaceKV.Kv[kv_k] = kv_v
-				}
+				MergeKvTags((*blockPackage)[index].SpaceKV.KvTags, spaceJobKv.KvTags)
 			} else {
 				newBpkg := &util.BlockPackage{
 					Origin:   bk,
@@ -683,12 +662,7 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 				finalKv := GetKVAll(strings.TrimLeft(text, headKey), "", nil, 4)
 				if headKey != "" {
 					kvAgain := GetKVAll(text, "", nil, 4)
-					for kv_k, kv_v := range kvAgain.Kv {
-						if finalKv.Kv[kv_k] == "" {
-							finalKv.Kv[kv_k] = kv_v
-							finalKv.KvTag[kv_k] = kvAgain.KvTag[kv_k]
-						}
-					}
+					MergeKvTags(finalKv.KvTags, kvAgain.KvTags)
 				}
 				newBpkg.ColonKV = finalKv
 				newBpkg.SpaceKV = SspacekvEntity.Entrance(text, "", nil)

src/jy/pretreated/spacekv.go

@@ -27,11 +27,10 @@ func (se *SpacekvEntity) Entrance(text, title string, contactFormat *util.Contac
 		kvMaps = append(kvMaps, kvMap...)
 	}
 	FormatContactKv(&kvMaps, title, nil, contactFormat)
-	kv, tagKv := KvTagsToKV(kvMaps, title, nil, 1)
+	kvTags := GetKvTags(kvMaps, title, nil)
 	return &util.JobKv{
-		Kvs:   kvMaps,
-		Kv:    kv,
-		KvTag: tagKv,
+		Kvs:    kvMaps,
+		KvTags: kvTags,
 	}
 }
 

src/jy/pretreated/tablev2.go

@@ -22,13 +22,12 @@ type TableResult struct {
 	Itype          int         //1全文 2是块
 	BlockTag       string      //块标签
 	Html           string
-	Tabs           []*Table           //子表集合，子表中包含标准化kv或原始kv
-	GoqueryTabs    *goquery.Selection //goquery对象
-	TableSize      int                //子表的个数0,1,n
-	IsMultiPackage bool               //是否有子包
-	PackageMap     *SortMap           //子包对象的sortmap，含标准化过的
-	SortKV         *SortMap           //全局KVmap值,标准化处理过的
-	SortKVWeight   map[string]int     //全局KVmap值,标准化处理过的
+	Tabs           []*Table            //子表集合，子表中包含标准化kv或原始kv
+	GoqueryTabs    *goquery.Selection  //goquery对象
+	TableSize      int                 //子表的个数0,1,n
+	IsMultiPackage bool                //是否有子包
+	PackageMap     *SortMap            //子包对象的sortmap，含标准化过的
+	KvTags         map[string][]*u.Tag //全局KVmap值,标准化处理过的
 	WinnerOrder    []map[string]interface{}
 	BrandData      [][]map[string]string //品牌抽取结果
 	HasKey         int                   //有key
@@ -40,17 +39,16 @@ type TableResult struct {
 //快速创建TableResult对象
 func NewTableResult(Id interface{}, Toptype, BlockTag, con string, Itype int, ruleBlock *u.RuleBlock) *TableResult {
 	return &TableResult{
-		Id:           Id,
-		Toptype:      Toptype,
-		Html:         con,
-		Itype:        Itype,
-		BlockTag:     BlockTag,
-		Tabs:         []*Table{},
-		GoqueryTabs:  &goquery.Selection{},
-		PackageMap:   NewSortMap(),
-		SortKV:       NewSortMap(),
-		SortKVWeight: map[string]int{},
-		RuleBlock:    ruleBlock,
+		Id:          Id,
+		Toptype:     Toptype,
+		Html:        con,
+		Itype:       Itype,
+		BlockTag:    BlockTag,
+		Tabs:        []*Table{},
+		GoqueryTabs: &goquery.Selection{},
+		PackageMap:  NewSortMap(),
+		KvTags:      map[string][]*u.Tag{},
+		RuleBlock:   ruleBlock,
 	}
 }
 
@@ -139,18 +137,12 @@ func NewTD(Goquery *goquery.Selection, tr *TR, table *Table) *TD {
 	if len(ub) > 0 {
 		for _, bl := range ub {
 			//冒号kv
-			for bl_ck, bl_cv := range bl.ColonKV.Kv {
-				if td.SortKV.Map[bl_ck] == nil || bl.ColonKV.KvTag[bl_ck].Weight >= td.SortKVWeight[bl_ck] {
-					td.SortKV.AddKey(bl_ck, bl_cv)
-					td.SortKVWeight[bl_ck] = bl.ColonKV.KvTag[bl_ck].Weight
-				}
+			for bl_ck, bl_cv := range bl.ColonKV.KvTags {
+				td.SortKV.AddKey(bl_ck, bl_cv)
 			}
 			//空格kv
-			for bl_sk, bl_sv := range bl.SpaceKV.Kv {
-				if td.SortKV.Map[bl_sk] == nil || bl.SpaceKV.KvTag[bl_sk].Weight >= td.SortKVWeight[bl_sk] {
-					td.SortKV.AddKey(bl_sk, bl_sv)
-					td.SortKVWeight[bl_sk] = bl.SpaceKV.KvTag[bl_sk].Weight
-				}
+			for bl_sk, bl_sv := range bl.SpaceKV.KvTags {
+				td.SortKV.AddKey(bl_sk, bl_sv)
 			}
 		}
 	}
@@ -158,13 +150,13 @@ func NewTD(Goquery *goquery.Selection, tr *TR, table *Table) *TD {
 	proCode := projectcodeReg.FindString(text)
 	if proCode != "" {
 		ckv := GetKVAll(proCode, "", nil, 1)
-		for k, v := range ckv.Kv {
+		for k, v := range ckv.KvTags {
 			td.SortKV.AddKey(k, v)
 			td.SortKVWeight[k] = -99
 		}
 	} else if proCode = projectcodeReg2.FindString(text); proCode != "" {
 		ckv := GetKVAll(proCode, "", nil, 1)
-		for k, v := range ckv.Kv {
+		for k, v := range ckv.KvTags {
 			td.SortKV.AddKey(k, v)
 			td.SortKVWeight[k] = -99
 		}
@@ -231,18 +223,10 @@ func (td *TD) tdHasTable(bsontable *bool, tr *TR) {
 
 				//sonts := AnalyTableV2(tabs, ts.Toptype, stag, td.Html, 2, ts.Id, table.TableResult.RuleBlock) //又一次调用解析表格入口
 				td.BH = false
-				for k, v := range sonts.SortKV.Map {
-					if td.TR.Table.TableResult == nil {
-						td.TR.Table.TableResult = NewTableResult(sonts.Id, sonts.Toptype, sonts.BlockTag, sonts.Html, sonts.Itype, sonts.RuleBlock)
-					}
-					if td.TR.Table.TableResult.SortKVWeight[k] < sonts.SortKVWeight[k]{
-						td.TR.Table.TableResult.SortKV.AddKey(k, v)
-						td.TR.Table.TableResult.SortKVWeight[k] = sonts.SortKVWeight[k]
-					}else {
-						td.SortKV.AddKey(k,v)
-						td.SortKVWeight[k]=sonts.SortKVWeight[k]
-					}
+				if td.TR.Table.TableResult == nil {
+					td.TR.Table.TableResult = NewTableResult(sonts.Id, sonts.Toptype, sonts.BlockTag, sonts.Html, sonts.Itype, sonts.RuleBlock)
 				}
+				MergeKvTags(td.TR.Table.TableResult.KvTags, sonts.KvTags)
 				td.SonTableResult = sonts
 				//for _, k := range sonts.SortKV.Keys {
 				//u.Debug(k, sonts.SortKV.Map[k])
@@ -281,11 +265,23 @@ func (td *TD) tdHasTable(bsontable *bool, tr *TR) {
 							tb1.AddKey(k, v)
 						} else {
 							bp := tb1.Map[k].(*u.BlockPackage)
-							if bp != nil && v1.TableKV != nil && v1.TableKV.Kv != nil {
-								for k2, v2 := range v1.TableKV.Kv {
-									if bp.TableKV.Kv != nil && bp.TableKV.Kv[k2] == "" {
-										bp.TableKV.Kv[k2] = v2
-										bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
+							if bp != nil && v1.TableKV != nil {
+								for k2, v2 := range v1.TableKV.KvTags {
+									if bp.TableKV == nil {
+										bp.TableKV = u.NewJobKv()
+									}
+									isExists := false
+									for _, v2v := range v2 {
+										for _, v2vv := range bp.TableKV.KvTags[k2] {
+											if v2v.Value == v2vv.Value {
+												isExists = true
+												break
+											}
+										}
+										if !isExists {
+											bp.TableKV.KvTags[k2] = append(bp.TableKV.KvTags[k2], v2v)
+											bp.Text += fmt.Sprintf("%v:%v\n", k2, v2)
+										}
 									}
 								}
 							}
@@ -311,22 +307,14 @@ func (td *TD) tdIsHb(tr *TR, table *Table, bsontable bool) {
 		ub, _ = DivideBlock("", td.Text, 2, table.TableResult.RuleBlock) //对td的原始值
 		//看是否划块
 		if len(ub) > 0 {
-			colonKvWeight := map[string]int{}
-			spaceKvWeight := map[string]int{}
 			for _, bl := range ub {
 				//冒号kv
-				for bl_ck, bl_cv := range bl.ColonKV.Kv {
-					if td.SortKV.Map[bl_ck] == nil || bl.ColonKV.KvTag[bl_ck].Weight >= colonKvWeight[bl_ck] {
-						colonKvWeight[bl_ck] = bl.ColonKV.KvTag[bl_ck].Weight
-						td.SortKV.AddKey(bl_ck, bl_cv)
-					}
+				for bl_ck, bl_cv := range bl.ColonKV.KvTags {
+					td.SortKV.AddKey(bl_ck, bl_cv)
 				}
 				//空格kv
-				for bl_sk, bl_sv := range bl.SpaceKV.Kv {
-					if td.SortKV.Map[bl_sk] == nil || bl.SpaceKV.KvTag[bl_sk].Weight >= spaceKvWeight[bl_sk] {
-						spaceKvWeight[bl_sk] = bl.SpaceKV.KvTag[bl_sk].Weight
-						td.SortKV.AddKey(bl_sk, bl_sv)
-					}
+				for bl_sk, bl_sv := range bl.SpaceKV.KvTags {
+					td.SortKV.AddKey(bl_sk, bl_sv)
 				}
 			}
 		}
@@ -367,16 +355,8 @@ func (td *TD) tdIsHb(tr *TR, table *Table, bsontable bool) {
 				if bp.TableKV == nil {
 					bp.TableKV = u.NewJobKv()
 				}
-				for k2, v2 := range bp_v.ColonKV.Kv {
-					if bp.TableKV.Kv[k2] == "" {
-						bp.TableKV.Kv[k2] = v2
-					}
-				}
-				for k2, v2 := range bp_v.SpaceKV.Kv {
-					if bp.TableKV.Kv[k2] == "" {
-						bp.TableKV.Kv[k2] = v2
-					}
-				}
+				MergeKvTags(bp.TableKV.KvTags, bp_v.ColonKV.KvTags)
+				MergeKvTags(bp.TableKV.KvTags, bp_v.SpaceKV.KvTags)
 				table.TableResult.PackageMap.Map[bp_k] = bp
 			}
 		}
@@ -582,9 +562,7 @@ type Table struct {
 	TDNum                  int                       //td个数
 	BPackage               bool                      //是否有包
 	SortKV                 *SortMap                  //带排序的KV值
-	SortKVWeight           map[string]int            //带排序的KV值
-	StandKV                map[string]string         //过滤后的标准化kv
-	StandKVWeight          map[string]int            //过滤后的标准化kv
+	StandKV                map[string][]*u.Tag       //过滤后的标准化kv
 	StandRuleKV            map[string]string         //过滤后的规则kv
 	kvscope                map[int]map[int][]*TD     //sortkey第几个元素的的第几个值的结束位置
 	kTD                    map[int]*TD               //根据索引找到key的TD元素
@@ -610,9 +588,7 @@ func NewTable(Html string, TableResult *TableResult, tab *goquery.Selection) *Ta
 	return &Table{
 		Html:                   Html,
 		SortKV:                 NewSortMap(),
-		SortKVWeight:           map[string]int{},
-		StandKV:                map[string]string{},
-		StandKVWeight:          map[string]int{},
+		StandKV:                map[string][]*u.Tag{},
 		kvscope:                map[int]map[int][]*TD{},
 		kTD:                    map[int]*TD{},
 		SonTables:              []*Table{},
@@ -651,20 +627,18 @@ func (t *Table) InsertTR(tr *TR) {
 
 //支持排序的map
 type SortMap struct {
-	Index     map[string]int
-	Keys      []string
-	Map       map[string]interface{}
-	Lock      sync.Mutex
-	NotTagKey map[string]bool
+	Index map[string]int
+	Keys  []string
+	Map   map[string]interface{}
+	Lock  sync.Mutex
 }
 
 //快速创建排序map
 func NewSortMap() *SortMap {
 	return &SortMap{
-		Index:     map[string]int{},
-		Keys:      []string{},
-		Map:       map[string]interface{}{},
-		NotTagKey: map[string]bool{},
+		Index: map[string]int{},
+		Keys:  []string{},
+		Map:   map[string]interface{}{},
 	}
 }
 

src/jy/pretreated/winnerorder.go

@@ -224,10 +224,10 @@ func (wo *WinnerOrderEntity) findByReg(content string, blocks []string, reg_2 *r
 				object["type"] = 1
 			}
 		} else { //中标金额
-			_, standardKvTag := KvTagsToKV([]*util.Kv{&util.Kv{Key: k, Value: v}}, "", []string{"中标金额"}, 1)
+			kvTags := GetKvTags([]*util.Kv{&util.Kv{Key: k, Value: v}}, "", []string{"中标金额"})
 			//找到了中标金额
-			if standardKvTag["中标金额"] != nil && object["entname"] != nil {
-				val := wo.clear("中标金额", standardKvTag["中标金额"].Value)
+			if len(kvTags["中标金额"]) > 0 && object["entname"] != nil {
+				val := wo.clear("中标金额", kvTags["中标金额"][0].Value)
 				if val != nil {
 					object["price"] = val
 				}

src/jy/util/article.go

@@ -154,20 +154,16 @@ type Kv struct {
 
 //最终放到job上的kv
 type JobKv struct {
-	Kvs     []*Kv             //有序的冒号kv
-	Kvs_2   []*Kv             //有序的冒号kv
-	Kv      map[string]string //table kv (分出的对应的KV值)
-	KvIndex map[string]int    //kv_index(流程)
-	KvTag   map[string]*Tag   //带权重的kv
+	Kvs    []*Kv             //有序的冒号kv
+	Kvs_2  []*Kv             //有序的冒号kv
+	KvTags map[string][]*Tag //带权重的kv
 }
 
 func NewJobKv() *JobKv {
 	return &JobKv{
-		Kvs:     []*Kv{},
-		Kvs_2:   []*Kv{},
-		Kv:      map[string]string{},
-		KvIndex: map[string]int{},
-		KvTag:   map[string]*Tag{},
+		Kvs:    []*Kv{},
+		Kvs_2:  []*Kv{},
+		KvTags: map[string][]*Tag{},
 	}
 }
 

src/jy/util/tagmatch.go

@@ -14,9 +14,11 @@ var lock sync.Mutex
 
 //单条tag
 type Tag struct {
-	Value  string //
-	Weight int    //权重
-	TagReg *regexp.Regexp
+	Key       string //原始的key
+	Value     string //
+	Weight    int    //权重
+	TagReg    *regexp.Regexp
+	IsInvalid bool //是否打上标签
 }
 type Tags []*Tag //
 //tag文件
@@ -53,7 +55,7 @@ func (t *TagFile) Load(path string) {
 	//排序
 	t.Items = make([]*Tag, len(lines))
 	for k, v := range lines {
-		t.Items[k] = &Tag{v, 0 - k, nil}
+		t.Items[k] = &Tag{"", v, 0 - k, nil, false}
 	}
 	sort.Sort(t.Items)
 	t.Name = name
@@ -142,11 +144,7 @@ func GetAppointTags(src string, array []string) Tags {
 			continue
 		}
 		if ok, tag := v.Match(src); ok {
-			ret = append(ret, &Tag{
-				v.Name,
-				tag.Weight,
-				tag.TagReg,
-			})
+			ret = append(ret, &Tag{src, v.Name, tag.Weight, tag.TagReg, false})
 		}
 	}
 	lock.Unlock()
@@ -160,11 +158,7 @@ func GetBlockTags(src string) Tags {
 	ret := make(Tags, 0)
 	for _, v := range blocktagdb {
 		if ok, tag := v.Match(src); ok {
-			ret = append(ret, &Tag{
-				v.Name,
-				tag.Weight,
-				nil,
-			})
+			ret = append(ret, &Tag{src, v.Name, tag.Weight, nil, false})
 		}
 	}
 	//sort.Sort(ret)

src/jy/util/util.go

@@ -21,10 +21,9 @@ var BrandRules map[string]map[string]string
 var GoodsConfig []string
 var BrandConfig []string
 
-var GoodsGet *DFA           //商品
-var BrandGet *DFA           //品牌
-var IsBrandGoods bool       //是否开启品牌抽取
-var RetainKvWeight = -99999 //没有标准化的kv的权重
+var GoodsGet *DFA     //商品
+var BrandGet *DFA     //品牌
+var IsBrandGoods bool //是否开启品牌抽取
 
 func init() {
 	syncint = make(chan bool, 1)

src/jy/util/util2.go

@@ -245,6 +245,13 @@ func FirstKeyValueInMap(m interface{}) (string, interface{}) {
 		for k, v := range mv {
 			return k, v.Weight
 		}
+	} else if mv, ok := m.(map[string][]*Tag); ok {
+		for k, v := range mv {
+			if len(v) > 0 {
+				return k, v[0].Weight
+			}
+			return k, 0
+		}
 	} else if mv, ok := m.(map[string]string); ok {
 		for k, v := range mv {
 			return k, v

src/main_blocktest.go

@@ -0,0 +1,224 @@
+package main
+
+import (
+	"fmt"
+	"jy/extract"
+	"jy/mongodbutil"
+	"jy/pretreated"
+	ju "jy/util"
+	"log"
+	"os"
+	qu "qfw/util"
+	"regexp"
+	"time"
+)
+
+var f *os.File
+var m = map[string]bool{}
+
+func main12() {
+	//log.Println(pretreated.ProcTitle("以上公告内容如有变动将在相关网络媒体上另行通知凡购买本招标文件的单位必须就此采购项目的相关事宜详细咨询否则参与投标即被视为已经充分了解了招标方的需求中标后承担该文件范围内的所有要求投标前如对招标文件存有疑问请在投标截止日期前三个工作日以实名制书面文件向我公司询问否则视为接受已报名购买招标文件的投标商未递交投标文件或虽递交投标文件但未参加开标大会的投标商不得再参加该项目的采购活动"))
+	//return
+	f, _ = os.OpenFile("./title.txt", os.O_RDWR|os.O_CREATE, 777)
+	//all()
+	one()
+}
+func all() {
+	m := mongodbutil.MgoFactory(3, 3, 120, "192.168.3.207:27082", "extract_kf")
+	sess := m.Get()
+	defer m.Close(sess)
+	it := sess.DB("extract_kf").C("bidding201901").Find(nil).Iter()
+	pool := make(chan bool, 5)
+	count := 0
+	for temp := make(map[string]interface{}); it.Next(&temp); {
+		pool <- true
+		count++
+		go func(d map[string]interface{}) {
+			defer func() {
+				<-pool
+			}()
+			com(d)
+		}(temp)
+		temp = make(map[string]interface{})
+		if count%200 == 0 {
+			log.Println(count)
+		}
+	}
+	log.Println("over...")
+	time.Sleep(time.Hour)
+}
+func one() {
+	m := mongodbutil.MgoFactory(3, 3, 120, "192.168.3.207:27081", "qfw")
+	d, _ := m.FindById("bidding", "5d423d13a5cb26b9b76e4479", nil)
+	com(*d)
+}
+func com(doc map[string]interface{}) {
+	detail := GetDetail(doc)
+	doc["detail"] = detail
+	toptype := qu.ObjToString(doc["toptype"])
+	subtype := qu.ObjToString(doc["subtype"])
+	if qu.ObjToString(doc["type"]) == "bid" {
+		toptype = "结果"
+	}
+	if toptype == "" {
+		toptype = "*"
+	}
+	e := &extract.ExtractTask{
+		TaskInfo: &extract.TaskInfo{
+			Version:     "V3.1.2",
+			VersionId:   "5cdd1c70e138234848c1d703",
+			ProcessPool: make(chan bool, 1),
+		},
+	}
+
+	e.Id = qu.ObjToString(ju.Config["udptaskid"])
+	e.InitTaskInfo()
+	//d.TaskInfo.FDB = db.MgoFactory(3, 5, 600, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
+	//d.TaskInfo.TDB = db.MgoFactory(3, 5, 600, ext.TaskInfo.ToDbAddr, ext.TaskInfo.ToDB)
+	e.InitRulePres()
+	e.InitRuleBacks()
+	e.InitRuleCore()
+	e.InitBlockRule()
+	e.InitTag()
+	e.InitClearFn()
+	if e.IsExtractCity { //版本上控制是否开始城市抽取
+		//初始化城市DFA信息
+		e.InitCityDFA()
+		e.InitAreaCode()
+		e.InitPostCode()
+	}
+	//质量审核
+	e.InitAuditFields()
+	e.InitAuditRule()
+	e.InitAuditClass()
+	e.InitAuditRecogField()
+
+	//品牌抽取是否开启
+	ju.IsBrandGoods, _ = ju.Config["brandgoods"].(bool)
+
+	j := &ju.Job{
+		SourceMid:      qu.BsonIdToSId(doc["_id"]),
+		Category:       toptype,
+		CategorySecond: subtype,
+		Content:        qu.ObjToString(doc["detail"]),
+		SpiderCode:     qu.ObjToString(doc["spidercode"]),
+		//Domain:     qu.ObjToString(doc["domain"]),
+		//Href:       qu.ObjToString(doc["href"]),
+		Title:     qu.ObjToString(doc["title"]),
+		Data:      &doc,
+		City:      qu.ObjToString(doc["city"]),
+		Province:  qu.ObjToString(doc["area"]),
+		Result:    map[string][]*ju.ExtField{},
+		BuyerAddr: qu.ObjToString(doc["buyeraddr"]),
+		RuleBlock: e.RuleBlock,
+	}
+	e.TaskInfo.ProcessPool <- true
+	pretreated.AnalyStart(j)
+	e.ExtractProcess(j, nil)
+	log.Println("=============KvTags================")
+	for _, v := range j.Block {
+		if v.ColonKV != nil {
+			for kk, vv := range v.ColonKV.KvTags {
+				for _, vvv := range vv {
+					log.Println("ColonKV", kk, "---", vvv.Key, "---", vvv.Value, "---", vvv.Weight)
+				}
+			}
+		}
+		if v.SpaceKV != nil {
+			for kk, vv := range v.SpaceKV.KvTags {
+				for _, vvv := range vv {
+					log.Println("SpaceKV", kk, "---", vvv.Key, "---", vvv.Value, "---", vvv.Weight)
+				}
+			}
+		}
+		if v.TableKV != nil {
+			for kk, vv := range v.TableKV.KvTags {
+				for _, vvv := range vv {
+					log.Println("TableKV", kk, "---", vvv.Key, "---", vvv.Value, "---", vvv.Weight)
+				}
+			}
+		}
+	}
+	log.Println("=============抽取结果================")
+	set := (e.ResultArr[0][1]["$set"]).(map[string]interface{})
+	for k, v := range set {
+		if k == "budget" || k == "amount" || k == "winner" || k == "amount" || k == "projectname" || k == "projectcode" || k == "buyer" || k == "buyerperson" || k == "buyertel" || k == "agency" {
+			log.Println(k, "---", v)
+		}
+	}
+	log.Println("=============抽取结果 result================")
+	for k, v := range set["result"].(map[string][]*ju.ExtField) {
+		break
+		for _, vv := range v {
+			log.Println(k, fmt.Sprintf("%+v", vv))
+		}
+	}
+	log.Println("=============正文================")
+	//log.Println(j.Content)
+	return
+	for _, v := range j.Block {
+		if v.ColonKV != nil && v.ColonKV.KvTags != nil {
+			for kk, vv := range v.ColonKV.KvTags {
+				for _, vvv := range vv {
+					log.Println(kk, vvv.Weight, vvv.Value)
+				}
+			}
+		}
+		if v.TableKV != nil && v.TableKV.KvTags != nil {
+			for kk, vv := range v.TableKV.KvTags {
+				for _, vvv := range vv {
+					log.Println(kk, vvv.Weight, vvv.Value)
+				}
+			}
+		}
+		if v.SpaceKV != nil && v.SpaceKV.KvTags != nil {
+			for kk, vv := range v.SpaceKV.KvTags {
+				for _, vvv := range vv {
+					log.Println(kk, vvv.Weight, vvv.Value)
+				}
+			}
+		}
+	}
+	log.Println(len(j.Block))
+	return
+	for _, v := range j.Block {
+		if m[v.Title] || v.Title == "" {
+			continue
+		}
+		if !regexp.MustCompile("或|和|以?及|与|、|或").MatchString(v.Title) {
+			//continue
+		}
+		m[v.Title] = true
+		f.WriteString(j.SourceMid + "-----" + v.Title + "---" + fmt.Sprint(v.Titles) + "\n")
+		continue
+		for _, kv := range v.ColonKV.Kvs {
+			log.Println("\n")
+			log.Println(kv.Key, "---", kv.Value)
+			log.Println(kv.Line)
+			log.Println("=======================")
+		}
+	}
+}
+func GetDetail(doc map[string]interface{}) (detail string) {
+	detail = ""
+	d1, _ := doc["detail"].(string)
+	d2, _ := doc["contenthtml"].(string)
+	if len(d1) >= len(d2) || d2 == "" {
+		detail = d1
+	} else {
+		detail = d2
+	}
+	detail = ju.CutLableStr(detail)
+	detail = ju.NewCut().ClearHtml(detail)
+	tabs, ration := pretreated.ComputeConRatio(detail, 1)
+	if len(tabs) > 0 {
+		newcon, newtabs, newration := pretreated.FindBigText(detail, ration, tabs)
+		//log.Println(newcon, newtabs, newration)
+		if newcon != "" && newration == 0 {
+			detail = newcon
+			tabs = newtabs
+			ration = newration
+		}
+	}
+	return detail
+}

src/res/fieldscore.json

@@ -1,12 +1,29 @@
 {
     "extractype": {
         "describe": "抽取类型打分",
-        "title": 2,
-        "table": 3,
-        "colon": 3,
-        "space": 3,
-        "regexp": 2,
-        "winnerorder": 3
+        "common": {
+            "title": 2,
+            "table": 3,
+            "colon": 3,
+            "space": 3,
+            "regexp": 2
+        },
+        "fields": {
+            "projectname": {
+                "title": 1,
+                "table": 3,
+                "colon": 3,
+                "space": 3,
+                "regexp": 2
+            },
+            "winner": {
+                "table": 3,
+                "colon": 3,
+                "space": 3,
+                "regexp": 2,
+                "winnerorder": 3
+            }
+        }
     },
     "other": {
         "block": {
@@ -69,7 +86,7 @@
                 "range": [
                     10,
                     35,
-                    3
+                    2
                 ]
             },
             {
@@ -95,7 +112,7 @@
         "positivewords": [
             {
                 "describe": "以*结尾",
-                "regstr": ".{2,100}(委员会|办公室|幼儿园|动物园|图书馆|殡仪馆|博物馆|基地|青年宫|少年宫|艺术宫|电视台|中心|协会|公司|政府|初中|集团|银行|\\[大中小\\]学|部|院|局|厂|店|所|队|社|室|厅|段|会|场)$",
+                "regstr": ".{2,100}(委员会|办公室|幼儿园|动物园|图书馆|殡仪馆|博物馆|基地|青年宫|少年宫|艺术宫|电视台|中心|协会|公司|政府|初中|集团|银行|[大中小]学|部|院|局|厂|店|所|队|社|室|厅|段|会|场)$",
                 "score": 3
             }
         ],
@@ -315,7 +332,7 @@
             },
             {
                 "describe": "包含负分",
-                "regstr": "(勘察|设计|设备|项目|标段|工程|监理|范围|分包|月|日|天|[,，。、：:“”‘’_\"])",
+                "regstr": "(null|勘察|设计|设备|项目|标段|工程|监理|范围|分包|月|日|天|[,，。、：:“”‘’_\"])",
                 "score": -1
             },
             {

src/web/templates/admin/blockinfo.html

@@ -94,7 +94,7 @@ $(function () {
 			var _tit="",htmlObj={},obj,tag=[]
 			switch(n){
 			case "classify":
-				window.location.href = "/admin/version/blockclassify?vid=5b88f682752d50045860188a";
+				window.location.href = "/admin/version/blockclassify?vid={{.vid}}";
 				break
 			case "edit":	
                 obj=ttable.row($(this).closest("tr")).data();