Merge branch 'dev3.2' of ssh://192.168.3.207:10022/qmx/jy-data-extract into dev3.2

src/jy/extract/extract.go

@@ -24,12 +24,12 @@ import (
 var (
 	lock, lockrule, lockclear sync.RWMutex
 
-	cut           = ju.NewCut()                          //获取正文并清理
-	ExtLogs       map[*TaskInfo][]map[string]interface{} //抽取日志
-	TaskList      map[string]*ExtractTask                //任务列表
-	ClearTaskList map[string]*ClearTask                  //清理任务列表
-	saveLimit     = 200                                  //抽取日志批量保存
-	PageSize      = 5000                                 //查询分页
+	cut     = ju.NewCut()                          //获取正文并清理
+	ExtLogs map[*TaskInfo][]map[string]interface{} //抽取日志
+	TaskList      map[string]*ExtractTask          //任务列表
+	ClearTaskList map[string]*ClearTask            //清理任务列表
+	saveLimit     = 200                            //抽取日志批量保存
+	PageSize      = 5000                           //查询分页
 	Fields        = `{"title":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1}`
 	Fields2       = `{"budget":1,"bidamount":1,"title":1,"projectname":1,"winner":1}`
 )
@@ -892,7 +892,7 @@ func extRegCoreToResult(extfrom, text string, tag *map[string]bool, j *ju.Job, v
 						if v.RegCore.NumSign == -1 { //正负值修正
 							val = "-" + val
 						}
-						exfield := ju.ExtField{BlockTag: *tag, Field: k, Code: v.Code, RuleText: v.RuleText, Type: "regexp", MatchType: "regcontent", ExtFrom: extfrom, SourceValue: val, Value: val}
+						exfield := ju.ExtField{BlockTag: *tag, Field: k, Code: v.Code, RuleText: v.RuleText, Type: "regexp", MatchType: "regcontent", ExtFrom: extfrom, SourceValue: text, Value: val}
 						if extfrom == "title" {
 							exfield.Score = 4
 						}
@@ -919,6 +919,59 @@ func extRegCoreToResult(extfrom, text string, tag *map[string]bool, j *ju.Job, v
 					}
 				}
 			}
+			if len(extinfo) == 0 {
+				regArr := strings.Split(v.RuleText, "__")
+				//fmt.Println(regArr[0])
+				if len(regArr) > 0 {
+					reg, err := regexp.Compile(regArr[0])
+					if err == nil {
+						datavals := reg.FindStringSubmatch(text)
+						tmps := []map[string]interface{}{}
+						for _, value := range datavals {
+							if value == "" {
+								continue
+							}
+							tmp := map[string]interface{}{
+								"field":     v.Field,
+								"code":      v.Code + "去除__*后",
+								"ruletext":  regArr[0],
+								"extfrom":   extfrom,
+								"value":     value,
+								"type":      "regexp",
+								"matchtype": "regcontent",
+								"blocktag":  *tag,
+							}
+							tmps = append(tmps, tmp)
+							extinfo[v.Field] = tmps
+
+							exfield := ju.ExtField{BlockTag: *tag, Field: v.Field, Code: v.Code + "去除__*后", RuleText: v.RuleText, Type: "regexp", MatchType: "regcontent", ExtFrom: extfrom, SourceValue: text, Value: value}
+							if extfrom == "title" {
+								exfield.Score = 4
+							}
+							if tmp["blocktag"] != nil {
+								exfield.BlockTag = tmp["blocktag"].(map[string]bool)
+							}
+							item := ju.ScoreItem{Des: "初始化抽取规则去除__*", Code: v.Code + "去除__*后", RuleText: v.RuleText, Type: "regexp", MatchType: "regcontent", ExtFrom: extfrom, Value: value}
+							if extfrom == "title" {
+								item.Score = 4
+							}
+							if strings.Contains(value, "\n") {
+								item.Score -= 1
+								exfield.Score -= 1
+							}
+							if tmp["scoreitem"] == nil {
+								sitems := make([]*ju.ScoreItem, 0)
+								sitems = append(sitems, &item)
+								exfield.ScoreItem = sitems
+							} else {
+								exfield.ScoreItem = append(exfield.ScoreItem, &item)
+							}
+							j.Result[v.Field] = append(j.Result[v.Field], &exfield)
+							//j.Result[k] = append(j.Result[k], &ju.ExtField{tmp["blocktag"].(map[string]bool), k, v.Code, v.RuleText, "regexp", "regcontent", extfrom, val, 0})
+						}
+					}
+				}
+			}
 		}
 	} else {
 		pos := v.RegCore.Reg.FindStringIndex(text)
@@ -948,7 +1001,7 @@ func extRegCoreToResult(extfrom, text string, tag *map[string]bool, j *ju.Job, v
 			if j.Result[v.Field] == nil {
 				j.Result[v.Field] = [](*ju.ExtField){}
 			}
-			field := &ju.ExtField{BlockTag: *tag, Field: v.Field, Code: v.Code, RuleText: v.RuleText, Type: "regexp", MatchType: "regcontent", ExtFrom: extfrom, SourceValue: val, Value: val}
+			field := &ju.ExtField{BlockTag: *tag, Field: v.Field, Code: v.Code, RuleText: v.RuleText, Type: "regexp", MatchType: "regcontent", ExtFrom: extfrom, SourceValue: text, Value: val}
 			if extfrom == "title" {
 				field.Score = 4
 			}
@@ -1501,7 +1554,7 @@ func (e *ExtractTask) QualityAudit(resulttmp map[string]interface{}) {
 func (e *ExtractTask) RedisMatch(field, fv string, val map[string]interface{}) {
 	defer qu.Catch()
 	i := redis.GetInt(field, field+"_"+fv) //查找redis
-	if i == 0 {                            //reids未找到，执行规则匹配
+	if i == 0 { //reids未找到，执行规则匹配
 		val[field+"_isredis"] = false
 		e.RuleMatch(field, fv, val) //规则匹配
 	} else { //redis找到，打标识存库

src/jy/pretreated/analystep.go

@@ -30,7 +30,7 @@ func AnalyStart(job *util.Job) {
 		}
 	}
 	blockArrays, _ := DivideBlock(job.CategorySecond, con, 1, job.RuleBlock) //分块
-	if len(blockArrays) > 0 {                                                //有分块
+	if len(blockArrays) > 0 { //有分块
 		//从块里面找分包
 		job.BlockPackage = FindPackageFromBlocks(&blockArrays, job.Title) //从块里面找分包
 		for _, bl := range blockArrays {
@@ -41,15 +41,15 @@ func AnalyStart(job *util.Job) {
 			t1, _ := ComputeConRatio(bl.Text, 2)
 			if len(t1) > 0 {
 				job.HasTable = 1
-				for i:=0;i<len(tabs);i++{
+				for i := 0; i < len(tabs); i++ {
 					bl := &util.Block{}
 					//添加标识:文本中有table
 					tabres := AnalyTableV2(t1[0], job.Category, bl.Title, bl.Text, 2, job.SourceMid, job.RuleBlock) //解析表格入口 返回：汇总表格对象
-					processTableResult(tabres, bl, job)                                                          //分析table解析结果
+					processTableResult(tabres, bl, job)                                                             //分析table解析结果
 					if bl.Title == "" && tabres.BlockTag != "" {
 						bl.Title = tabres.BlockTag
 					}
-					if len(bl.TableKV.Kv)>0{
+					if len(bl.TableKV.Kv) > 0 {
 						bl.Text = tabs[i].Text()
 						job.Block = append(job.Block, bl)
 					}
@@ -71,15 +71,15 @@ func AnalyStart(job *util.Job) {
 			job.HasTable = 1 //添加标识:文本中有table
 			newCon = TextAfterRemoveTable(con)
 			job.BlockPackage = FindPackageFromText(job.Title, newCon)
-			for i:=0;i<len(tabs);i++{
+			for i := 0; i < len(tabs); i++ {
 				bl := &util.Block{}
 				//添加标识:文本中有table
-				tabres := AnalyTableV2(tabs[i], job.Category, "", con, 1, job.SourceMid, job.RuleBlock)//解析表格入口 返回：汇总表格对象
-				processTableResult(tabres, bl, job)                                                          //分析table解析结果
+				tabres := AnalyTableV2(tabs[i], job.Category, "", con, 1, job.SourceMid, job.RuleBlock) //解析表格入口 返回：汇总表格对象
+				processTableResult(tabres, bl, job)                                                     //分析table解析结果
 				if bl.Title == "" && tabres.BlockTag != "" {
 					bl.Title = tabres.BlockTag
 				}
-				if len(bl.TableKV.Kv) >0 {
+				if len(bl.TableKV.Kv) > 0 {
 					bl.Text = tabs[i].Text()
 					job.Block = append(job.Block, bl)
 				}
@@ -122,17 +122,19 @@ func FindProjectCode(newCon string, job *util.Job) {
 	var proCode string
 	proCode = projectcodeReg.FindString(newCon)
 	blCode := &util.Block{}
-	blCode.Text = proCode
 	if proCode != "" {
 		ckv := GetKVAll(proCode, job.Title, nil, 1)
 		blCode.ColonKV = ckv
+		blCode.Text = proCode
 		job.Block = append(job.Block, blCode)
 	} else if proCode = projectcodeReg2.FindString(newCon); proCode != "" {
 		ckv := GetKVAll(proCode, job.Title, nil, 1)
 		blCode.ColonKV = ckv
+		blCode.Text = proCode
 		job.Block = append(job.Block, blCode)
 	} else if proCode = projectcodeReg3.FindString(newCon); proCode != "" {
 		ckv := GetKVAll(proCode, job.Title, nil, 1)
+		blCode.Text = proCode
 		blCode.ColonKV = ckv
 		job.Block = append(job.Block, blCode)
 	}

src/jy/pretreated/analytable.go

@@ -1322,7 +1322,7 @@ func (table *Table) ComputeRowColIsKeyRation() {
 									if ((tdn != nil && v.Rationmap[v2] > fv) || tdn == nil) && td.Valtype != "BO" {
 										td.KeyDirect = 2
 										td.KVDirect = 1
-										//td.BH = true
+										td.BH = true
 									}
 								}
 							}
@@ -2322,9 +2322,10 @@ func (tn *Table) assemblePackage(k1, v1, key string) {
 					bp.TableKV.KvTag[k3] = &u.Tag{Value: v2, Weight: w1[pos]}
 				} else {
 					bp.TableKV.Kv[k1] = qutil.ObjToString(v1)
-					if tn.SortKV.Map[k3] == nil {
-						tn.SortKV.AddKey(k3, v2) //添加匹配到抽取关键词的key，value
-					}
+					//if tn.SortKV.Map[k3] == nil {
+					//	tn.SortKV.AddKey(k3, v2) //添加匹配到抽取关键词的key，value
+					//	tn.StandKVWeight[k3]=w1[pos]
+					//}
 				}
 			}
 		} else {

src/jy/pretreated/tablev2.go

@@ -232,9 +232,6 @@ func (td *TD) tdHasTable(bsontable *bool, tr *TR, table *Table) {
 					if td.TR.Table.TableResult == nil {
 						td.TR.Table.TableResult = NewTableResult(sonts.Id, sonts.Toptype, sonts.BlockTag, sonts.Html, sonts.Itype, sonts.RuleBlock)
 					}
-					if td.Val == "" {
-						td.SortKV.AddKey(k, v)
-					}
 					td.TR.Table.TableResult.SortKV.AddKey(k, v)
 					td.TR.Table.TableResult.SortKVWeight[k] = sonts.SortKVWeight[k]
 				}

src/main.go

@@ -9,16 +9,17 @@ import (
 	_ "jy/front"
 	. "jy/router"
 	"jy/util"
-	"log"
 	qu "qfw/util"
-	"qfw/util/elastic"
+	//"qfw/util/elastic"
 	redis "qfw/util/redis"
+
+	log "github.com/donnie4w/go-logger/logger"
 )
 
 func init() {
-	util.SetConsole(false)
-	util.SetLevel(util.DEBUG)
-	util.SetRollingDaily("./", "out.log")
+	log.SetConsole(false)
+	log.SetLevel(log.DEBUG)
+	log.SetRollingDaily("./", "out.log")
 	qu.ReadConfig(&util.Config)
 	qu.ReadConfig("./res/brandrule.json", &util.BrandRules)
 	qu.ReadConfig("./res/goods.json", &util.GoodsConfig)
@@ -32,7 +33,7 @@ func init() {
 	redis.InitRedisBySize(qu.ObjToString(util.Config["redis"]), 50, 30, 240)
 	//初始化elastic连接
 	//"winner=172.17.145.179:2710,buyer=172.17.145.179:2711"
-	elastic.InitElasticSize(qu.ObjToString(util.Config["elasticsearch"]), qu.IntAllDef(util.Config["elasticPoolSize"], 30))
+	//elastic.InitElasticSize(qu.ObjToString(util.Config["elasticsearch"]), qu.IntAllDef(util.Config["elasticPoolSize"], 30))
 }
 
 func main() {
@@ -40,7 +41,7 @@ func main() {
 	extract.ClearUdp()   //udp通知清理
 	go extract.Export()
 	go Router.Run(":" + qu.ObjToString(util.Config["port"]))
-	go log.Println("启动..", qu.ObjToString(util.Config["port"]))
+	go log.Debug("启动..", qu.ObjToString(util.Config["port"]))
 	lock := make(chan bool)
 	<-lock
 }

versioncomparison/config.json

@@ -11,6 +11,13 @@
         "buyer",
         "bidamount",
         "budget",
-        "winner"
+        "winner",
+        "agency",
+        "buyerperson",
+        "buyertel",
+        "buyeraddr",
+        "agencyperson",
+        "agencytel",5d39d253a5cb26b9b7404ae1,5d3b23aaa5cb26b9b7c1ec59
+        "agencyaddr"
     ]
 }