jy-data-extract/src/jy/pretreated/analystep.go

/**
信息预处理入口
**/
package pretreated

import (
	"fmt"
	"jy/util"
	qutil "qfw/util"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func AnalyStart(job *util.Job) {
	con := job.Content
	//全文的需要修复表格
	con = RepairCon(con)
	//格式化正文
	con = formatText(con, "all")
	job.Content = con
	//
	tabs, ration := ComputeConRatio(con, 1)
	if len(tabs) > 0 {
		newcon, newtabs, newration := findBigText(con, ration, tabs)
		if newcon != "" && newration == 0 {
			con = newcon
			tabs = newtabs
			ration = newration
		}
	}
	blockArrays, _ := DivideBlock(con, 1)
	if len(blockArrays) > 0 { //有分块
		//从块里面找分包
		job.BlockPackage = FindPackageFromBlocks(&blockArrays, job.Title)
		for _, bl := range blockArrays {
			if len([]rune(bl.Text)) > 80 {
				ba1, _ := DivideBlock(bl.Text, 1)
				if len(ba1) > 0 {
					t := ""
					for _, t1 := range ba1 {
						t += t1.Text
					}
					bl.Text = t
					bl.ColonKV = GetKVAll(t, bl.Title, 1)
					bl.SpaceKV = SspacekvEntity.Entrance(t, bl.Title)
				}
			}
			//块中再查找表格(块,处理完把值赋到块)
			t1, _ := ComputeConRatio(bl.Text, 2)
			if len(t1) > 0 {
				job.HasTable = 1 //添加标识:文本中有table
				fmt.Println("分块")
				tabres := AnalyTableV2(t1, job.Category, bl.Title, bl.Text, 2, job.SourceMid)
				processTableResult(tabres, bl, job)
				if bl.Title == "" && tabres.BlockTag != "" {
					bl.Title = tabres.BlockTag
				}
				//				for k, v := range bl.TableKV.Kv {
				//					log.Println("bl.TableKV.Kv", k, v)
				//				}
			}
			job.Block = append(job.Block, bl)
		}
	} else { //未分块,创建分块
		bl := &util.Block{}
		newCon := con
		if len(tabs) > 0 { //解析表格逻辑
			job.HasTable = 1 //添加标识:文本中有table
			newCon = TextAfterRemoveTable(con)
			job.BlockPackage = FindPackageFromText(job.Title, newCon)
			fmt.Println("未分块")
			tabres := AnalyTableV2(tabs, job.Category, "", con, 1, job.SourceMid)
			processTableResult(tabres, bl, job)
			//			for k, v := range bl.TableKV.Kv {
			//				log.Println("bl.TableKV.Kv", k, v)
			//			}
		} else {
			//从正文里面找分包
			job.BlockPackage = FindPackageFromText(job.Title, newCon)
		}
		//调用kv解析
		bl.ColonKV = GetKVAll(newCon, "", 1)
		bl.SpaceKV = SspacekvEntity.Entrance(newCon, "")
		job.Block = append(job.Block, bl)
	}
}

//分析table解析结果
func processTableResult(tabres *TableResult, block *util.Block, job *util.Job) {
	//解析结果中的kv
	fmt.Println("-------------", tabres.SortKV.Keys)
	kv := map[string]string{}
	for k, v := range tabres.SortKV.Map {
		kv[k] = qutil.ObjToString(v)
	}
	kvIndex := map[string]int{}
	for k, v := range tabres.SortKVWeight {
		kvIndex[k] = v
	}
	block.TableKV = &util.JobKv{Kv: kv, KvIndex: kvIndex}

	//分包
	tablePackage := map[string]*util.BlockPackage{}
	if tabres.IsMultiPackage {
		//分包中的map
		for k, v := range tabres.PackageMap.Map {
			blockPackage, ok := v.(*util.BlockPackage)
			if !ok {
				continue
			}
			//解析kv
			//找到key是“包1中标单位”这种的key，过滤掉包1，再次到标签库中匹配
			labelKVs := []*util.Kv{}
			if blockPackage.TableKV != nil && blockPackage.TableKV.Kv != nil {
				for tk, tv := range blockPackage.TableKV.Kv {
					if regReplKey.MatchString(tk) || regSplit.MatchString(tk) {
						labelKVs = append(labelKVs, &util.Kv{
							Key:   tk,
							Value: tv,
						})
					}
				}
			}
			labelKV, _ := KvTagsToKV(labelKVs, "", nil, 2)
			for lk, lv := range labelKV {
				if blockPackage.TableKV.Kv[lk] != "" {
					continue
				}
				blockPackage.TableKV.Kv[lk] = lv
			}
			tablePackage[k] = blockPackage
		}
	}
	//处理中标人排序
	wror := []map[string]interface{}{}
	for _, v := range tabres.WinnerOrder {
		entName, _ := v["entname"].(string)
		v["entname"] = winnerOrderEntity.clear("中标单位", entName)
		if price, ok := v["price"].(string); ok {
			v["price"] = winnerOrderEntity.clear("中标金额", price)
		}
		v["type"] = 2
		wror = append(wror, v)
	}
	if len(wror) > 0 {
		job.Winnerorder = wror
	}
	//分包
	if len(tablePackage) > 0 {
		pkgMap := map[string]*util.BlockPackage{}
		for tk, tv := range tablePackage {
			bv := job.BlockPackage[tk]
			if bv == nil {
				pkgMap[tk] = tv
				continue
			}
			bv.Text += "\n" + tv.Text
			/************table中的分包替换块里面找到的****************/
			//
			if tv.ColonKV != nil {
				if bv.ColonKV == nil {
					bv.ColonKV = util.NewJobKv()
				}
				for k, v := range tv.ColonKV.Kv {
					if bv.ColonKV.Kv[k] != "" {
						continue
					}
					bv.ColonKV.Kv[k] = v
				}
			}
			//
			if tv.TableKV != nil {
				if bv.TableKV == nil {
					bv.TableKV = util.NewJobKv()
				}
				for k, v := range tv.TableKV.Kv {
					if bv.TableKV.Kv[k] != "" {
						continue
					}
					bv.TableKV.Kv[k] = v
				}
			}
			//
			if tv.Origin != "" {
				bv.Origin = tv.Origin
			}
			//
			if tv.Index != "" {
				bv.Index = tv.Index
			}
			//
			if tv.Type != "" {
				bv.Type = tv.Type
			}
			//
			if tv.BidStatus != "" {
				bv.BidStatus = tv.BidStatus
			}
			//
			if tv.WinnerOrder != nil && len(tv.WinnerOrder) > 0 {
				bv.WinnerOrder = tv.WinnerOrder
			}
		}
		for k, v := range pkgMap {
			job.BlockPackage[k] = v
		}
	}
	//增加brand
	if tabres.HasKey != 0 {
		job.HasKey = tabres.HasKey
	}
	if tabres.HasBrand != 0 {
		job.HasBrand = tabres.HasBrand
	}
	if tabres.HasGoods != 0 {
		job.HasGoods = tabres.HasGoods
	}
	job.HasGoods = tabres.HasGoods

	if len(tabres.BrandData) > 0 { //分块table合并
		for _, v := range tabres.BrandData {
			job.BrandData = append(job.BrandData, v) //加入job
		}
	}
}

//一行多列 一列多行，按照分块逻辑处理
//ration==1 遍历所有tabs，ration!=1 tabs只有一个
func tableDivideBlock(con string, ration float32, tabs []*goquery.Selection) string {
	if len(tabs) != 1 {
		return ""
	}
	for _, tab := range tabs {
		content := ""
		tbody := tab.ChildrenFiltered("tbody,thead")
		var tr *goquery.Selection
		if tbody.Length() == 1 {
			tr = tbody.ChildrenFiltered("tr")
		} else {
			tr = tab.ChildrenFiltered("tr")
		}
		if tr.Length() == 1 {
			tds := tr.ChildrenFiltered("td")
			tds.Each(func(index int, sn *goquery.Selection) {
				ret, _ := sn.Html()
				if strings.TrimSpace(ret) != "" {
					content += ret + "\n"
				}
			})
		} else {
			flag := true
			tr.EachWithBreak(func(index int, sn *goquery.Selection) bool {
				th := sn.ChildrenFiltered("th")
				td := sn.ChildrenFiltered("td")
				if th.Length() > 0 || td.Length() > 1 {
					flag = false
					return false
				} else if td.Length() == 1 {
					ret, _ := td.Html()
					if strings.TrimSpace(ret) != "" {
						content += ret + "\n"
					}
				}
				return true
			})
			if !flag {
				return ""
			}
		}
		if content != "" {
			content = regMoreWrap.ReplaceAllString(content, "\n")
			content = regEndWrap.ReplaceAllString(content, "")
			doc, _ := goquery.NewDocumentFromReader(strings.NewReader(con))
			doc.Find("table").Eq(0).ReplaceWithHtml(content)
			con, _ = doc.Html()
		}
	}
	return con
}

//查找大文本，5次
func findBigText(con string, r float32, t []*goquery.Selection) (content string, tabs []*goquery.Selection, ration float32) {
	content = tableDivideBlock(con, r, t)
	if content == "" {
		return
	}
	for i := 0; i < 4; i++ {
		if content != "" {
			tabs, ration = ComputeConRatio(content, 1)
			if len(tabs) > 0 {
				content = tableDivideBlock(content, ration, tabs)
				if content == "" {
					return
				}
			} else {
				doc, _ := goquery.NewDocumentFromReader(strings.NewReader(con))
				content = doc.Text()
				return
			}
		} else {
			return
		}
	}
	return
}