分包优化

src/jy/extract/extpackage.go

@@ -52,9 +52,6 @@ func kvparse(p *ju.JobKv, e *ExtractTask, sonJobResult *map[string]interface{},
 			}
 			tags := ju.GetTags(pk, isSite, codeSite)
 			if tags.Len() > 0 {
-				if ((*sonJobResult)["name"] == nil || (*sonJobResult)["name"] == "") && tags[0].Key == "项目名称" {
-					(*sonJobResult)["name"] = pv[0].Value
-				}
 				if qu.Float64All((*sonJobResult)["budget"]) == 0 && tags[0].Key == "预算" {
 					lock.Lock()
 					cfn := e.ClearFn["budget"]
@@ -81,18 +78,24 @@ func kvparse(p *ju.JobKv, e *ExtractTask, sonJobResult *map[string]interface{},
 					if winnerorderNotReg.MatchString(pv[0].Value) {
 						continue
 					}
-					(*sonJobResult)["winner"] = pv[0].Value
+					for _, ev := range e.PkgRuleCores {
+						for _, evb := range ev.RuleBacks {
+							if evb.Field == "winner" && !evb.IsLua && pv[0].Value != "" {
+								pv[0].Value = evb.RegPreBac.Reg.ReplaceAllString(pv[0].Value, evb.RegPreBac.Replace)
+							}
+						}
+					}
+					if pv[0].Value != "" {
+						(*sonJobResult)["winner"] = pv[0].Value
+					}
 				}
-
-			}
-			if (*sonJobResult)["name"] == nil && pk == "名称" {
-				(*sonJobResult)["name"] = pv[0].Value
 			}
 		}
 	}
 }
 
 var winnerorderNotReg = regexp.MustCompile(`(附件|否决原因|候选|招标失败|注册表|交易中心|序号内容|不足|公告|变更|采购|招标|废标|废止|流标|中标|投标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\d[\s]{0,10}(\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})`)
+
 //处理分包信息
 func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 	qu.Try(func() {
@@ -110,7 +113,7 @@ func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 				}
 			}
 			ordertmp := map[int][]map[string]interface{}{}
-			ordertmpint := []int{}//中标排序人一共几组
+			ordertmpint := []int{} //中标排序人一共几组
 			//中标候选人
 			for _, v := range j.Winnerorder {
 				if vtype, ok := v["type"].(int); ok {
@@ -134,7 +137,7 @@ func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 			sort.Strings(tmpkeys)
 			packageResult := map[string]map[string]interface{}{}
 			//packagenum := len(j.BlockPackage)
-			tmpindex :=-1
+			tmpindex := -1
 			for _, pkName := range tmpkeys {
 				tmpindex++
 				pkg, ok := j.BlockPackage[pkName]
@@ -148,27 +151,27 @@ func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 					sonJobResult["origin"] = pkg.Origin
 					sonJobResult["text"] = pkg.Text
 					sonJobResult["name"] = pkg.Name
-					if pkg.Winner!= ""{
+					if pkg.Winner != "" {
 						sonJobResult["winner"] = pkg.Winner
 					}
 					if pkg.WinnerPerson != "" {
 						sonJobResult["winnertel"] = pkg.WinnerTel
 						sonJobResult["winnerperson"] = pkg.WinnerPerson
 					}
-					if pkg.IsTrueBudget{
+					if pkg.IsTrueBudget {
 						sonJobResult["budget"] = pkg.Budget
 					}
 					if pkg.IsTrueBidamount {
 						sonJobResult["bidamount"] = pkg.Bidamount
 					}
 					if pkg.Winner == "" && len(j.Winnerorder) > 0 {
-						if sonJobResult["winnerorder"] == nil && len(tmpkeys) == len(ordertmpint) {//分包和中标候选人长度一样
+						if sonJobResult["winnerorder"] == nil && len(tmpkeys) == len(ordertmpint) { //分包和中标候选人长度一样
 							//ordertmp[ordertmpint[tmpindex]] 取中标候选人组
 							sonJobResult["winnerorder"] = ordertmp[ordertmpint[tmpindex]]
 							if sonJobResult["bidamount"] == nil || sonJobResult["bidamount"].(float64) <= 0 {
 								if j.Winnerorder[0]["price"] != nil {
-									moneys := clear.ObjToMoney([]interface{}{j.Winnerorder[0]["price"],""})
-									if len(moneys) > 0 &&moneys[len(moneys)-1].(bool){
+									moneys := clear.ObjToMoney([]interface{}{j.Winnerorder[0]["price"], ""})
+									if len(moneys) > 0 && moneys[len(moneys)-1].(bool) {
 										if vf, ok := moneys[0].(float64); ok {
 											sonJobResult["bidamount"] = vf
 										}

src/jy/extract/extract.go

@@ -460,17 +460,6 @@ func (e *ExtractTask) ExtractProcess(j, jf *ju.Job, isSite bool) {
 		}
 	}
 
-	//	for _, b := range j.Block {
-	//		for _, vv := range b.ColonKV.Kvs {
-	//			qu.Debug(vv.Key, vv.Value)
-	//		}
-	//		for _, vv := range b.SpaceKV.Kvs {
-	//			qu.Debug(vv.Key, vv.Value)
-	//		}
-	//		//		for _, vv := range b.TableKV.Kvs {
-	//		//			qu.Debug(vv.Key, vv.Value)
-	//		//		}
-	//	}
 	//分析抽取结果并保存
 	AnalysisSaveResult(j, jf, e)
 	<-e.TaskInfo.ProcessPool

src/jy/pretreated/analytable.go

@@ -49,8 +49,9 @@ var (
 	FindVal_1  = regexp.MustCompile("[第]?([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)((子|合同|分|施工|监理)?(标段?|包|合同段|标包))|((子|合同|分|施工|监理)?(标|包)(段|号)?)[ 　\u3000\u2003\u00a0]*([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)")
 	FindVal2_1 = regexp.MustCompile("([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+)|^(设计|施工|监理|验收)[分子]?[标包]?[段号]?$")
 	//判断分包前排除
-	excludeKey  = regexp.MustCompile("(标段代码|涉及包号|分包数量|包件号?|项目标号|规格|型号|招标范围|业绩|废标|标段选择要求)|(^编号$)|([^包段标]编号)") //编号|划分
+	excludeKey  = regexp.MustCompile("(分包个数|标段代码|涉及包号|分包数量|包件号?|项目标号|规格|型号|招标范围|业绩|废标|标段选择要求)|(^编号$)|([^包段标]编号)") //编号|划分
 	excludeKey2 = regexp.MustCompile("包/[0-9]{0,4}[|箱|纸|张]")
+	excludeKey3 = regexp.MustCompile("(分包个数|每包[0-9]*元|标线|国标)")
 	//-------------
 
 	cut = u.NewCut()
@@ -2167,7 +2168,7 @@ func (tn *Table) manyPackageProcessByIndex(index []string, standIndex_pos []int,
 							}
 						}
 					}
-					if k1 == "单价"{
+					if k1 == "单价" {
 						continue
 					}
 					tn.assemblePackage(k1, v, sv2, isSite, codeSite)
@@ -2254,7 +2255,7 @@ func (tn *Table) manyPackageProcessByIndex(index []string, standIndex_pos []int,
 func (tn *Table) isGoonNext(isSite bool, codeSite string) {
 	blockPackage := map[string]*u.BlockPackage{}
 	for _, k := range tn.SortKV.Keys {
-		if excludeKey.MatchString(k) || strings.Contains(k, "批复") {
+		if excludeKey.MatchString(k) || strings.Contains(k, "批复") || excludeKey3.MatchString(k) {
 			continue
 		}
 		str := "" //拼装为冒号kv
@@ -2324,7 +2325,7 @@ func (tn *Table) isGoonNext(isSite bool, codeSite string) {
 func foundPacBySortKV(tn *Table, val int, index []string, index_pos []int, keyExistsCount *map[string]int, commonKeyVals *map[string][]string, key_index int, hasPkgTd map[string]bool) (rval int, rindex []string, rindex_pos []int) {
 	keyIsPkg := false
 	for in, k := range tn.SortKV.Keys {
-		if excludeKey.MatchString(BracketsTextReg.ReplaceAllString(k, "")) ||regPDFWarap.MatchString(k)||regAZWarap.MatchString(k){ //判断分包前排除
+		if excludeKey.MatchString(BracketsTextReg.ReplaceAllString(k, "")) || excludeKey3.MatchString(k) || regFJWarap.MatchString(k) || regAZWarap.MatchString(k) { //判断分包前排除
 			continue
 		}
 		v := tn.SortKV.Map[k]
@@ -2406,7 +2407,7 @@ func foundPacBySortKV(tn *Table, val int, index []string, index_pos []int, keyEx
 func initCheckMultiPackageByTable(tn *Table, key_index int, index []string, index_pos []int, val int, pac int, hasPkgTd map[string]bool) (rkey_index int, rindex []string, rindex_pos []int, rval int, rpac int, rhasPkgTd map[string]bool) {
 	for in, k := range tn.SortKV.Keys {
 		//涉及包号|包件号?|项目标号|规格|型号|招标范围|业绩|废标)|(^编号$)|([^包段标]编号)就跳过
-		if excludeKey.MatchString(BracketsTextReg.ReplaceAllString(k, "")) || strings.Contains(k, "批复") {
+		if excludeKey.MatchString(BracketsTextReg.ReplaceAllString(k, "")) || excludeKey3.MatchString(k) || strings.Contains(k, "批复") {
 			continue
 		}
 		v := tn.SortKV.Map[k]
@@ -3421,7 +3422,7 @@ func initLineMapLineMapArr(table *Table) (lineMapArr map[string]*SortMap, lineMa
 	for _, key := range table.SortKV.Keys { //遍历table.SortKV.Keys而不是直接遍历table.SortKV.Map是为了得到table头的顺序
 		val := table.SortKV.Map[key]
 		key = regReplAllSpace.ReplaceAllString(key, "")
-		key = strings.Replace(key, "", "", -1) //处理一个特殊的采购量 经上层处理空格后未处理掉
+		key = strings.Replace(key, "", "", -1)    //处理一个特殊的采购量 经上层处理空格后未处理掉
 		if realTypeVal, ok := val.([]string); ok { //val为数组 {"数量":["1","2","3"]}
 			/*
 				{

src/jy/pretreated/division.go

@@ -47,26 +47,26 @@ var (
 	regTrimSpace       = regexp.MustCompile("^[\u3000\u2003\u00a0\\s]+|[\u3000\u2003\u00a0\\s]+$")
 	regReplWrapSpace   = regexp.MustCompile("^[\r\n][\u3000\u2003\u00a0\\s]*|[\r\n][\u3000\u2003\u00a0\\s]*$")
 	regReplAllSymbol   = regexp.MustCompile("[（\\(<《【\\[{｛〔）\\)>》】\\]}｝〕,，;；:：'\"“”。.\\?？/+=\\-_——*&……\\^%$￥@#!！`~·]")
-	regFilterTitle     = regexp.MustCompile("[（\\(<《【\\[{｛〔].+?[）\\)>》】\\]}｝〕]")
-	regDivision        = regexp.MustCompile("[:：]")
-	regSpliteSegment   = regexp.MustCompile("[\r\n]")
-	regFilterNumber    = regexp.MustCompile("^[\\d一二三四五六七八九十]+")
-	regSplit           = regexp.MustCompile("或|和|以?及|与|、|或")
-	regStartWrap       = regexp.MustCompile("^[\r\n]")
-	regEndWrap         = regexp.MustCompile("[\r\n]$")
-	regMoreWrap        = regexp.MustCompile("[\r\n]{2,}")
-	regStrWrap         = regexp.MustCompile("分包名称[：:]")
-	regBZJWarap        = regexp.MustCompile("保证金.*")
-	regPDFWarap        = regexp.MustCompile("[a-zA-Z](包|标段).(pdf|PDF)")
-	regAZWarap         = regexp.MustCompile("(标[a-zA-Z]取值|标段划分|标液)")
-	replSerial         = regexp.MustCompile("(\r\n|^)([\\d一二三四五六七八九十][、．.:：，])+\\d")
-	moreColonReg       = regexp.MustCompile("[:：]+")
-	regFilter          = regexp.MustCompile("等$")
-	pkgFilter          = regexp.MustCompile("第[一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ](子|合同|分|施工|监理)?(标段?|包|合同段|标包)|(子|合同|分|施工|监理)?(标|包)(段|号)?")
-	indexTile          = regexp.MustCompile("[0-9.]{2,3}[\\s\u4e00-\u9fa5]{2,8}[:：]+") //小标题
-	indexTile2         = regexp.MustCompile("[\\s\u4e00-\u9fa5]{2,8}")
-	regReplAllSpace2   = regexp.MustCompile("[\u3000\u2003\u00a0\\s0-9.:：、\\(\\)]+")
-	confusion          = map[string]string{
+	regFilterTitle   = regexp.MustCompile("[（\\(<《【\\[{｛〔].+?[）\\)>》】\\]}｝〕]")
+	regDivision      = regexp.MustCompile("[:：]")
+	regSpliteSegment = regexp.MustCompile("[\r\n]")
+	regFilterNumber  = regexp.MustCompile("^[\\d一二三四五六七八九十]+")
+	regSplit         = regexp.MustCompile("或|和|以?及|与|、|或")
+	regStartWrap     = regexp.MustCompile("^[\r\n]")
+	regEndWrap       = regexp.MustCompile("[\r\n]$")
+	regMoreWrap      = regexp.MustCompile("[\r\n]{2,}")
+	regStrWrap       = regexp.MustCompile("分包名称[：:]")
+	regBZJWarap      = regexp.MustCompile("(保证金.*|每包[0-9]*元|标志|一包一投|上包|标线|国标|第[\\d一二三四五六七八九十]标室)")
+	regFJWarap       = regexp.MustCompile("[a-zA-Z0-9](包|标段)[公告附件]*.(pdf|PDF|docx|doc|DOCX|DOC)")
+	regAZWarap       = regexp.MustCompile("(标[a-zA-Z]取值|标段划分|标液|分包个数|物资[\\d一二三四五六七八九十]?包)")
+	replSerial       = regexp.MustCompile("(\r\n|^)([\\d一二三四五六七八九十][、．.:：，])+\\d")
+	moreColonReg     = regexp.MustCompile("[:：]+")
+	regFilter        = regexp.MustCompile("等$")
+	pkgFilter        = regexp.MustCompile("第[一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ](子|合同|分|施工|监理)?(标段?|包|合同段|标包)|(子|合同|分|施工|监理)?(标|包)(段|号)?")
+	indexTile        = regexp.MustCompile("[0-9.]{2,3}[\\s\u4e00-\u9fa5]{2,8}[:：]+") //小标题
+	indexTile2       = regexp.MustCompile("[\\s\u4e00-\u9fa5]{2,8}")
+	regReplAllSpace2 = regexp.MustCompile("[\u3000\u2003\u00a0\\s0-9.:：、\\(\\)]+")
+	confusion        = map[string]string{
 		"参与": "canyu",
 	}
 	//查找分包之前，先对内容进行预处理
@@ -653,7 +653,7 @@ func FindPackageFromText(title string, content string, isSite bool, codeSite str
 //分块之后分包
 func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content, title string, isFindWinnerOrder, accuracy bool, isSite bool, codeSite string) (bool, string) {
 	//查找知否有分包
-	content = regPDFWarap.ReplaceAllString(content, "\n")
+	content = regFJWarap.ReplaceAllString(content, "\n")
 	content = regAZWarap.ReplaceAllString(content, "\n")
 	content = regStrWrap.ReplaceAllString(content, "\n")
 	content = regMoreWrap.ReplaceAllString(content, "\n")
@@ -777,8 +777,9 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 			index := util.PackageNumberConvert(bk)
 			//去掉前缀,空格必须要加，分kv的时候要用
 			text = regexp.MustCompile(bv[0]+"[:：]*").ReplaceAllString(text, "")
-			if strings.TrimLeft(tmptext, bv[0]) == text || strings.TrimLeft(tmptext, bv[0]+":") == text {
+			if strings.TrimLeft(tmptext, bv[0]) == text || strings.TrimLeft(tmptext, bv[0]+":") == text || strings.TrimLeft(tmptext, bv[0]+"：") == text {
 				var tagtitle string
+
 				for i, v := range titleindexs {
 					if i == 0 {
 						continue
@@ -789,6 +790,11 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 					}
 				}
 				tagtitle = regReplAllSpace2.ReplaceAllString(tagtitle, "")
+				if tagtitle == "" {
+					tagtitle = title
+				} else  if strings.Contains(tagtitle, bv[0]) && title != "" {
+					tagtitle = title
+				}
 				text = tagtitle + ":" + text
 			}
 			headKey := ""

src/res/tablev1.json

@@ -1,10 +1,10 @@
 {
 	"normalhead":[
-		"^((.{2,6}(描述|名称|编号|代码|时间|类型|性质|行政区域|原因|意见|须知|程度))|标段(编号)?|招标金额|规模|统一社会信用代码|拟?中标供应商|质量|(质量)?承诺|地址|招标代理|序号|材料|结构|结构层数|评委|单位|数量|排名|标的|标项|开户银行|邮编|账号|电话|传真|网址|得分|名次|包件?号|职务|(建设|招标|采购|中标|成交|甲|乙)(单位|人|供应商|方|规模).{0,2}|.{0,5}(价格?|额|资金|[预概]算|投资|费用|报价|投标价)(万?元?([大小]写)?))$__M",
+		"^((.{2,6}(描述|名称|编号|代码|时间|类型|性质|行政区域|原因|意见|须知|程度))|标段(编号)?|招标金额|规模|统一社会信用代码|拟?中标供应商|质量|(质量)?承诺|地址|招标代理|序号|材料|结构|结构层数|评委|单位|数量|排名|标的|标项|开户银行|邮编|账号|电话|传真|网址|得分|名次|包件?号|职务|(建设|招标|采购|中标|成交|甲|乙)(单位|人|供应商|方|规模).{0,2}|.{0,5}(价格?|额|资金|[预概]算|投资|费用|报价|投标价)[（]?(万?元?([大小]写)?)[）]?)$__M",
 		"^.{0,7}(((单位)?名称|总监|经理|负责人|信息|率|费|期|人|号|码|(价格?|额|资金)(万?元?([大小]写)?)|员|品目|标包|代表|区域|方式|因素|合价|合计|小计|地点|条件|(资质|类别和)等级|类别|状态)|得分|注册专业|方法|家数|全称|简称|邮件|执业或职业资格|证书|部门|事项|来源|划分|长度|规模|保证金|目标)$__",
 		"(名单|证号|名称|要求|时间|日期|地点|单位|条款|机构|范围|情况|概况|品名|规格|参数|标准|指标|型号|限价|数量|方式|等级|依据|明细|概况|内容|次数|产品|性质|地区|地址|币种|主题|详情|说明|代理(公司|机构)|节支率|名单|结果|结果公示)$|^(职称|姓名|级别|职称专业|证书名称|证书编号)$__",
-		"^(联系|评标|单位|公告|采购|商品|附件|质保|用途|公示|机构|评审|品名|规格|参数|指标|型号|数量|证书).{0,10}$__",
-		"(专家|评委|打分)$__",
+		"^(包号|联系|评标|单位|公告|采购|商品|附件|质保|用途|公示|机构|评审|品名|规格|参数|指标|型号|数量|证书).{0,10}$__",
+		"(专家|评委|打分|附件材料)$__",
 		"品牌",
         "姓名",
 		"起讫桩号",
@@ -23,7 +23,9 @@
 		"原因",
 		"行业",
 		"价格",
-		"注册资金"
+		"注册资金",
+		"[\\d]+标段$__M",
+		"(\\W{2,10}(名称|参数[及]?要求))$__M"
 	],
 	"jghead":[
 		"^.{0,2}[预拟]?(成交|中标|候选)(供应商|单位|企业|人|机构|价|金额).{0,2}$__M",