Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

src/jy/admin/rule.go

@@ -70,20 +70,20 @@ func init() {
 	Admin.POST("/rulelogic/use", RuleLogicUse)
 
 	//pkglogic
-	Admin.GET("/pkglogic", func(c *gin.Context) {
-		vid := c.Query("vid")
-		pid := c.Query("pid")
-		c.HTML(200, "pkg_logiclist.html", gin.H{"vid": vid, "pid": pid})
-	})
-	Admin.POST("/pkglogic/data", PkgLogicData)
-	Admin.POST("/pkglogic/save", PkgLogicSave)
-	Admin.POST("/pkgrulelogic/del", func(c *gin.Context) {
-		_id, _ := c.GetPostForm("_id")
-		b := Mgo.UpdateById("pkg_logic", _id, `{"$set":{"delete":true}}`)
-		go DelPkgLogicBack(_id)
-		c.JSON(200, gin.H{"rep": b})
-	})
-	Admin.POST("/pkglogic/use", PkgLogicUse)
+	//Admin.GET("/pkglogic", func(c *gin.Context) {
+	//	vid := c.Query("vid")
+	//	pid := c.Query("pid")
+	//	c.HTML(200, "pkg_logiclist.html", gin.H{"vid": vid, "pid": pid})
+	//})
+	//Admin.POST("/pkglogic/data", PkgLogicData)
+	//Admin.POST("/pkglogic/save", PkgLogicSave)
+	//Admin.POST("/pkgrulelogic/del", func(c *gin.Context) {
+	//	_id, _ := c.GetPostForm("_id")
+	//	b := Mgo.UpdateById("pkg_logic", _id, `{"$set":{"delete":true}}`)
+	//	go DelPkgLogicBack(_id)
+	//	c.JSON(200, gin.H{"rep": b})
+	//})
+	//Admin.POST("/pkglogic/use", PkgLogicUse)
 
 	Admin.GET("/logicpre", func(c *gin.Context) {
 		vid := c.Query("vid")
@@ -119,13 +119,26 @@ func init() {
 	Admin.POST("/logicore/del", RuleLogicCoreDel)
 	Admin.POST("/logicore/use", RuleLogicCoreUse)
 
-	Admin.GET("/pkglogicback", func(c *gin.Context) {
+	Admin.GET("/logickv", func(c *gin.Context) {
 		vid := c.Query("vid")
 		pid := c.Query("pid")
 		sid := c.Query("sid")
 		//查询属性字段
-		tmp, _ := Mgo.FindById("pkg_info", pid, `{"s_field":1}`)
-		c.HTML(200, "pkg_logicbacklist.html", gin.H{"vid": vid, "pid": pid, "sid": sid, "field": (*tmp)["s_field"]})
+		tmp, _ := Mgo.FindById("versioninfo", pid, `{"s_field":1}`)
+		c.HTML(200, "rule_logickv.html", gin.H{"vid": vid, "pid": pid, "sid": sid, "field": (*tmp)["s_field"]})
+	})
+	Admin.POST("/logickv/data", RuleLogicCoreDataKv)
+	Admin.POST("/logickv/save", RuleLogicCoreSaveKv)
+	Admin.POST("/logickv/del", RuleLogicCoreDelKv)
+	Admin.POST("/logickv/use", RuleLogicCoreUseKv)
+
+	Admin.GET("/pkglogicback", func(c *gin.Context) {
+		vid := c.Query("vid")
+		s_field := c.Query("s_field")
+		sid := c.Query("sid")
+		//查询属性字段
+		//tmp, _ := Mgo.FindById("pkg_info", pid, `{"s_field":1}`)
+		c.HTML(200, "pkg_logicbacklist.html", gin.H{"vid": vid, "sid": sid, "field": s_field,"s_field":s_field})
 	})
 	Admin.POST("/pkglogicback/data", PkgLogicBackData)
 	Admin.POST("/pkglogicback/save", PkgLogicBackSave)
@@ -366,10 +379,10 @@ func RuleLogicBackData(c *gin.Context) {
 }
 func PkgLogicBackData(c *gin.Context) {
 	vid, _ := c.GetPostForm("vid")
-	pid, _ := c.GetPostForm("pid")
+	s_field, _ := c.GetPostForm("s_field")
 	sid, _ := c.GetPostForm("sid")
 	data, _ := Mgo.Find("pkg_logicback", `{"sid":"`+sid+`","delete":false}`, `{"_id":-1}`, nil, false, -1, -1)
-	c.JSON(200, gin.H{"data": data, "vid": vid, "pid": pid, "sid": sid})
+	c.JSON(200, gin.H{"data": data, "vid": vid, "s_field": s_field, "sid": sid})
 }
 func RuleLogicBackSave(c *gin.Context) {
 	data := GetPostForm(c)
@@ -439,6 +452,13 @@ func RuleLogicCoreData(c *gin.Context) {
 	data, _ := Mgo.Find("rule_logicore", `{"sid":"`+sid+`","delete":false}`, `{"_id":-1}`, nil, false, -1, -1)
 	c.JSON(200, gin.H{"data": data, "vid": vid, "pid": pid, "sid": sid})
 }
+func RuleLogicCoreDataKv(c *gin.Context) {
+	vid, _ := c.GetPostForm("vid")
+	pid, _ := c.GetPostForm("pid")
+	sid, _ := c.GetPostForm("sid")
+	data, _ := Mgo.Find("rule_logickv", `{"sid":"`+sid+`","delete":false}`, `{"_id":-1}`, nil, false, -1, -1)
+	c.JSON(200, gin.H{"data": data, "vid": vid, "pid": pid, "sid": sid})
+}
 func RuleLogicCoreSave(c *gin.Context) {
 	data := GetPostForm(c)
 	_id, _ := c.GetPostForm("_id")
@@ -464,18 +484,55 @@ func RuleLogicCoreSave(c *gin.Context) {
 	}
 	c.JSON(200, gin.H{"rep": b})
 }
+func RuleLogicCoreSaveKv(c *gin.Context) {
+	data := GetPostForm(c)
+	_id, _ := c.GetPostForm("_id")
+	b := false
+	if _id == "" {
+		data["l_createtime"] = time.Now().Unix()
+		data["s_username"] = sessions.Default(c).Get("username")
+		data["delete"] = false
+		s_type := data["s_type"]
+		code := ""
+		if s_type == "0" { //抽取逻辑正则
+			code = util.GetSyncIndex(E_CE)
+		} else { //抽取逻辑脚本
+			code = util.GetSyncIndex(E_CL)
+		}
+		data["s_code"] = code
+		b = Mgo.Save("rule_logickv", data) != ""
+	} else {
+		data["l_lasttime"] = time.Now().Unix()
+		b = Mgo.Update("rule_logickv", `{"_id":"`+_id+`"}`, map[string]interface{}{
+			"$set": data,
+		}, true, false)
+	}
+	c.JSON(200, gin.H{"rep": b})
+}
 func RuleLogicCoreDel(c *gin.Context) {
 	_id, _ := c.GetPostForm("_id")
 	//b := Mgo.Del("rule_logicore", `{"_id":"`+_id+`"}`)
 	b := Mgo.UpdateById("rule_logicore", _id, `{"$set":{"delete":true}}`)
 	c.JSON(200, gin.H{"rep": b})
 }
+func RuleLogicCoreDelKv(c *gin.Context) {
+	_id, _ := c.GetPostForm("_id")
+	//b := Mgo.Del("rule_logicore", `{"_id":"`+_id+`"}`)
+	b := Mgo.UpdateById("rule_logickv", _id, `{"$set":{"delete":true}}`)
+	c.JSON(200, gin.H{"rep": b})
+}
 func RuleLogicCoreUse(c *gin.Context) {
 	_id, _ := c.GetPostForm("_id")
 	isuse, _ := c.GetPostForm("isuse")
 	b := Mgo.UpdateById("rule_logicore", _id, `{"$set":{"isuse":`+isuse+`}}`)
 	c.JSON(200, gin.H{"rep": b})
 }
+func RuleLogicCoreUseKv(c *gin.Context) {
+	_id, _ := c.GetPostForm("_id")
+	isuse, _ := c.GetPostForm("isuse")
+	b := Mgo.UpdateById("rule_logickv", _id, `{"$set":{"isuse":`+isuse+`}}`)
+	c.JSON(200, gin.H{"rep": b})
+}
 
 func DelVersionInfo(vid string, isSite bool) { //刪除属性配置
 	var versioninfodb string

src/jy/admin/rulecheck.go

@@ -324,6 +324,7 @@ func checkBackScript(table, code, name, version, infoid, script string, alone bo
 	e.TaskInfo = &extract.TaskInfo{Version: qu.ObjToString((*vsion)["s_version"]), VersionId: qu.BsonIdToSId((*vsion)["_id"]), TestLua: true}
 	e.InitSite()
 	e.InitRulePres()
+	e.InitPkgCore()
 	e.InitRuleBacks(false)
 	e.InitRuleBacks(true)
 	e.InitRuleCore(false)

src/jy/admin/site_management.go

@@ -91,6 +91,7 @@ func init() {
 				go Mgo.Del("site_rule_logic", bson.M{"pid": (*vinfotmp)["_id"].(bson.ObjectId).Hex(),"vid":util.ObjToString((*vstmp)["vid"])})
 				go Mgo.Del("site_rule_logicback", bson.M{"pid": (*vinfotmp)["_id"].(bson.ObjectId).Hex(),"vid":util.ObjToString((*vstmp)["vid"])})
 				go Mgo.Del("site_rule_logicore", bson.M{"pid": (*vinfotmp)["_id"].(bson.ObjectId).Hex(),"vid":util.ObjToString((*vstmp)["vid"])})
+				go Mgo.Del("site_rule_logickv", bson.M{"pid": (*vinfotmp)["_id"].(bson.ObjectId).Hex(),"vid":util.ObjToString((*vstmp)["vid"])})
 				go Mgo.Del("site_rule_logicpre", bson.M{"pid": (*vinfotmp)["_id"].(bson.ObjectId).Hex(),"vid":util.ObjToString((*vstmp)["vid"])})
 			}
 			go Mgo.Del("site_versioninfo", bson.M{"pid": _id,"vid":util.ObjToString((*vstmp)["vid"])})
@@ -282,6 +283,15 @@ func init() {
 		tmp, _ := Mgo.FindById("site_versioninfo", pid, `{"s_field":1}`)
 		c.HTML(200, "site_rule_logicore.html", gin.H{"vid": vid, "pid": pid, "sid": sid, "field": (*tmp)["s_field"], "pzid": (*verinfo)["pid"]})
 	})
+	Admin.GET("/site_management/logickv", func(c *gin.Context) {
+		vid := c.Query("vid")
+		pid := c.Query("pid")
+		sid := c.Query("sid")
+		verinfo, _ := Mgo.FindById("site_versioninfo", pid, bson.M{"pid": 1})
+		//查询属性字段
+		tmp, _ := Mgo.FindById("site_versioninfo", pid, `{"s_field":1}`)
+		c.HTML(200, "site_rule_logickv.html", gin.H{"vid": vid, "pid": pid, "sid": sid, "field": (*tmp)["s_field"], "pzid": (*verinfo)["pid"]})
+	})
 	//抽取规则
 	Admin.POST("/site_management/logicore/data", func(c *gin.Context) {
 		vid, _ := c.GetPostForm("vid")
@@ -290,6 +300,14 @@ func init() {
 		data, _ := Mgo.Find("site_rule_logicore", `{"sid":"`+sid+`","delete":false}`, `{"_id":-1}`, nil, false, -1, -1)
 		c.JSON(200, gin.H{"data": data, "vid": vid, "pid": pid, "sid": sid})
 	})
+	//抽取规则
+	Admin.POST("/site_management/logickv/data", func(c *gin.Context) {
+		vid, _ := c.GetPostForm("vid")
+		pid, _ := c.GetPostForm("pid")
+		sid, _ := c.GetPostForm("sid")
+		data, _ := Mgo.Find("site_rule_logickv", `{"sid":"`+sid+`","delete":false}`, `{"_id":-1}`, nil, false, -1, -1)
+		c.JSON(200, gin.H{"data": data, "vid": vid, "pid": pid, "sid": sid})
+	})
 	Admin.POST("/site_management/logicore/save", func(c *gin.Context) {
 		data := GetPostForm(c)
 		_id, _ := c.GetPostForm("_id")
@@ -315,18 +333,55 @@ func init() {
 		}
 		c.JSON(200, gin.H{"rep": b})
 	})
+	Admin.POST("/site_management/logickv/save", func(c *gin.Context) {
+		data := GetPostForm(c)
+		_id, _ := c.GetPostForm("_id")
+		b := false
+		if _id == "" {
+			data["l_createtime"] = time.Now().Unix()
+			data["s_username"] = sessions.Default(c).Get("username")
+			data["delete"] = false
+			s_type := data["s_type"]
+			code := ""
+			if s_type == "0" { //抽取逻辑正则
+				code = getSyncIndex(E_CE)
+			} else { //抽取逻辑脚本
+				code = getSyncIndex(E_CL)
+			}
+			data["s_code"] = code
+			b = Mgo.Save("site_rule_logickv", data) != ""
+		} else {
+			data["l_lasttime"] = time.Now().Unix()
+			b = Mgo.Update("site_rule_logickv", `{"_id":"`+_id+`"}`, map[string]interface{}{
+				"$set": data,
+			}, true, false)
+		}
+		c.JSON(200, gin.H{"rep": b})
+	})
 	Admin.POST("/site_management/logicore/del", func(c *gin.Context) {
 		_id, _ := c.GetPostForm("_id")
 		//b := Mgo.Del("rule_logicore", `{"_id":"`+_id+`"}`)
 		b := Mgo.UpdateById("site_rule_logicore", _id, `{"$set":{"delete":true}}`)
 		c.JSON(200, gin.H{"rep": b})
 	})
+	Admin.POST("/site_management/logickv/del", func(c *gin.Context) {
+		_id, _ := c.GetPostForm("_id")
+		//b := Mgo.Del("rule_logicore", `{"_id":"`+_id+`"}`)
+		b := Mgo.UpdateById("site_rule_logickv", _id, `{"$set":{"delete":true}}`)
+		c.JSON(200, gin.H{"rep": b})
+	})
 	Admin.POST("/site_management/logicore/use", func(c *gin.Context) {
 		_id, _ := c.GetPostForm("_id")
 		isuse, _ := c.GetPostForm("isuse")
 		b := Mgo.UpdateById("site_rule_logicore", _id, `{"$set":{"isuse":`+isuse+`}}`)
 		c.JSON(200, gin.H{"rep": b})
 	})
+	Admin.POST("/site_management/logickv/use", func(c *gin.Context) {
+		_id, _ := c.GetPostForm("_id")
+		isuse, _ := c.GetPostForm("isuse")
+		b := Mgo.UpdateById("site_rule_logickv", _id, `{"$set":{"isuse":`+isuse+`}}`)
+		c.JSON(200, gin.H{"rep": b})
+	})
 
 	//后置规则
 	Admin.GET("/site_management/logicback", func(c *gin.Context) {

src/jy/admin/version.go

@@ -262,6 +262,17 @@ func init() {
 		//b := Mgo.Del("versioninfo", `{"_id":"`+_id+`"}`)
 		c.JSON(200, gin.H{"rep": b})
 	})
+
+	Admin.GET("/version/pkglogicore", func(c *gin.Context) {
+		vid := c.Query("vid")
+		s_field := c.Query("s_field")
+		sid := c.Query("sid")
+		c.HTML(200, "pkg_logicore.html", gin.H{"vid": vid, "sid": sid, "field": s_field})
+	})
+	Admin.POST("/version/pkglogicore/data", PkgRuleLogicCoreData)
+	Admin.POST("/version/pkglogicore/save", PkgRuleLogicCoreSave)
+	Admin.POST("/version/pkglogicore/del", PkgRuleLogicCoreDel)
+	Admin.POST("/version/pkglogicore/use", PkgRuleLogicCoreUse)
 	//分块配置
 	Admin.GET("/version/blockinfo", func(c *gin.Context) {
 		vid := c.Query("vid")
@@ -645,19 +656,21 @@ func copyPkgRules(vid, pid, s_field, oldvid, s_username string) {
 //克隆版本Field
 func copyFieldRules(vid, pid, s_field, oldvid, s_username string, isSite bool) {
 	log.Println(vid, pid, s_field, oldvid, s_username)
-	var versioninfo, rule_logic, rule_logicpre, rule_logicore, rule_logicback string
+	var versioninfo, rule_logic, rule_logicpre, rule_logicore, rule_logicback,rule_logickv string
 	if isSite {
 		versioninfo = "site_versioninfo"
 		rule_logic = "site_rule_logic"
 		rule_logicpre = "site_rule_logicpre"
 		rule_logicore = "site_rule_logicore"
 		rule_logicback = "site_rule_logicback"
+		rule_logickv = "site_rule_logickv"
 	} else {
 		versioninfo = "versioninfo"
 		rule_logic = "rule_logic"
 		rule_logicpre = "rule_logicpre"
 		rule_logicore = "rule_logicore"
 		rule_logicback = "rule_logicback"
+		rule_logickv = "rule_logickv"
 	}
 	tmp, _ := Mgo.FindOne(versioninfo, `{"vid":"`+oldvid+`","s_field":"`+s_field+`","delete":false}`)
 	oldpid := qu.BsonIdToSId((*tmp)["_id"])
@@ -699,6 +712,19 @@ func copyFieldRules(vid, pid, s_field, oldvid, s_username string, isSite bool) {
 				v["l_lasttime"] = time.Now().Unix()
 				Mgo.Save(rule_logicore, v)
 			}
+			//克隆kv抽取规则
+			kvlist, _ := Mgo.Find(rule_logickv, `{"sid":"`+oldsid+`","delete":false}`, nil, nil, false, -1, -1)
+			for _, v := range *kvlist {
+				delete(v, "_id")
+				v["vid"] = vid
+				v["pid"] = pid
+				v["sid"] = sid
+				v["s_code"] = util.GetSyncIndex(getCode(v["s_code"].(string)))
+				v["s_username"] = s_username
+				v["l_createtime"] = time.Now().Unix()
+				v["l_lasttime"] = time.Now().Unix()
+				Mgo.Save(rule_logickv, v)
+			}
 			//克隆过滤规则
 			blist, _ := Mgo.Find(rule_logicback, `{"sid":"`+oldsid+`","delete":false}`, nil, nil, false, -1, -1)
 			for _, v := range *blist {
@@ -774,3 +800,47 @@ func getCode(code string) string {
 	tmp := tmps[:len(tmps)-1]
 	return strings.Join(tmp, "_")
 }
+func PkgRuleLogicCoreData(c *gin.Context) {
+	vid, _ := c.GetPostForm("vid")
+	sid, _ := c.GetPostForm("sid")
+	s_field, _ := c.GetPostForm("s_field")
+	data, _ := Mgo.Find("pkg_logicore", `{"sid":"`+sid+`","delete":false}`, `{"_id":-1}`, nil, false, -1, -1)
+	c.JSON(200, gin.H{"data": data, "vid": vid, "s_field": s_field, "sid": sid})
+}
+func PkgRuleLogicCoreSave(c *gin.Context) {
+	data := GetPostForm(c)
+	_id, _ := c.GetPostForm("_id")
+	b := false
+	if _id == "" {
+		data["l_createtime"] = time.Now().Unix()
+		data["s_username"] = sessions.Default(c).Get("username")
+		data["delete"] = false
+		s_type := data["s_type"]
+		code := ""
+		if s_type == "0" { //抽取逻辑正则
+			code = util.GetSyncIndex(E_CE)
+		} else { //抽取逻辑脚本
+			code = util.GetSyncIndex(E_CL)
+		}
+		data["s_code"] = code
+		b = Mgo.Save("pkg_logicore", data) != ""
+	} else {
+		data["l_lasttime"] = time.Now().Unix()
+		b = Mgo.Update("pkg_logicore", `{"_id":"`+_id+`"}`, map[string]interface{}{
+			"$set": data,
+		}, true, false)
+	}
+	c.JSON(200, gin.H{"rep": b})
+}
+func PkgRuleLogicCoreDel(c *gin.Context) {
+	_id, _ := c.GetPostForm("_id")
+	//b := Mgo.Del("rule_logicore", `{"_id":"`+_id+`"}`)
+	b := Mgo.UpdateById("pkg_logicore", _id, `{"$set":{"delete":true}}`)
+	c.JSON(200, gin.H{"rep": b})
+}
+func PkgRuleLogicCoreUse(c *gin.Context) {
+	_id, _ := c.GetPostForm("_id")
+	isuse, _ := c.GetPostForm("isuse")
+	b := Mgo.UpdateById("pkg_logicore", _id, `{"$set":{"isuse":`+isuse+`}}`)
+	c.JSON(200, gin.H{"rep": b})
+}

src/jy/extract/exportask.go

@@ -63,6 +63,7 @@ func extractAndExport(v string, t map[string]interface{}) {
 	e.InitClearFn(true)
 	e.InfoTypeList()
 	e.InitBlockRule()
+	e.InitPkgCore()
 	//品牌抽取是否开启
 	ju.IsBrandGoods = ju.Config["brandgoods"].(bool)
 

src/jy/extract/extpackage.go

@@ -7,29 +7,33 @@ import (
 	"log"
 	qu "qfw/util"
 	"reflect"
+	"regexp"
 	"sort"
 )
 
-func pkvdata(pkg *ju.BlockPackage, sonJobResult *map[string]interface{}, e *ExtractTask,isSite bool,codeSite string) {
+func pkvdata(pkg *ju.BlockPackage, sonJobResult *map[string]interface{}, e *ExtractTask, isSite bool, codeSite string) {
 
 	if pkg.ColonKV != nil {
-		kvparse(pkg.ColonKV,  e, sonJobResult,isSite,codeSite)
+		kvparse(pkg.ColonKV, e, sonJobResult, isSite, codeSite)
 	}
 	if pkg.TableKV != nil {
-		kvparse(pkg.TableKV,  e, sonJobResult,isSite,codeSite)
+		kvparse(pkg.TableKV, e, sonJobResult, isSite, codeSite)
 	}
 	if pkg.SpaceKV != nil {
-		kvparse(pkg.SpaceKV,  e, sonJobResult,isSite,codeSite)
+		kvparse(pkg.SpaceKV, e, sonJobResult, isSite, codeSite)
 	}
 }
 
-func kvparse(p *ju.JobKv,  e *ExtractTask, sonJobResult *map[string]interface{},isSite bool,codeSite string) {
+func kvparse(p *ju.JobKv, e *ExtractTask, sonJobResult *map[string]interface{}, isSite bool, codeSite string) {
 	if p != nil {
 		for pk, pv2 := range p.KvTags {
 			if len(pv2) > 1 && !(pk == "预算" || pk == "中标金额") {
 				tmp := []*ju.Tag{}
 				var tmpindex, tmpweight int = -9999, -9999
 				for ii, vv := range pv2 {
+					if pk == "中标单位" && regexp.MustCompile("[0-9.元人￥$]").MatchString(vv.Value){
+						continue
+					}
 					if tmpweight < vv.Weight {
 						tmpindex = ii
 						tmpweight = vv.Weight
@@ -43,9 +47,9 @@ func kvparse(p *ju.JobKv,  e *ExtractTask, sonJobResult *map[string]interface{},
 			if len(pv) == 0 {
 				continue
 			}
-			tags := ju.GetTags(pk,isSite,codeSite)
+			tags := ju.GetTags(pk, isSite, codeSite)
 			if tags.Len() > 0 {
-				if ((*sonJobResult)["name"]  == nil || (*sonJobResult)["name"] == "")&& tags[0].Key == "项目名称"{
+				if ((*sonJobResult)["name"] == nil || (*sonJobResult)["name"] == "") && tags[0].Key == "项目名称" {
 					(*sonJobResult)["name"] = pv[0].Value
 				}
 				if qu.Float64All((*sonJobResult)["budget"]) == 0 && tags[0].Key == "预算" {
@@ -64,7 +68,7 @@ func kvparse(p *ju.JobKv,  e *ExtractTask, sonJobResult *map[string]interface{},
 					(*sonJobResult)["bidamount"] = data[0]
 					continue
 				}
-				if ((*sonJobResult)["winner"] == nil || (*sonJobResult)["winner"] == "" )&& tags[0].Key == "中标单位"{
+				if ((*sonJobResult)["winner"] == nil || (*sonJobResult)["winner"] == "") && tags[0].Key == "中标单位" {
 					(*sonJobResult)["winner"] = pv[0].Value
 				}
 
@@ -77,12 +81,24 @@ func kvparse(p *ju.JobKv,  e *ExtractTask, sonJobResult *map[string]interface{},
 }
 
 //处理分包信息
-func PackageDetail(j *ju.Job, e *ExtractTask,isSite bool,codeSite string) {
+func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 	qu.Try(func() {
 		if len(j.BlockPackage) > 0 {
+			for _, ev := range e.PkgRuleCores {
+				for _, eve := range ev.RuleCores {
+					if !eve.IsLua {
+						ExtRuleCoreByPkgReg(j, eve, e) // 分包正则抽取   预算   中标单位 中标价 成交状态
+					}
+				}
+				for _, evb := range ev.RuleBacks {
+					if !evb.IsLua {
+						ExtRegBackPkg(j, evb) // 分包正则清理  中标单位 成交状态   内容 名称
+					}
+				}
+			}
 			tmpkeys := []string{}
 			for k, _ := range j.BlockPackage {
-				if k == ""{
+				if k == "" {
 					continue
 				}
 				tmpkeys = append(tmpkeys, k)
@@ -90,7 +106,7 @@ func PackageDetail(j *ju.Job, e *ExtractTask,isSite bool,codeSite string) {
 			sort.Strings(tmpkeys)
 			packageResult := map[string]map[string]interface{}{}
 			//packagenum := len(j.BlockPackage)
-			for i, pkName := range tmpkeys {
+			for _, pkName := range tmpkeys {
 				pkg, ok := j.BlockPackage[pkName]
 				if !ok {
 					continue
@@ -103,36 +119,32 @@ func PackageDetail(j *ju.Job, e *ExtractTask,isSite bool,codeSite string) {
 					sonJobResult["text"] = pkg.Text
 					sonJobResult["budget"] = pkg.Budget
 					sonJobResult["bidamount"] = pkg.Bidamount
-					wins := make([]map[string]interface{}, 0)
 					if pkg.Winner == "" && len(j.Winnerorder) > 0 {
 						if sonJobResult["winnerorder"] == nil {
-							for _, tv := range j.Winnerorder {
-									if tv["type"].(int) == i{
-										wins = append(wins, tv)
-									}
+							sonJobResult["winnerorder"] = j.Winnerorder
+							if sonJobResult["bidamount"].(float64) <= 0 {
+								sonJobResult["bidamount"] = qu.Float64All(j.Winnerorder[0]["price"])
+							}
+							if sonJobResult["winner"] == "" {
+								sonJobResult["winner"] = j.Winnerorder[0]["entname"]
 							}
-							sonJobResult["winnerorder"] = wins
-						}
-						if len(wins) >0{
-							sonJobResult["bidamount"] = wins[0]["price"]
-							sonJobResult["winner"] = wins[0]["entname"]
 						}
 					} else {
 						if len(j.Winnerorder) > 0 {
-							sonJobResult["bidamount"] = j.Winnerorder[0]["price"]
-							sonJobResult["winner"] = wins[0]["entname"]
+							sonJobResult["bidamount"] = qu.Float64All(j.Winnerorder[0]["price"])
+							sonJobResult["winner"] = j.Winnerorder[0]["entname"]
 						}
 						sonJobResult["winnerorder"] = pkg.WinnerOrder
 					}
-					pkvdata(pkg, &sonJobResult, e,isSite,codeSite)
+					pkvdata(pkg, &sonJobResult, e, isSite, codeSite)
 
 					sonJobResult["type"] = pkg.Type
-					if len(tmpkeys) == 1{
-						if qu.Float64All(sonJobResult["budget"])==0{
-							for _,bv := range j.Block{
-								kvparse(bv.ColonKV,e,&sonJobResult,isSite,codeSite)
-								kvparse(bv.TableKV,e,&sonJobResult,isSite,codeSite)
-								kvparse(bv.SpaceKV,e,&sonJobResult,isSite,codeSite)
+					if len(tmpkeys) == 1 {
+						if qu.Float64All(sonJobResult["budget"]) == 0 {
+							for _, bv := range j.Block {
+								kvparse(bv.ColonKV, e, &sonJobResult, isSite, codeSite)
+								kvparse(bv.TableKV, e, &sonJobResult, isSite, codeSite)
+								kvparse(bv.SpaceKV, e, &sonJobResult, isSite, codeSite)
 							}
 						}
 					}
@@ -253,7 +265,7 @@ func PackageDetail(j *ju.Job, e *ExtractTask,isSite bool,codeSite string) {
 				j.PackageInfo = packageResult
 			}
 		}
-		extRegBackPack(j, e)
+		//extRegBackPack(j, e)
 	}, func(err interface{}) {
 		log.Println("PackageDetail err", err)
 	})

src/jy/extract/extract.go

@@ -421,13 +421,13 @@ func (e *ExtractTask) ExtractProcess(j, jf *ju.Job, isSite bool) {
 			//合并数据
 			j.Block = append(j.Block, tmpj.Block...)
 			j.Winnerorder = append(j.Winnerorder, tmpj.Winnerorder...)
-			for tmpk,_:= range j.Result{
-				if len(tmpj.Result[tmpk]) >0 {
+			for tmpk, _ := range j.Result {
+				if len(tmpj.Result[tmpk]) > 0 {
 					j.Result[tmpk] = append(j.Result[tmpk], tmpj.Result[tmpk]...)
 				}
 			}
-			for tmpk ,_ :=range tmpj.Result{
-				if len(j.Result[tmpk]) == 0{
+			for tmpk, _ := range tmpj.Result {
+				if len(j.Result[tmpk]) == 0 {
 					j.Result[tmpk] = append(j.Result[tmpk], tmpj.Result[tmpk]...)
 				}
 			}
@@ -503,6 +503,10 @@ func (e *ExtractTask) ExtractDetail(j *ju.Job, isSite bool, codeSite string) {
 				for _, v := range vc.RuleBacks {
 					ExtRegBack(j, v, e.TaskInfo)
 				}
+				//kv规则
+				for _, v := range vc.KVRuleCores {
+					ExtRuleKV(j, v, e.TaskInfo)
+				}
 				// log.Debug("抽取-后置规则", tmp)
 
 				//项目名称未能抽取到，标题来凑
@@ -517,7 +521,7 @@ func (e *ExtractTask) ExtractDetail(j *ju.Job, isSite bool, codeSite string) {
 						}
 						if isextitle { //标题加入选举
 							field := &ju.ExtField{Field: vc.Field, Code: vc.Id + "_title", RuleText: "title", Type: "title", MatchType: "title", ExtFrom: vc.ExtFrom, SourceValue: j.Title, Value: j.Title}
-							if isSite{
+							if isSite {
 								field.Score = 1
 							}
 							j.Result[vc.Field] = append(j.Result[vc.Field], field)
@@ -743,10 +747,10 @@ func ExtRuleCore(doc map[string]interface{}, e *ExtractTask, vc *RuleCore, j *ju
 									SourceValue: vv.Key,
 									Value:       text,
 								}
-								if isSite{
+								if isSite {
 									tmp.Score = 1
 								}
-								j.Result[vc.Field] = append(j.Result[vc.Field],tmp)
+								j.Result[vc.Field] = append(j.Result[vc.Field], tmp)
 							}
 						}
 					}
@@ -843,6 +847,139 @@ func ExtRuleCoreByReg(extfrom string, doc map[string]interface{}, j *ju.Job, in
 	}
 }
 
+//pkg抽取-规则-正则
+func ExtRuleCoreByPkgReg(j *ju.Job, in *RegLuaInfo, e *ExtractTask) {
+	defer qu.Catch()
+	//根据field配置项目，是否抽取。例如：废标、流标等跳过,
+	b := IsExtract(in.Field, j.Title, j.Content)
+	if !b {
+		return
+	}
+	//块抽取
+	if in.Field != "" {
+		for k, vbpkg := range j.BlockPackage {
+			rep := map[string]string{}
+			if in.RegCore.Bextract { //正则是两部分的，可以直接抽取的（含下划线）
+				if (in.Field == "budget" && vbpkg.Budget > 0) || (in.Field == "bidamount" && vbpkg.Bidamount > 0) || (in.Field == "winner" && vbpkg.Winner == "") || (in.Field == "bidstatus" && vbpkg.BidStatus == "")|| (in.Field == "projectname" && vbpkg.Name == "") {
+					continue
+				}
+				//处理正负数修正
+				ptmp := strings.Split(in.RuleText, "#")
+				sign := 0
+				if len(ptmp) == 2 {
+					if ptmp[1] == "正" {
+						sign = 1
+					} else if ptmp[1] == "负" {
+						sign = -1
+					}
+				}
+				tmp := strings.Split(ptmp[0], "__")
+				if len(tmp) == 2 {
+					epos := strings.Split(tmp[1], ",")
+					posm := map[string]int{}
+					for _, v := range epos {
+						ks := strings.Split(v, ":")
+						if len(ks) == 2 { //(.*)招标公告(.*)__2:projectname,4:area
+							posm[ks[1]] = qu.IntAll(ks[0])
+						} else {
+							posm[in.Field] = qu.IntAll(ks[0])
+						}
+					}
+					var pattern string
+					if strings.Contains(tmp[0], "\\u") {
+						tmp[0] = strings.Replace(tmp[0], "\\", "\\\\", -1)
+						tmp[0] = strings.Replace(tmp[0], "\\\\u", "\\u", -1)
+						pattern, _ = strconv.Unquote(`"` + tmp[0] + `"`)
+					} else {
+						pattern = tmp[0]
+					}
+					//log.Debug("pattern", pattern)
+					//fmt.Println(text)
+					reg := regexp.MustCompile(pattern)
+					apos := reg.FindAllStringSubmatchIndex(vbpkg.Text, -1)
+					for i, _ := range apos {
+						pos := apos[i]
+						for k, p := range posm {
+							if len(pos) > p {
+								if pos[p] == -1 || pos[p+1] == -1 {
+									continue
+								}
+								val := vbpkg.Text[pos[p]:pos[p+1]]
+								if string(val) == "" {
+									continue
+								}
+								if sign == -1 {
+									rep[k+"_"+fmt.Sprint(i)] = "-" + val
+								} else {
+									rep[k+"_"+fmt.Sprint(i)] = val
+								}
+							}
+						}
+					}
+					//fmt.Println(text)
+					for i := 0; i < len(apos); i++ {
+						if strings.TrimSpace(rep[in.Field+"_"+fmt.Sprint(i)]) != "" {
+							if in.Field == "budget" && vbpkg.Budget <= 0 {
+								lock.Lock()
+								cfn := e.ClearFn[in.Field]
+								lock.Unlock()
+								data := clear.DoClearFn(cfn, []interface{}{strings.TrimSpace(rep[in.Field+"_"+fmt.Sprint(i)]), j.Content})
+								j.BlockPackage[k].Budget = qu.Float64All(data[0])
+								break
+							}
+							if in.Field == "bidamount" && vbpkg.Bidamount <= 0 {
+								lock.Lock()
+								cfn := e.ClearFn[in.Field]
+								lock.Unlock()
+								data := clear.DoClearFn(cfn, []interface{}{strings.TrimSpace(rep[in.Field+"_"+fmt.Sprint(i)]), j.Content})
+								j.BlockPackage[k].Bidamount = qu.Float64All(data[0])
+								break
+							} else {
+								//if j.BlockPackage[k].RegKV == nil{
+								//	j.BlockPackage[k].RegKV = ju.NewJobKv()
+								//	j.BlockPackage[k].RegKV.KvTags
+								//}
+							}
+						}
+					}
+				}
+			} else {
+				pos := in.RegCore.Reg.FindStringIndex(vbpkg.Text)
+				val := ""
+				if len(pos) == 2 {
+					//"text" = "text"[pos[1]:]
+					val = "text"[pos[1]:]
+					rs := regexp.MustCompile("[^\r\n\t]+")
+					tmp := rs.FindAllString("text", -1)
+					if len(tmp) > 0 {
+						val = tmp[0]
+					}
+				}
+				if val != "" {
+					if in.Field == "budget" && vbpkg.Budget <= 0 {
+						lock.Lock()
+						cfn := e.ClearFn[in.Field]
+						lock.Unlock()
+						data := clear.DoClearFn(cfn, []interface{}{val, j.Content})
+						j.BlockPackage[k].Budget = qu.Float64All(data[0])
+						break
+					}
+					if in.Field == "bidamount" && vbpkg.Bidamount <= 0 {
+						lock.Lock()
+						cfn := e.ClearFn[in.Field]
+						lock.Unlock()
+						data := clear.DoClearFn(cfn, []interface{}{val, j.Content})
+						j.BlockPackage[k].Bidamount = qu.Float64All(data[0])
+						break
+					} else {
+
+					}
+				}
+			}
+		}
+	}
+}
+
 //lua脚本根据属性设置提取kv值
 func getKvByLuaFields(vc *RuleCore, j *ju.Job, et *ExtractTask) (map[string][]map[string]interface{}, bool) {
 	kvmap := map[string][]map[string]interface{}{}
@@ -952,10 +1089,10 @@ func extractFromKv(field, fieldname string, blocks []*ju.Block, vc *RuleCore, kv
 }
 
 //正则提取结果
-func extRegCoreToResult(extfrom, text string, tag *map[string]string, j *ju.Job, vre *RegLuaInfo,isSite bool) map[string][]map[string]interface{} {
+func extRegCoreToResult(extfrom, text string, tag *map[string]string, j *ju.Job, vre *RegLuaInfo, isSite bool) map[string][]map[string]interface{} {
 	defer qu.Catch()
 	var score int
-	if isSite{
+	if isSite {
 		score = 1
 	}
 	extinfo := map[string][]map[string]interface{}{}
@@ -1027,7 +1164,7 @@ func extRegCoreToResult(extfrom, text string, tag *map[string]string, j *ju.Job,
 						"type":      "regexp",
 						"matchtype": "regcontent",
 						"blocktag":  *tag,
-						"score" :score,
+						"score":     score,
 					}
 					tmps = append(tmps, tmp)
 
@@ -1064,7 +1201,7 @@ func extRegCoreToResult(extfrom, text string, tag *map[string]string, j *ju.Job,
 				"type":      "regexp",
 				"matchtype": "regcontent",
 				"blocktag":  *tag,
-				"score" :score,
+				"score":     score,
 			}
 			tmps = append(tmps, tmp)
 			extinfo[vre.Field] = tmps
@@ -1136,8 +1273,8 @@ func ExtRegBack(j *ju.Job, in *RegLuaInfo, t *TaskInfo) {
 						"value":     text,
 					})
 				}
-				extinfo[in.Field] = exts
-				if len(extinfo) > 0 {
+				if len(exts) > 0 {
+					extinfo[in.Field] = exts
 					AddExtLog("clear", j.SourceMid, tmp, extinfo, in, t) //抽取日志
 				}
 			}
@@ -1166,10 +1303,64 @@ func ExtRegBack(j *ju.Job, in *RegLuaInfo, t *TaskInfo) {
 						"value":     text,
 					})
 				}
-				extinfo[key] = exts
+				if len(exts) > 0 {
+					extinfo[key] = exts
+					AddExtLog("clear", j.SourceMid, j.Result, extinfo, in, t) //抽取日志
+				}
 			}
-			if len(extinfo) > 0 {
-				AddExtLog("clear", j.SourceMid, j.Result, extinfo, in, t) //抽取日志
+		}
+	}
+}
+
+//后置过滤
+func ExtRegBackPkg(j *ju.Job, in *RegLuaInfo) {
+	defer qu.Catch()
+	for k,v := range j.BlockPackage{
+		if in.Field == "winner"{
+			j.BlockPackage[k].Winner = in.RegPreBac.Reg.ReplaceAllString(v.Winner, in.RegPreBac.Replace)
+		}else if in.Field == "bidstatus"{
+			j.BlockPackage[k].BidStatus = in.RegPreBac.Reg.ReplaceAllString(v.BidStatus, in.RegPreBac.Replace)
+		}else if in.Field == ""{
+			j.BlockPackage[k].Text = in.RegPreBac.Reg.ReplaceAllString(v.Text, in.RegPreBac.Replace)
+		}else if in.Field== "projectname"{
+			j.BlockPackage[k].Name = in.RegPreBac.Reg.ReplaceAllString(v.Name, in.RegPreBac.Replace)
+		}
+	}
+}
+
+//KV过滤
+func ExtRuleKV(j *ju.Job, in *RegLuaInfo, t *TaskInfo) {
+	defer qu.Catch()
+	extinfo := map[string]interface{}{}
+	if in.Field != "" {
+		if j.Result[in.Field] != nil {
+			tmp := j.Result[in.Field]
+			exts := []interface{}{}
+			for k, v := range tmp {
+				if v.Type != "table" && !strings.Contains(v.Type, "colon") && !strings.Contains(v.Type, "space") {
+					continue
+				}
+				text := qu.ObjToString(v.Value)
+				if text != "" {
+					text = in.RegPreBac.Reg.ReplaceAllString(text, in.RegPreBac.Replace)
+				}
+				if text == qu.ObjToString(v.Value) { //值未发生改变，不存日志
+					continue
+				}
+				j.Result[in.Field][k].Value = text
+				exts = append(exts, map[string]interface{}{
+					"field":     v.Field,
+					"code":      v.Code,
+					"ruletext":  v.RuleText,
+					"type":      v.Type,
+					"matchtype": v.MatchType,
+					"extfrom":   v.ExtFrom,
+					"value":     text,
+				})
+			}
+			if len(exts) > 0 {
+				extinfo[in.Field] = exts
+				AddExtLog("clear", j.SourceMid, tmp, extinfo, in, t) //抽取日志
 			}
 		}
 	}
@@ -1726,7 +1917,7 @@ func resetWinnerorder(j *ju.Job) {
 	if len(j.Winnerorder) == 0 {
 		return
 	}
-	maxlen := len(j.Winnerorder)-1
+	maxlen := len(j.Winnerorder) - 1
 	//中标单位
 	i := 0
 	winners := []*ju.ExtField{}

src/jy/extract/extractInit.go

@@ -33,14 +33,15 @@ type ExtReg struct {
 	NumSign    int //正负修正标记，例如浮动率（上浮正1、下浮负-1）
 }
 type RuleCore struct {
-	Id        string            //id
-	Field     string            //逻辑字段
-	LuaLogic  string            //进入逻辑
-	ExtFrom   string            //从哪个字段抽取
-	RulePres  []*RegLuaInfo     //抽取前置规则
-	RuleBacks []*RegLuaInfo     //抽取后置规则
-	RuleCores []*RegLuaInfo     //抽取规则
-	LFields   map[string]string //所有字段属性组
+	Id          string            //id
+	Field       string            //逻辑字段
+	LuaLogic    string            //进入逻辑
+	ExtFrom     string            //从哪个字段抽取
+	RulePres    []*RegLuaInfo     //抽取前置规则
+	RuleBacks   []*RegLuaInfo     //抽取后置规则
+	RuleCores   []*RegLuaInfo     //抽取规则
+	KVRuleCores []*RegLuaInfo     //KV抽取清理规则
+	LFields     map[string]string //所有字段属性组
 }
 
 type Tag struct {
@@ -84,15 +85,20 @@ type ExtractTask struct {
 
 	ResultChanel chan bool //抽取结果详情
 	sync.RWMutex
-	ResultArr [][]map[string]interface{} //抽取结果详情
-	BidChanel chan bool                  //抽取结果
-	BidArr    [][]map[string]interface{} //抽取结果
-	BidTotal int                         //结果数量
+	ResultArr [][]map[string]interface {
+	}                   //抽取结果详情
+	BidChanel chan bool //抽取结果
+	BidArr    [][]map[string]interface {
+	}                   //抽取结果
+	BidTotal int        //结果数量
 
-	RecogFieldMap map[string]map[string]interface{}   //识别字段
-	FidClassMap   map[string][]map[string]interface{} //分类
-	CidRuleMap map[string][]map[string]interface{}    //规则
-	AuditFields []string                              //需要审核的字段名称
+	RecogFieldMap map[string]map[string]interface {
+	}                    //识别字段
+	FidClassMap map[string][]map[string]interface {
+	}                    //分类
+	CidRuleMap map[string][]map[string]interface {
+	}                    //规则
+	AuditFields []string //需要审核的字段名称
 
 	SiteCityMap          map[string]*SiteCity //站点对应的省市区
 	ProvinceMap          map[string]string    //省全称简称(key:浙江省 val:浙江)
@@ -118,7 +124,8 @@ type ExtractTask struct {
 	PostCodeMap map[string]*PostCode //邮编
 	AreaCodeMap map[string]*AreaCode //区号
 
-	InfoType []map[string]interface{}
+	InfoType []map[string]interface {
+	}
 
 	Trie_Full_Province  *ju.Trie       //省全称 省、直辖市、自治区
 	Trie_Full_City      *ju.Trie       //市全称 地级市
@@ -246,11 +253,11 @@ func (e *ExtractTask) InitSite() {
 		if vv, ok := v["site_script"].([]interface{}); ok {
 			for _, vvv := range vv {
 				e.Luacodes.Store(vvv, map[string]interface{}{})
-				e.SiteMerge.Store(vvv,v["ismerge"].(bool))
+				e.SiteMerge.Store(vvv, v["ismerge"].(bool))
 			}
 		} else if vv, ok := v["site_script"].(interface{}); ok {
 			e.Luacodes.Store(vv, map[string]interface{}{})
-			e.SiteMerge.Store(vv,v["ismerge"].(bool))
+			e.SiteMerge.Store(vv, v["ismerge"].(bool))
 		}
 	}
 }
@@ -384,7 +391,7 @@ func (e *ExtractTask) InitRuleCore(isSite bool) {
 	defer qu.Catch()
 	allFields := getALLFields()
 	e.Fields = map[string]int{}
-	var versioninfodb, rule_logicdb, rule_logicpredb, rule_logicbackdb, rule_logicoredb string
+	var versioninfodb, rule_logicdb, rule_logicpredb, rule_logicbackdb, rule_logicoredb, rule_logickvdb string
 	eSiteRuleCores := make(map[string]map[string][]*RuleCore)
 	if isSite {
 		versioninfodb = "site_versioninfo"
@@ -392,6 +399,7 @@ func (e *ExtractTask) InitRuleCore(isSite bool) {
 		rule_logicpredb = "site_rule_logicpre"
 		rule_logicbackdb = "site_rule_logicback"
 		rule_logicoredb = "site_rule_logicore"
+		rule_logickvdb = "site_rule_logickv"
 		e.SiteRuleCores = make(map[string]map[string][]*RuleCore)
 	} else {
 		versioninfodb = "versioninfo"
@@ -399,6 +407,7 @@ func (e *ExtractTask) InitRuleCore(isSite bool) {
 		rule_logicpredb = "rule_logicpre"
 		rule_logicbackdb = "rule_logicback"
 		rule_logicoredb = "rule_logicore"
+		rule_logickvdb = "rule_logickv"
 		e.RuleCores = make(map[string]map[string][]*RuleCore)
 	}
 
@@ -549,7 +558,43 @@ func (e *ExtractTask) InitRuleCore(isSite bool) {
 				}
 			}
 			rcore.RuleCores = ruleCores
-			//
+			//kv规则
+			kvRuleCores := []*RegLuaInfo{}
+			kvlist, _ := db.Mgo.Find(rule_logickvdb, `{"sid":"`+qu.BsonIdToSId(vv["_id"])+`","delete":false}`, nil, nil, false, -1, -1)
+			for _, v := range *kvlist {
+				if b, _ := v["isuse"].(bool); !b {
+					continue
+				}
+				field := qu.ObjToString(v["s_field"])
+				e.Fields[field] = 1 //加入抽取属性组备用
+				rinfo := &RegLuaInfo{
+					Field: field,
+					Code:  v["s_code"].(string),
+					Name:  v["s_name"].(string),
+					IsLua: qu.If(v["s_type"].(string) == "1", true, false).(bool),
+				}
+				qu.Try(func() {
+					rinfo.RuleText = v["s_rule"].(string)
+					tmp := strings.Split(rinfo.RuleText, "__")
+					var pattern string
+					if strings.Contains(tmp[0], "\\u") {
+						tmp[0] = strings.Replace(tmp[0], "\\", "\\\\", -1)
+						tmp[0] = strings.Replace(tmp[0], "\\\\u", "\\u", -1)
+						pattern, _ = strconv.Unquote(`"` + tmp[0] + `"`)
+					} else {
+						pattern = tmp[0]
+					}
+					if len(tmp) == 2 {
+						rinfo.RegPreBac = &ExtReg{Reg: regexp.MustCompile(pattern), Replace: tmp[1]}
+					} else {
+						rinfo.RegPreBac = &ExtReg{Reg: regexp.MustCompile(pattern), Replace: ""}
+					}
+					kvRuleCores = append(kvRuleCores, rinfo)
+				}, func(err interface{}) {
+					log.Debug(rinfo.Code, rinfo.Field, err)
+				})
+			}
+			rcore.KVRuleCores = kvRuleCores
 			if fieldrules[s_field] == nil {
 				fieldrules[s_field] = []*RuleCore{}
 			}
@@ -640,55 +685,101 @@ func (e *ExtractTask) InitPkgCore() {
 			continue
 		}
 		s_field := qu.ObjToString(pkginfo["s_field"])
-		pid := qu.BsonIdToSId(pkginfo["_id"])
-		logicList, _ := db.Mgo.Find("pkg_logic", `{"pid":"`+pid+`","delete":false}`, nil, nil, false, -1, -1)
-		for _, vv := range *logicList {
-			if b, _ := vv["isuse"].(bool); !b {
-				continue
+		sid := qu.BsonIdToSId(pkginfo["_id"])
+		rcore := &RuleCore{}
+		rcore.Field = s_field
+		rcore.ExtFrom = "detail"
+		//后置规则
+		ruleBacks := []*RegLuaInfo{}
+		blist, _ := db.Mgo.Find("pkg_logicback", `{"sid":"`+sid+`","delete":false}`, nil, nil, false, -1, -1)
+		for _, v := range *blist {
+			rinfo := &RegLuaInfo{
+				Field: qu.ObjToString(v["s_field"]),
+				Code:  v["s_code"].(string),
+				Name:  v["s_name"].(string),
+				IsLua: qu.If(v["s_type"].(string) == "1", true, false).(bool),
 			}
-			rcore := &RuleCore{}
-			rcore.Field = s_field
-			rcore.LuaLogic = qu.ObjToString(vv["s_luascript"]) //是否进入逻辑脚本
-			rcore.ExtFrom = qu.If(vv["extfrom"].(bool), "title", "detail").(string)
-			//后置规则
-			ruleBacks := []*RegLuaInfo{}
-			blist, _ := db.Mgo.Find("pkg_logicback", `{"sid":"`+qu.BsonIdToSId(vv["_id"])+`","delete":false}`, nil, nil, false, -1, -1)
-			for _, v := range *blist {
-				rinfo := &RegLuaInfo{
-					Field: qu.ObjToString(v["s_field"]),
-					Code:  v["s_code"].(string),
-					Name:  v["s_name"].(string),
-					IsLua: qu.If(v["s_type"].(string) == "1", true, false).(bool),
-				}
-				if rinfo.IsLua {
-					rinfo.RuleText = v["s_luascript"].(string)
+			if rinfo.IsLua {
+				rinfo.RuleText = v["s_luascript"].(string)
+				ruleBacks = append(ruleBacks, rinfo)
+			} else {
+				qu.Try(func() {
+					rinfo.RuleText = v["s_rule"].(string)
+					tmp := strings.Split(rinfo.RuleText, "__")
+					var pattern string
+					if strings.Contains(tmp[0], "\\u") {
+						tmp[0] = strings.Replace(tmp[0], "\\", "\\\\", -1)
+						tmp[0] = strings.Replace(tmp[0], "\\\\u", "\\u", -1)
+						pattern, _ = strconv.Unquote(`"` + tmp[0] + `"`)
+					} else {
+						pattern = tmp[0]
+					}
+					if len(tmp) == 2 {
+						rinfo.RegPreBac = &ExtReg{Reg: regexp.MustCompile(pattern), Replace: tmp[1]}
+					} else {
+						rinfo.RegPreBac = &ExtReg{Reg: regexp.MustCompile(pattern), Replace: ""}
+					}
 					ruleBacks = append(ruleBacks, rinfo)
-				} else {
-					qu.Try(func() {
-						rinfo.RuleText = v["s_rule"].(string)
-						tmp := strings.Split(rinfo.RuleText, "__")
-						var pattern string
-						if strings.Contains(tmp[0], "\\u") {
-							tmp[0] = strings.Replace(tmp[0], "\\", "\\\\", -1)
-							tmp[0] = strings.Replace(tmp[0], "\\\\u", "\\u", -1)
-							pattern, _ = strconv.Unquote(`"` + tmp[0] + `"`)
-						} else {
-							pattern = tmp[0]
-						}
-						if len(tmp) == 2 {
-							rinfo.RegPreBac = &ExtReg{Reg: regexp.MustCompile(pattern), Replace: tmp[1]}
-						} else {
-							rinfo.RegPreBac = &ExtReg{Reg: regexp.MustCompile(pattern), Replace: ""}
+				}, func(err interface{}) {
+					log.Debug(rinfo.Code, rinfo.Field, err)
+				})
+			}
+		}
+		rcore.RuleBacks = ruleBacks
+		//抽取规则
+		ruleCores := []*RegLuaInfo{}
+		clist, _ := db.Mgo.Find("pkg_logicore", `{"sid":"`+sid+`","delete":false}`, nil, nil, false, -1, -1)
+		for _, v := range *clist {
+			if b, _ := v["isuse"].(bool); !b {
+				continue
+			}
+			field := qu.ObjToString(v["s_field"])
+			e.Fields[field] = 1 //加入抽取属性组备用
+			rinfo := &RegLuaInfo{
+				Field: field,
+				Code:  v["s_code"].(string),
+				Name:  v["s_name"].(string),
+				IsLua: qu.If(v["s_type"].(string) == "1", true, false).(bool),
+			}
+			if rinfo.IsLua {
+				rinfo.RuleText = v["s_luascript"].(string)
+				//提取全部属性
+				ruleCores = append(ruleCores, rinfo)
+			} else {
+				qu.Try(func() {
+					rinfo.RuleText = v["s_rule"].(string)
+					tmp := strings.Split(rinfo.RuleText, "__")
+					var pattern string
+					if strings.Contains(tmp[0], "\\u") {
+						tmp[0] = strings.Replace(tmp[0], "\\", "\\\\", -1)
+						tmp[0] = strings.Replace(tmp[0], "\\\\u", "\\u", -1)
+						pattern, _ = strconv.Unquote(`"` + tmp[0] + `"`)
+					} else {
+						pattern = tmp[0]
+					}
+					if len(tmp) == 2 {
+						epos := strings.Split(tmp[1], ",")
+						posm := map[string]int{}
+						for _, v := range epos {
+							ks := strings.Split(v, ":")
+							if len(ks) == 2 { //(.*)招标公告(.*)__2:projectname,4:area
+								posm[ks[1]] = qu.IntAll(ks[0])
+							} else { //(.*)招标公告__2
+								posm[rinfo.Field] = qu.IntAll(ks[0])
+							}
 						}
-						ruleBacks = append(ruleBacks, rinfo)
-					}, func(err interface{}) {
-						log.Debug(rinfo.Code, rinfo.Field, err)
-					})
-				}
+						rinfo.RegCore = &ExtReg{Reg: regexp.MustCompile(pattern), Bextract: true, ExtractPos: posm}
+					} else {
+						rinfo.RegCore = &ExtReg{Reg: regexp.MustCompile(pattern), Bextract: false}
+					}
+					ruleCores = append(ruleCores, rinfo)
+				}, func(err interface{}) {
+					log.Debug(rinfo.Code, rinfo.Field, err)
+				})
 			}
-			rcore.RuleBacks = ruleBacks
-			e.PkgRuleCores = append(e.PkgRuleCores, rcore)
 		}
+		rcore.RuleCores = ruleCores
+		e.PkgRuleCores = append(e.PkgRuleCores, rcore)
 	}
 }
 

src/jy/extract/extractudp.go

@@ -106,6 +106,7 @@ func ExtractByUdp(sid, eid string, ra *net.UDPAddr, instanceId ...string) {
 		ext.InitRuleCore(false)
 		ext.InitRuleCore(true)
 		ext.InitBlockRule()
+		ext.InitPkgCore()
 		ext.InitTag(false)
 		ext.InitTag(true)
 		ext.InitClearFn(false)

src/jy/extract/score.go

@@ -168,7 +168,7 @@ func ScoreFields(j *ju.Job, ftag map[string][]*Tag) map[string][]*ju.ExtField {
 			//kv权重打分
 			if fieldscore != nil { //指定抽取属性打分配置
 				if tmpsvalue.Type == "colon" || tmpsvalue.Type == "space" || tmpsvalue.Type == "table" {
-					if taglength == 0{
+					if taglength == 0 {
 						continue
 					}
 					weightscore := ju.FloatFormat(float64(qu.Float64All(fieldscore["kvweight"]))+float64(tmps[tmpsindex].Weight)/float64(taglength), 4)
@@ -179,7 +179,7 @@ func ScoreFields(j *ju.Job, ftag map[string][]*Tag) map[string][]*ju.ExtField {
 				}
 			} else {
 				if tmpsvalue.Type == "colon" || tmpsvalue.Type == "space" || tmpsvalue.Type == "table" {
-					if taglength == 0{
+					if taglength == 0 {
 						continue
 					}
 					weightscore := ju.FloatFormat(float64(qu.Float64All(CommonScore["kvweight"]))+float64(tmps[tmpsindex].Weight)/float64(taglength), 4)
@@ -270,7 +270,7 @@ func ScoreFields(j *ju.Job, ftag map[string][]*Tag) map[string][]*ju.ExtField {
 				max := qu.IntAll(scoreRule["max"])
 				val := qu.IntAll(tmpsvalue.Value)
 				scores, _ := scoreRule["score"].([]interface{})
-				if len(scores) < 3 || val == 0{
+				if len(scores) < 3 || val == 0 {
 					continue
 				}
 				if val < min && 0 < val {
@@ -324,12 +324,13 @@ func ScoreFields(j *ju.Job, ftag map[string][]*Tag) map[string][]*ju.ExtField {
 //项目编号权重清理
 func projectWeightClear(tmps []*ju.ExtField) []*ju.ExtField {
 	newList := make([]*ju.ExtField, 0)
-	if len(tmps)<1{
+	if len(tmps) < 1 {
 		return newList
 	}
 	ju.Sort(tmps)
 	tmpWeight := -999 //记录最大权重
 	tmpIndex := -999  //记录最大权重下标
+	vmap := make(map[string]int, 0)
 	for i, v := range tmps {
 		if v.Weight == 0 {
 			newList = append(newList, v)
@@ -337,10 +338,19 @@ func projectWeightClear(tmps []*ju.ExtField) []*ju.ExtField {
 		} else if v.Weight > tmpWeight {
 			tmpWeight = v.Weight
 			tmpIndex = i
+		} else if v.Weight == tmpWeight {
+			if utf8.RuneCountInString(qu.ObjToString(v.Value)) >= 5 && utf8.RuneCountInString(qu.ObjToString(v.Value)) <= 38 && v.Value != tmps[tmpIndex].Value {
+				vmap[qu.ObjToString(v.Value)] = i
+			}
 		}
 	}
 	if tmpIndex != -999 {
 		newList = append(newList, tmps[tmpIndex])
 	}
+	if len(vmap) > 0 {
+		for _, v := range vmap {
+			newList = append(newList, tmps[v])
+		}
+	}
 	return newList
 }

src/jy/pretreated/analystep.go

@@ -208,13 +208,13 @@ func processTableResult(tabres *TableResult, block *util.Block, job *util.Job,is
 	}
 	//处理中标人排序
 	wror := []map[string]interface{}{}
-	for _, v := range tabres.WinnerOrder {
+	for i, v := range tabres.WinnerOrder {
 		entName, _ := v["entname"].(string)
 		v["entname"] = winnerOrderEntity.clear("中标单位", entName)
 		if price, ok := v["price"].(string); ok {
 			v["price"] = winnerOrderEntity.clear("中标金额", price)
 		}
-		v["type"] = 2
+		v["type"] = i
 		wror = append(wror, v)
 	}
 	if len(wror) > 0 {

src/jy/pretreated/analytable.go

@@ -1691,6 +1691,9 @@ func GetBidSort(str string, n int) int {
 
 //查找每一个单元格的表头,调用FindNear
 func (table *Table) FindTdVal(td *TD, direct, vdirect int) (b bool) {
+	if td.Val == "" || strings.TrimSpace(td.Val) == ""{
+		return
+	}
 	near := table.FindNear(td, direct)
 	//	if near != nil {
 	//		fmt.Println("near----", near.Val, td.Val)

src/jy/pretreated/division.go

@@ -597,7 +597,6 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 		if len(pkg) == 1 && strings.HasSuffix(con, v[0]) {
 			return false, ""
 		}
-		//
 		is := regexp.MustCompile(v[0]+"[:：]*").FindAllStringIndex(con, -1)
 		for _, sv := range is {
 			appendWarpIndex = append(appendWarpIndex, sv[0])
@@ -731,6 +730,7 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 					Type:     bv[1],
 					Accuracy: accuracy,
 				}
+				//fmt.Println(text)
 				finalKv := GetKVAll(strings.TrimLeft(text, headKey), "", nil, 4,isSite,codeSite)
 				if headKey != "" {
 					kvAgain := GetKVAll(text, "", nil, 4,isSite,codeSite)
@@ -785,10 +785,12 @@ func interceptText(indexs []int, indexPkgMap map[int]string, pkgIndexMap map[str
 		} else {
 			text = con[iv:]
 		}
+		//fmt.Println(text)
 		tmptext := text
-		if strings.Contains(text, "、") {
-			text = strings.Split(text, "、")[0]
-		} else if strings.Contains(text, "\n") {
+		//if strings.Contains(text, "、") {
+		//	text = strings.Split(text, "、")[0]
+		//} else
+		if strings.Contains(text, "\n") {
 			texts := strings.Split(text, "\n")
 			text2 :=""
 			if ik+1 < len(indexs)-1 {

src/jy/pretreated/tablev2.go

@@ -405,7 +405,7 @@ func (td *TD) tdIsHb(tr *TR, table *Table, bsontable,isSite bool,codeSite string
 			if lenval > 15 && !strings.Contains(txt, "采购代理机构名称、地址和联系方式") {
 				btw = false
 			}
-			if strings.Contains(td.Val, "个项目") {
+			if strings.Contains(td.Val, "个项目") ||strings.Contains(td.Val, "奥图码"){
 				must = false
 				btw = false
 			}

src/jy/util/article.go

@@ -126,7 +126,7 @@ type Segment struct {
 
 //包
 type BlockPackage struct {
-	Origin   string                   //包的原始值
+	Origin      string                   //包的原始值
 	Name        string                   //标段（包）名称
 	Text        string                   //包文 (包对应的正文)
 	Budget      float64                  //标段（包）预算

src/main_blocktest.go

@@ -84,6 +84,7 @@ func com(doc map[string]interface{}) {
 	e.InitRuleCore(false)
 	e.InitRuleCore(true)
 	e.InitBlockRule()
+	e.InitPkgCore()
 	e.InitTag(false)
 	e.InitTag(true)
 	e.InitClearFn(false)

src/res/fieldscore.json

@@ -147,7 +147,7 @@
         "negativewords": [
             {
                 "describe": "包含负分",
-                "regstr": "(附件|委托|代理|咨询|顾问|管理有限公司|管理顾问|招标失败|交易中心|不足|公告|变更|招标|废标|废止|流标|中标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
+                "regstr": "(标人|附件|委托|代理|咨询|顾问|管理有限公司|管理顾问|招标失败|交易中心|不足|公告|变更|招标|废标|废止|流标|中标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
                 "score": -5
             },
             {

src/web/templates/admin/pkg_info.html

@@ -85,9 +85,13 @@ $(function () {
 				}
 				return tmp
 			}},
-			{"data":"_id",render:function(val,a,row){
-				return '<a class="btn btn-sm btn-success" href="/admin/pkglogic?vid={{.vid}}&pid='+val+'">配置逻辑</a>'
-			}},
+			{"data":"_id","width":"25%",render:function(val,a,row){
+					tmp = '<div class="btn-group">'+
+							'<a class="btn btn-sm btn-success" href="/admin/version/pkglogicore?vid={{.vid}}&s_field='+row.s_field+'&sid='+row._id+'">抽取规则</a>'+
+							'<a class="btn btn-sm btn-info" href="/admin/pkglogicback?vid={{.vid}}&s_field='+row.s_field+'&sid='+row._id+'">后置规则</a>'+
+							'</div>';
+					return  tmp
+				}},
 			{"data":"_id",render:function(val,a,row){
 				return '<a class="btn btn-sm btn-primary opr" opr="edit">编辑</a>&nbsp;<a class="btn btn-sm btn-danger" href="#" onclick="del(\''+val+'\')">删除</a>'
 			}}

src/web/templates/admin/pkg_logicbacklist.html

@@ -9,13 +9,11 @@
 	<section class="content-header">
 		<h1>
 			<small><a class="btn btn-primary opr" opr="new">新增正则</a></small>
-			<small><a class="btn btn-primary opr" opr="newlua">新增脚本</a></small>
 		</h1>
 		<ol class="breadcrumb">
 		  <li><a href="/admin/version"><i class="fa fa-dashboard"></i>版本管理</a></li>
-		  <li><a href="/admin/version/pkginfo?vid={{.vid}}&pid={{.pid}}">分包属性</a></li>
-		  <li><a href="/admin/pkglogic?vid={{.vid}}&pid={{.pid}}">抽取逻辑</a></li>
-		  <li class="active"><a href="/admin/pkglogicback?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}">后置规则</a></li>
+		  <li><a href="/admin/version/pkginfo?vid={{.vid}}">分包属性</a></li>
+		  <li class="active"><a href="/admin/pkglogicback?vid={{.vid}}&s_field={{.s_field}}&sid={{.sid}}">后置规则</a></li>
 		</ol>
     </section>
   <!-- Main content -->
@@ -61,7 +59,7 @@ $(function () {
 		"ajax": {
 			"url": "/admin/pkglogicback/data",
 			"type": "post",
-			"data":{"vid":{{.vid}},"pid":{{.pid}},"sid":{{ .sid}} }
+			"data":{"vid":{{.vid}},"s_field":{{.s_field}},"sid":{{ .sid}} }
 		 },
 		"language": {
             "url": "/res/dist/js/dataTables.chinese.lang"
@@ -119,7 +117,7 @@ $(function () {
 								if (bcon){								
 									$.post("/admin/pkglogicback/save",obj,function(data){
 										if(data&&data.rep){
-											window.location.href="/admin/pkglogicback?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}"								
+											window.location.href="/admin/pkglogicback?vid={{.vid}}&s_field={{.s_field}}&sid={{.sid}}"
 										}else{
 											showTip(data.msg,1000)
 										}
@@ -140,9 +138,9 @@ $(function () {
 			case "new":
 				comtag=[{label:"名称",s_label:"s_name",placeholder:"",must:true},{label:"描述",s_label:"s_descript",type:"tpl_text"}]
 				regtag=[{label:"字段",s_label:"s_field",type:"tpl_list_local",url:"/admin/getfields",default:{{.field}}},{label:"正则",s_label:"s_rule",type:"tpl_text",must:true}]
-				luatag=[{label:"脚本",s_label:"s_luascript",type:"tpl_text",must:true}]
+				luatag=[{label:"脚本",s_label:"s_field",type:"tpl_list_local",url:"/admin/getfields",default:{{.field}}},{label:"脚本",s_label:"s_luascript",type:"tpl_text",must:true}]
 				testcon=[{label:"测试内容",s_label:"s_testcon",type:"tpl_text",must:true}]
-				hiddentag=[{s_label:"_id",type:"tpl_hidden"},{s_label:"vid",type:"tpl_hidden"},{s_label:"pid",type:"tpl_hidden"},{s_label:"sid",type:"tpl_hidden"},{s_label:"s_type",type:"tpl_hidden"}]
+				hiddentag=[{s_label:"_id",type:"tpl_hidden"},{s_label:"vid",type:"tpl_hidden"},{s_label:"s_field",type:"tpl_hidden"},{s_label:"sid",type:"tpl_hidden"},{s_label:"s_type",type:"tpl_hidden"}]
 				islua=false
 				tag = com.pushArry(tag,comtag)
 				if(n=="edit"){
@@ -179,7 +177,7 @@ $(function () {
 					tag = com.pushArry(tag,hiddentag)
 					check=[{label:"测试",class:"btn-warning",
 								fun:function(){
-									window.location.href="/admin/check/lua/pkgcoreback?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}&s_code="+obj.s_code
+									window.location.href="/admin/check/lua/pkgcoreback?vid={{.vid}}&s_field={{.s_field}}&sid={{.sid}}&s_code="+obj.s_code
 								}
 							}]
 					bts = com.pushArry(bts,check)
@@ -187,12 +185,12 @@ $(function () {
 					_tit="新增规则"
 					if(n=="newlua"){
 						_tit="新增脚本"
-						obj={"s_luascript":"--code脚本代码，result 抽取结果，过滤后返回result，对象结构不可改变\nfunction main(code,result)\n\t--过滤操作\n\treturn result\nend","vid":"{{.vid}}","pid":"{{.pid}}","sid":"{{.sid}}","s_type":"1"}
+						obj={"s_luascript":"--code脚本代码，result 抽取结果，过滤后返回result，对象结构不可改变\nfunction main(code,result)\n\t--过滤操作\n\treturn result\nend","vid":"{{.vid}}","s_field":"{{.s_field}}","sid":"{{.sid}}","s_type":"1"}
 						tag = com.pushArry(tag,luatag)
 						tag = com.pushArry(tag,hiddentag)
 						islua=true
 					}else{
-						obj={"vid":"{{.vid}}","pid":"{{.pid}}","sid":"{{.sid}}","s_type":"0"}
+						obj={"vid":"{{.vid}}","s_field":"{{.s_field}}","sid":"{{.sid}}","s_type":"0"}
 						tag = com.pushArry(tag,regtag)
 						tag = com.pushArry(tag,hiddentag)
 					}
@@ -218,7 +216,7 @@ function del(_id){
 			data:{"_id":_id},
 			success:function(r){
 				if(r.rep){				
-					window.location.href="/admin/pkglogicback?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}"
+					window.location.href="/admin/pkglogicback?vid={{.vid}}&s_field={{.s_field}}&sid={{.sid}}"
 				}else{
 					showTip("删除失败", 1000);
 				}

src/web/templates/admin/pkg_logiclist.html

@@ -1,198 +0,0 @@
-{{template "inc"}}
-<!-- Main Header -->
-{{template "header"}}
-<!-- Left side column. 权限菜单 -->
-{{template "memu"}}
-
-<!-- Content Wrapper. Contains page content -->
-<div class="content-wrapper">
-	<section class="content-header">
-		<h1>
-			<small><a class="btn btn-primary opr" opr="new">新增逻辑</a></small>
-		</h1>
-		<ol class="breadcrumb">
-		  <li><a href="/admin/version"><i class="fa fa-dashboard"></i>版本管理</a></li>
-		  <li class="active"><a href="/admin/version/pkginfo?vid={{.vid}}">分包属性</a></li>
-		  <li class="active"><a href="/admin/pkglogic?vid={{.vid}}&pid={{.pid}}">抽取逻辑</a></li>
-		</ol>
-    </section>
-  <!-- Main content -->
-  <section class="content">
-      <div class="row">
-	      <div class="col-xs-12">
-	        <div class="box">
-		        <div class="box-body">
-		            <table id="dataTable" class="table table-bordered table-hover">
-		              <thead>
-		              <tr>
-						<th>名称</th>
-						<th>创建人</th>
-						<th>描述</th>
-						<th>是否启用</th>
-						<th>配置规则</th>
-						<th>操作</th>
-		              </tr>
-		              </thead>
-		            </table>
-		        </div>
-	          <!-- /.box-body -->
-	        </div>
-        <!-- /.box -->
-		</div>
-	</div>
-  </section>
-</div>
-{{template "luares"}}
-{{template "dialog"}}
-{{template "footer"}}
-
-<script>
-menuActive("version")
-$(function () {
-	ttable=$('#dataTable').DataTable({
-		"paging"      : false,
-		"lengthChange": false,
-		"searching"   : true,
-		"ordering"    : false,
-		"info"        : true,
-		"autoWidth"   : false,
-		"ajax": {
-			"url": "/admin/pkglogic/data",
-			"type": "post",
-			"data":{"vid":{{ .vid}} ,"pid":{{ .pid}} }
-		 },
-		"language": {
-            "url": "/res/dist/js/dataTables.chinese.lang"
-        },
-		"columns": [
-			{ "data": "s_name"},
-			{ "data": "s_username"},
-			{ "data": "s_descript"},
-			{ "data": "isuse",render:function(val,a,row){
-				tmp=""
-				if(val){
-					tmp="<a href='#' title='停用' onclick='use(\""+row._id+"\",false)'><i class='fa fa-fw fa-circle text-green'></i></a>已启用"
-				}else{
-					tmp="<a href='#' title='启用' onclick='use(\""+row._id+"\",true)'><i class='fa fa-fw fa-circle text-red'></i></a>未启用"
-				}
-				return tmp
-			}},
-			{"data":"_id","width":"25%",render:function(val,a,row){
-				tmp = '<div class="btn-group">'+
-					'<a class="btn btn-sm btn-info" href="/admin/pkglogicback?vid='+{{.vid}}+'&pid='+{{.pid}}+'&sid='+row._id+'">后置规则</a>'+
-					'</div>';
-				return  tmp
-			}},
-			{"data":"s_version","width":"15%",render:function(val,a,row,pos){
-				return '<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>'+
-					'&nbsp;&nbsp;<a class="btn btn-sm btn-warning" onclick="del(\''+row._id+'\')">删除</a>';
-			}}
-       	]
-	});
-	ttable.on('init.dt', function () {
-		$(".opr").click(function(){
-			var n=$(this).attr("opr")
-			var htmlObj={},obj,tag=[]
-			var _tit="" 
-			switch(n){
-			case "edit":			
-				obj=ttable.row($(this).closest("tr")).data()
-			case "editlua":
-				obj=ttable.row($(this).closest("tr")).data()
-			case "new":
-				tag=[{label:"名称",s_label:"s_name",placeholder:"",must:true},
-					{label:"描述",s_label:"s_descript",type:"tpl_text"},
-					{label:"启用",s_label:"isuse",type:"tpl_list_local",list:[{"s_name":"是","_id":true},{"s_name":"否","_id":false}],default:true},
-					{label:"标题抽取",s_label:"extfrom",type:"tpl_list_local",list:[{"s_name":"是","_id":true},{"s_name":"否","_id":false}],default:false},
-					{label:"是否适用",s_label:"s_luascript",type:"tpl_text",must:true},
-					{s_label:"_id",type:"tpl_hidden"},
-					{s_label:"vid",type:"tpl_hidden"},{s_label:"pid",type:"tpl_hidden"}]
-				if(n=="edit"){
-					_tit="编辑-"+obj.s_name
-				}else{
-					_tit="新增逻辑"
-					obj={"s_luascript":"function logic(doc)\n\treturn true\nend","vid":"{{.vid}}","pid":"{{.pid}}","s_type":"0"}
-				}
-				
-				htmlObj={
-					title:_tit,
-					tag:tag,
-					lua:true,
-					bts:[
-						{label:"保存",class:"btn-primary",
-							fun:function(){
-								var obj={}
-								var bcon=true
-								$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
-									var val=$(el).val();
-									if(el.id=="s_luascript"){
-										val=editor_1.getValue()
-										obj[el.id]=val
-									}else{
-										obj[el.id]=$(el).val()
-									}
-									if(el.id!="_id"&&$(el).attr("must")&&!val){
-										bcon=false
-										return false
-									}
-								})
-								if (bcon){								
-									$.post("/admin/pkglogic/save",obj,function(data){
-										if(data&&data.rep){
-											window.location.href="/admin/pkglogic?vid={{.vid}}&pid={{.pid}}"	
-										}else{
-											showTip(data.msg,1000)
-										}
-									},'json')
-								}else{
-									alert("红色标签的表单不能为空！")
-								}
-							}
-						}
-					]
-				}
-			OpenDialog(htmlObj,obj)
-			break;
-			}
-		});
-	})
-})
-function del(_id){
-	showConfirm("确定删除？", function() {
-		$.ajax({
-			url:"/admin/pkgrulelogic/del",
-			type:"post",
-			data:{"_id":_id},
-			success:function(r){
-				if(r.rep){				
-					window.location.href="/admin/pkglogic?vid={{.vid}}&pid={{.pid}}";
-				}else{
-					showTip("删除失败", 1000);
-				}
-			}
-		})
-	});
-}
-function use(_id,utype){
-	smg=""
-	if(utype){
-		smg="确定启用？"
-	}else{
-		smg="确定停用？"
-	}
-	showConfirm(smg, function() {
-		$.ajax({
-			url:"/admin/pkglogic/use",
-			type:"post",
-			data:{"_id":_id,"isuse":utype},
-			success:function(r){
-				if(r.rep){				
-					window.location.href="/admin/pkglogic?vid={{.vid}}&pid={{.pid}}"
-				}else{
-					showTip("启用失败", 1000, function() {});
-				}
-			}
-		})
-	});
-}
-</script>

src/web/templates/admin/pkg_logicore.html

@@ -0,0 +1,308 @@
+{{template "inc"}}
+<!-- Main Header -->
+{{template "header"}}
+<!-- Left side column. 权限菜单 -->
+{{template "memu"}}
+
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>
+            <small><a class="btn btn-primary opr" opr="new">新增正则</a></small>
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="/admin/version"><i class="fa fa-dashboard"></i>版本管理</a></li>
+            <li><a href="/admin/version/pkginfo?vid={{.vid}}">分包属性</a></li>
+            <li class="active"><a
+                        href="/admin/version/pkglogicore?vid={{.vid}}&sid={{.sid}}&s_field={{.field}}">抽取规则</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="row">
+            <div class="col-xs-12">
+                <div class="box">
+                    <div class="box-body">
+                        <table id="dataTable" class="table table-bordered table-hover">
+                            <thead>
+                            <tr>
+                                <th>代码</th>
+                                <th>名称</th>
+                                <th>创建人</th>
+                                <th>描述</th>
+                                <th>类型</th>
+                                <th>是否启用</th>
+                                <th>操作</th>
+                            </tr>
+                            </thead>
+                        </table>
+                    </div>
+                    <!-- /.box-body -->
+                </div>
+                <!-- /.box -->
+            </div>
+        </div>
+    </section>
+    <!--
+    <div class="form-group"><select class="form-control select2" multiple="multiple" data-placeholder="Select a State" style="width: 100%;"><option>Alabama</option><option>Alaska</option></select></div>
+    -->
+</div>
+{{template "luares"}}
+{{template "dialog"}}
+{{template "footer"}}
+
+<script>
+    $('.select2').select2()
+    menuActive("version")
+    $(function () {
+        ttable = $('#dataTable').DataTable({
+            "paging": false,
+            "lengthChange": false,
+            "searching": true,
+            "ordering": false,
+            "info": true,
+            "autoWidth": false,
+            "ajax": {
+                "url": "/admin/version/pkglogicore/data",
+                "type": "post",
+                "data": {"s_field":{{.field}},"vid":{{.vid}}, "sid":{{ .sid}} },
+            },
+            "language": {
+                "url": "/res/dist/js/dataTables.chinese.lang"
+            },
+            "columns": [
+                {"data": "s_code"},
+                {"data": "s_name"},
+                {"data": "s_username"},
+                {"data": "s_descript"},
+                {
+                    "data": "s_type", render: function (val, a, row) {
+                        if (val == "0") {
+                            return "正则"
+                        } else {
+                            return "lua脚本"
+                        }
+                    }
+                },
+                {
+                    "data": "isuse", render: function (val, a, row) {
+                        tmp = ""
+                        if (val) {
+                            tmp = "<a href='#' title='停用' onclick='use(\"" + row._id + "\",false)'><i class='fa fa-fw fa-circle text-green'></i></a>已启用"
+                        } else {
+                            tmp = "<a href='#' title='启用' onclick='use(\"" + row._id + "\",true)'><i class='fa fa-fw fa-circle text-red'></i></a>未启用"
+                        }
+                        return tmp
+                    }
+                },
+                {
+                    "data": "_id", render: function (val, a, row, pos) {
+                        tmp = ""
+                        if (row.s_type == "0") {
+                            tmp = '<div>' +
+                                '<a class="btn btn-sm btn-primary opr" opr="edit" row="' + pos.row + '" >编辑</a>　' +
+                                '<a class="btn btn-sm btn-warning" onclick="del(\'' + val + '\')">删除</a>　' +
+                                '</div>';
+                        } else {
+                            tmp = '<div>' +
+                                '<a class="btn btn-sm btn-primary opr" opr="editlua" row="' + pos.row + '" >编辑</a>　' +
+                                '<a class="btn btn-sm btn-warning" onclick="del(\'' + val + '\')">删除</a>　' +
+                                '</div>';
+                        }
+                        return tmp
+                    }
+                }
+            ]
+        });
+        ttable.on('init.dt', function () {
+            $(".opr").click(function () {
+                var n = $(this).attr("opr")
+                var htmlObj = {}, obj, tag = []
+                var bts = [{
+                    label: "保存", class: "btn-primary",
+                    fun: function () {
+                        var obj = {}
+                        var bcon = true
+                        $("#_con").find("input[id!=s_show],textarea").each(function (i, el) {
+                            var val = $(el).val();
+                            if (el.id == "s_luascript") {
+                                val = editor_1.getValue()
+                                obj[el.id] = val
+                            } else {
+                                obj[el.id] = $(el).val()
+                            }
+                            if (el.id != "_id" && $(el).attr("must") && !val) {
+                                bcon = false
+                                return false
+                            }
+                        })
+                        if (bcon) {
+                            $.post("/admin/version/pkglogicore/save", obj, function (data) {
+                                if (data && data.rep) {
+                                    window.location.href = "/admin/version/pkglogicore?vid={{.vid}}&sid={{.sid}}&s_field={{.field}}"
+                                } else {
+                                    showTip(data.msg, 1000)
+                                }
+                            }, 'json')
+                        } else {
+                            alert("红色标签的表单不能为空！")
+                        }
+                    }
+                }
+                ]
+                var _tit = ""
+                switch (n) {
+                    case "edit":
+                        obj = ttable.row($(this).closest("tr")).data();
+                    case "editlua":
+                        obj = ttable.row($(this).closest("tr")).data();
+                    case "newlua":
+                    case "new":
+                        comtag = [{label: "名称", s_label: "s_name", placeholder: "", must: true}, {
+                            label: "描述",
+                            s_label: "s_descript",
+                            type: "tpl_text"
+                        }, {
+                            label: "启用",
+                            s_label: "isuse",
+                            type: "tpl_list_local",
+                            list: [{"s_name": "是", "_id": true}, {"s_name": "否", "_id": false}],
+                            default: true
+                        }]
+                        regtag = [{
+                            label: "字段",
+                            s_label: "s_field",
+                            type: "tpl_list_local",
+                            url: "/admin/getfields",
+                            default:{{.field}}}, {label: "正则", s_label: "s_rule", type: "tpl_text", must: true}]
+                        luatag = [{
+                            label: "脚本",
+                            s_label: "s_field",
+                            type: "tpl_list_local",
+                            url: "/admin/getfields",
+                            default:{{.field}}}, {label: "脚本", s_label: "s_luascript", type: "tpl_text", must: true}]
+                        testcon = [{label: "测试内容", s_label: "s_testcon", type: "tpl_text", must: true}]
+                        hiddentag = [{s_label: "_id", type: "tpl_hidden"}, {
+                            s_label: "vid",
+                            type: "tpl_hidden"
+                        }, {s_label: "s_field", type: "tpl_hidden"}, {
+                            s_label: "sid",
+                            type: "tpl_hidden"
+                        }, {s_label: "s_type", type: "tpl_hidden"}]
+                        islua = false
+                        tag = com.pushArry(tag, comtag)
+                        if (n == "edit") {
+                            _tit = "编辑-" + obj.s_name
+                            tag = com.pushArry(tag, regtag)
+                            tag = com.pushArry(tag, hiddentag)
+                            tag = com.pushArry(tag, testcon)
+                            check = [{
+                                label: "测试", class: "btn-warning",
+                                fun: function () {
+                                    var obj = {}
+                                    var bcon = true
+                                    $("#_con").find("input[id!=s_show],textarea").each(function (i, el) {
+                                        var val = $(el).val();
+                                        obj[el.id] = $(el).val()
+                                        if (el.id != "_id" && $(el).attr("must") && !val) {
+                                            bcon = false
+                                            return false
+                                        }
+                                    })
+                                    if (bcon) {
+                                        $.post("/admin/check/extrule", obj, function (data) {
+                                            showMsg(JSON.stringify(data.rep))
+                                        }, 'json')
+                                    } else {
+                                        alert("红色标签的表单不能为空！")
+                                    }
+                                }
+                            }]
+                            bts = com.pushArry(bts, check)
+                        } else if (n == "editlua") {
+                            _tit = "编辑-" + obj.s_name
+                            islua = true
+                            tag = com.pushArry(tag, luatag)
+                            tag = com.pushArry(tag, hiddentag)
+                            check = [{
+                                label: "测试", class: "btn-warning",
+                                fun: function () {
+                                    window.location.href = "/admin/check/lua/core?vid={{.vid}}&sid={{.sid}}&s_code=" + obj.s_code
+                                }
+                            }]
+                            bts = com.pushArry(bts, check)
+                        } else {
+                            _tit = "新增规则"
+                            if (n == "newlua") {
+                                _tit = "新增脚本"
+                                obj = {
+                                    "s_luascript": "--code脚本代码，doc数据源，block块对象，kvs抽取kv对象，返回kvs对象，kvs结构不可改变\nfunction main(code,doc,block,kvs)\n\t--自定义抽取\n\treturn kvs\nend",
+                                    "vid": "{{.vid}}",
+                                    "s_field": "{{.field}}",
+                                    "sid": "{{.sid}}",
+                                    "s_type": "1"
+                                }
+
+                                tag = com.pushArry(tag, luatag)
+                                tag = com.pushArry(tag, hiddentag)
+                                islua = true
+                            } else {
+                                obj = {"vid": "{{.vid}}", "s_field":{{.field}}, "sid": "{{.sid}}", "s_type": "0"}
+                                tag = com.pushArry(tag, regtag)
+                                tag = com.pushArry(tag, hiddentag)
+                            }
+                        }
+                        htmlObj = {
+                            title: _tit,
+                            tag: tag,
+                            lua: islua,
+                            bts: bts
+                        }
+                        OpenDialog(htmlObj, obj)
+                        break;
+                }
+            });
+        })
+    })
+
+    function del(_id) {
+        showConfirm("确定删除？", function () {
+            $.ajax({
+                url: "/admin/version/pkglogicore/del",
+                type: "post",
+                data: {"_id": _id},
+                success: function (r) {
+                    if (r.rep) {
+                        window.location.href = "/admin/version/pkglogicore?vid={{.vid}}&sid={{.sid}}&s_field={{.field}}";
+                    } else {
+                        showTip("删除失败", 1000);
+                    }
+                }
+            })
+        });
+    }
+
+    function use(_id, utype) {
+        smg = ""
+        if (utype) {
+            smg = "确定启用？"
+        } else {
+            smg = "确定停用？"
+        }
+        showConfirm(smg, function () {
+            $.ajax({
+                url: "/admin/version/pkglogicore/use",
+                type: "post",
+                data: {"_id": _id, "isuse": utype},
+                success: function (r) {
+                    if (r.rep) {
+                        window.location.reload()
+                    } else {
+                        showTip("启用失败", 1000, function () {
+                        });
+                    }
+                }
+            })
+        });
+    }
+</script>

src/web/templates/admin/rule_logickv.html

@@ -0,0 +1,256 @@
+{{template "inc"}}
+<!-- Main Header -->
+{{template "header"}}
+<!-- Left side column. 权限菜单 -->
+{{template "memu"}}
+
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增正则</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  <li><a href="/admin/version"><i class="fa fa-dashboard"></i>抽取版本</a></li>
+		  <li><a href="/admin/version/info?vid={{.vid}}&pid={{.pid}}">属性配置</a></li>
+		  <li><a href="/admin/rulelogic?vid={{.vid}}&pid={{.pid}}">抽取逻辑</a></li>
+		  <li class="active"><a href="/admin/logickv?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}">kv规则</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+						<th>代码</th>
+		                <th>名称</th>
+						<th>创建人</th>
+						<th>描述</th>
+						<th>类型</th>
+						<th>是否启用</th>
+						<th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+<!--
+<div class="form-group"><select class="form-control select2" multiple="multiple" data-placeholder="Select a State" style="width: 100%;"><option>Alabama</option><option>Alaska</option></select></div>
+-->
+</div>
+{{template "luares"}}
+{{template "dialog"}}
+{{template "footer"}}
+  
+<script>
+$('.select2').select2()
+menuActive("version")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : true,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/admin/logickv/data",
+			"type": "post",
+			"data":{"vid":{{.vid}},"pid":{{.pid}},"sid":{{ .sid}} }
+		 },
+		"language": {
+            "url": "/res/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+			{ "data": "s_code"},
+            { "data": "s_name"},
+			{ "data": "s_username"},
+			{ "data": "s_descript"},
+			{ "data": "s_type",render:function(val,a,row){
+				if(val=="0"){
+					return "正则"
+				}else{
+					return "lua脚本"
+				}
+			}},
+			{ "data": "isuse",render:function(val,a,row){
+				tmp=""
+				if(val){
+					tmp="<a href='#' title='停用' onclick='use(\""+row._id+"\",false)'><i class='fa fa-fw fa-circle text-green'></i></a>已启用"
+				}else{
+					tmp="<a href='#' title='启用' onclick='use(\""+row._id+"\",true)'><i class='fa fa-fw fa-circle text-red'></i></a>未启用"
+				}
+				return tmp
+			}},
+			{ "data": "_id",render:function(val,a,row,pos){
+				tmp=""
+				if(row.s_type=="0"){
+					tmp = '<div>'+
+						'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+						'<a class="btn btn-sm btn-warning" onclick="del(\''+val+'\')">删除</a>　'+
+						'</div>';
+				}else{
+					tmp = '<div>'+
+						'<a class="btn btn-sm btn-primary opr" opr="editlua" row="'+pos.row+'" >编辑</a>　'+
+						'<a class="btn btn-sm btn-warning" onclick="del(\''+val+'\')">删除</a>　'+
+						'</div>';
+				}
+				return tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$(".opr").click(function(){
+			var n=$(this).attr("opr")
+			var htmlObj={},obj,tag=[]
+			var bts=[{label:"保存",class:"btn-primary",
+							fun:function(){
+								var obj={}
+								var bcon=true
+								$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+									var val=$(el).val();
+									if(el.id=="s_luascript"){
+										val=editor_1.getValue()
+										obj[el.id]=val
+									}else{
+										obj[el.id]=$(el).val()
+									}
+									if(el.id!="_id"&&$(el).attr("must")&&!val){
+										bcon=false
+										return false
+									}
+								})
+								if (bcon){								
+									$.post("/admin/logickv/save",obj,function(data){
+										if(data&&data.rep){
+											window.location.href="/admin/logickv?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}"
+										}else{
+											showTip(data.msg,1000)
+										}
+									},'json')
+								}else{
+									alert("红色标签的表单不能为空！")
+								}
+							}
+						}
+					]
+			var _tit="" 
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();
+			case "editlua":			
+				obj=ttable.row($(this).closest("tr")).data();
+			case "newlua":
+			case "new":
+				comtag=[{label:"名称",s_label:"s_name",placeholder:"",must:true},{label:"描述",s_label:"s_descript",type:"tpl_text"},{label:"启用",s_label:"isuse",type:"tpl_list_local",list:[{"s_name":"是","_id":true},{"s_name":"否","_id":false}],default:true}]
+				regtag=[{label:"字段",s_label:"s_field",type:"tpl_list_local",url:"/admin/getfields",default:{{.field}}},{label:"正则",s_label:"s_rule",type:"tpl_text",must:true}]
+				testcon=[{label:"测试内容",s_label:"s_testcon",type:"tpl_text",must:true}]
+				hiddentag=[{s_label:"_id",type:"tpl_hidden"},{s_label:"vid",type:"tpl_hidden"},{s_label:"pid",type:"tpl_hidden"},{s_label:"sid",type:"tpl_hidden"},{s_label:"s_type",type:"tpl_hidden"}]
+				islua=false
+				tag = com.pushArry(tag,comtag)
+				if(n=="edit"){
+					_tit="编辑-"+obj.s_name
+					tag = com.pushArry(tag,regtag)
+					tag = com.pushArry(tag,hiddentag)
+					tag = com.pushArry(tag,testcon)
+					check=[{label:"测试",class:"btn-warning",
+								fun:function(){
+									var obj={}
+									var bcon=true
+									$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+										var val=$(el).val(); 
+										obj[el.id]=$(el).val()
+										if(el.id!="_id"&&$(el).attr("must")&&!val){
+											bcon=false
+											return false
+										}
+									})
+									if (bcon){								
+										$.post("/admin/check/backrule",obj,function(data){
+											showMsg(JSON.stringify(data.rep))
+										},'json')
+									}else{
+										alert("红色标签的表单不能为空！")
+									}
+								}
+							}]
+					bts = com.pushArry(bts,check)
+				}else if(n=="editlua"){
+					_tit="编辑-"+obj.s_name
+					islua=true
+					tag = com.pushArry(tag,luatag)
+					tag = com.pushArry(tag,hiddentag)
+					check=[{label:"测试",class:"btn-warning",
+								fun:function(){
+									window.location.href="/admin/check/lua/ckv?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}&s_code="+obj.s_code
+								}
+							}]
+					bts = com.pushArry(bts,check)
+				}else{
+					_tit="新增规则"
+						obj={"vid":"{{.vid}}","pid":"{{.pid}}","sid":"{{.sid}}","s_type":"0"}
+						tag = com.pushArry(tag,regtag)
+						tag = com.pushArry(tag,hiddentag)
+
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					lua:islua,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+})
+
+function del(_id){
+	showConfirm("确定删除？", function() {
+		$.ajax({
+			url:"/admin/logickv/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				if(r.rep){				
+					window.location.href="/admin/logickv?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}";
+				}else{
+					showTip("删除失败", 1000);
+				}
+			}
+		})
+	});
+}
+function use(_id,utype){
+	smg=""
+	if(utype){
+		smg="确定启用？"
+	}else{
+		smg="确定停用？"
+	}
+	showConfirm(smg, function() {
+		$.ajax({
+			url:"/admin/logickv/use",
+			type:"post",
+			data:{"_id":_id,"isuse":utype},
+			success:function(r){
+				if(r.rep){				
+					window.location.reload()
+				}else{
+					showTip("启用失败", 1000, function() {});
+				}
+			}
+		})
+	});
+}
+</script>

src/web/templates/admin/rule_logiclist.html

@@ -81,7 +81,8 @@ $(function () {
 				tmp = '<div class="btn-group">'+
 					'<a class="btn btn-sm btn-primary" href="/admin/logicpre?vid={{.vid}}&pid={{.pid}}&sid='+row._id+'">前置规则</a>'+
 					'<a class="btn btn-sm btn-success" href="/admin/logicore?vid={{.vid}}&pid={{.pid}}&sid='+row._id+'">抽取规则</a>'+
-					'<a class="btn btn-sm btn-info" href="/admin/logicback?vid={{.vid}}&pid={{.pid}}&sid='+row._id+'">后置规则</a>'+
+						'<a class="btn btn-sm btn-info" href="/admin/logicback?vid={{.vid}}&pid={{.pid}}&sid='+row._id+'">后置规则</a>'+
+						'<a class="btn btn-sm btn-warning" href="/admin/logickv?vid={{.vid}}&pid={{.pid}}&sid='+row._id+'">kv规则</a>'+
 					'</div>';
 				return  tmp
 			}},

src/web/templates/admin/site_rule_logickv.html

@@ -0,0 +1,257 @@
+{{template "inc"}}
+<!-- Main Header -->
+{{template "header"}}
+<!-- Left side column. 权限菜单 -->
+{{template "memu"}}
+
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增正则</a></small>
+		</h1>
+		<ol class="breadcrumb">
+			<li><a href="/admin/version"><i class="fa fa-dashboard"></i>抽取版本</a></li>
+			<li><a href="/admin/site_management?vid={{.vid}}">站点列表</a></li>
+			<li><a href="/admin/site_management/info?vid={{.vid}}&pid={{.pzid}}">属性配置</a></li>
+			<li><a href="/admin/site_management/rulelogic?vid={{.vid}}&pid={{.pid}}">抽取逻辑</a></li>
+		  <li class="active"><a href="/admin/site_management/logickv?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}">抽取规则</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+						<th>代码</th>
+		                <th>名称</th>
+						<th>创建人</th>
+						<th>描述</th>
+						<th>类型</th>
+						<th>是否启用</th>
+						<th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+<!--
+<div class="form-group"><select class="form-control select2" multiple="multiple" data-placeholder="Select a State" style="width: 100%;"><option>Alabama</option><option>Alaska</option></select></div>
+-->
+</div>
+{{template "luares"}}
+{{template "dialog"}}
+{{template "footer"}}
+  
+<script>
+$('.select2').select2()
+menuActive("version")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : true,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/admin/site_management/logickv/data",
+			"type": "post",
+			"data":{"vid":{{.vid}},"pid":{{.pid}},"sid":{{ .sid}} }
+		 },
+		"language": {
+            "url": "/res/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+			{ "data": "s_code"},
+            { "data": "s_name"},
+			{ "data": "s_username"},
+			{ "data": "s_descript"},
+			{ "data": "s_type",render:function(val,a,row){
+				if(val=="0"){
+					return "正则"
+				}else{
+					return "lua脚本"
+				}
+			}},
+			{ "data": "isuse",render:function(val,a,row){
+				tmp=""
+				if(val){
+					tmp="<a href='#' title='停用' onclick='use(\""+row._id+"\",false)'><i class='fa fa-fw fa-circle text-green'></i></a>已启用"
+				}else{
+					tmp="<a href='#' title='启用' onclick='use(\""+row._id+"\",true)'><i class='fa fa-fw fa-circle text-red'></i></a>未启用"
+				}
+				return tmp
+			}},
+			{ "data": "_id",render:function(val,a,row,pos){
+				tmp=""
+				if(row.s_type=="0"){
+					tmp = '<div>'+
+						'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+						'<a class="btn btn-sm btn-warning" onclick="del(\''+val+'\')">删除</a>　'+
+						'</div>';
+				}else{
+					tmp = '<div>'+
+						'<a class="btn btn-sm btn-primary opr" opr="editlua" row="'+pos.row+'" >编辑</a>　'+
+						'<a class="btn btn-sm btn-warning" onclick="del(\''+val+'\')">删除</a>　'+
+						'</div>';
+				}
+				return tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$(".opr").click(function(){
+			var n=$(this).attr("opr")
+			var htmlObj={},obj,tag=[]
+			var bts=[{label:"保存",class:"btn-primary",
+							fun:function(){
+								var obj={}
+								var bcon=true
+								$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+									var val=$(el).val();
+									if(el.id=="s_luascript"){
+										val=editor_1.getValue()
+										obj[el.id]=val
+									}else{
+										obj[el.id]=$(el).val()
+									}
+									if(el.id!="_id"&&$(el).attr("must")&&!val){
+										bcon=false
+										return false
+									}
+								})
+								if (bcon){								
+									$.post("/admin/site_management/logickv/save",obj,function(data){
+										if(data&&data.rep){
+											window.location.href="/admin/site_management/logickv?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}"
+										}else{
+											showTip(data.msg,1000)
+										}
+									},'json')
+								}else{
+									alert("红色标签的表单不能为空！")
+								}
+							}
+						}
+					]
+			var _tit="" 
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();
+			case "editlua":			
+				obj=ttable.row($(this).closest("tr")).data();
+			case "newlua":
+			case "new":
+				comtag=[{label:"名称",s_label:"s_name",placeholder:"",must:true},{label:"描述",s_label:"s_descript",type:"tpl_text"},{label:"启用",s_label:"isuse",type:"tpl_list_local",list:[{"s_name":"是","_id":true},{"s_name":"否","_id":false}],default:true}]
+				regtag=[{label:"字段",s_label:"s_field",type:"tpl_list_local",url:"/admin/getfields",default:{{.field}}},{label:"正则",s_label:"s_rule",type:"tpl_text",must:true}]
+				testcon=[{label:"测试内容",s_label:"s_testcon",type:"tpl_text",must:true}]
+				hiddentag=[{s_label:"_id",type:"tpl_hidden"},{s_label:"vid",type:"tpl_hidden"},{s_label:"pid",type:"tpl_hidden"},{s_label:"sid",type:"tpl_hidden"},{s_label:"s_type",type:"tpl_hidden"}]
+				islua=false
+				tag = com.pushArry(tag,comtag)
+				if(n=="edit"){
+					_tit="编辑-"+obj.s_name
+					tag = com.pushArry(tag,regtag)
+					tag = com.pushArry(tag,hiddentag)
+					tag = com.pushArry(tag,testcon)
+					check=[{label:"测试",class:"btn-warning",
+								fun:function(){
+									var obj={}
+									var bcon=true
+									$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+										var val=$(el).val(); 
+										obj[el.id]=$(el).val()
+										if(el.id!="_id"&&$(el).attr("must")&&!val){
+											bcon=false
+											return false
+										}
+									})
+									if (bcon){								
+										$.post("/admin/check/backrule",obj,function(data){
+											showMsg(JSON.stringify(data.rep))
+										},'json')
+									}else{
+										alert("红色标签的表单不能为空！")
+									}
+								}
+							}]
+					bts = com.pushArry(bts,check)
+				}else if(n=="editlua"){
+					_tit="编辑-"+obj.s_name
+					islua=true
+					tag = com.pushArry(tag,luatag)
+					tag = com.pushArry(tag,hiddentag)
+					check=[{label:"测试",class:"btn-warning",
+								fun:function(){
+									window.location.href="/admin/site_management/check/lua/core?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}&s_code="+obj.s_code
+								}
+							}]
+					bts = com.pushArry(bts,check)
+				}else{
+					_tit="新增规则"
+
+						obj={"vid":"{{.vid}}","pid":"{{.pid}}","sid":"{{.sid}}","s_type":"0"}
+						tag = com.pushArry(tag,regtag)
+						tag = com.pushArry(tag,hiddentag)
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					lua:islua,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+})
+
+function del(_id){
+	showConfirm("确定删除？", function() {
+		$.ajax({
+			url:"/admin/site_management/logickv/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				if(r.rep){				
+					window.location.href="/admin/site_management/logickv?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}";
+				}else{
+					showTip("删除失败", 1000);
+				}
+			}
+		})
+	});
+}
+function use(_id,utype){
+	smg=""
+	if(utype){
+		smg="确定启用？"
+	}else{
+		smg="确定停用？"
+	}
+	showConfirm(smg, function() {
+		$.ajax({
+			url:"/admin/site_management/logickv/use",
+			type:"post",
+			data:{"_id":_id,"isuse":utype},
+			success:function(r){
+				if(r.rep){				
+					window.location.reload()
+				}else{
+					showTip("启用失败", 1000, function() {});
+				}
+			}
+		})
+	});
+}
+</script>

src/web/templates/admin/site_rule_logiclist.html

@@ -83,6 +83,7 @@ $(function () {
 						'<a class="btn btn-sm btn-primary" href="/admin/site_management/logicpre?vid={{.vid}}&pid={{.pid}}&sid='+row._id+'">前置规则</a>'+
 						'<a class="btn btn-sm btn-success" href="/admin/site_management/logicore?vid={{.vid}}&pid={{.pid}}&sid='+row._id+'">抽取规则</a>'+
 						'<a class="btn btn-sm btn-info" href="/admin/site_management/logicback?vid={{.vid}}&pid={{.pid}}&sid='+row._id+'">后置规则</a>'+
+						'<a class="btn btn-sm btn-warning" href="/admin/site_management/logickv?vid={{.vid}}&pid={{.pid}}&sid='+row._id+'">kv规则</a>'+
 						'</div>';
 				return  tmp
 			}},

udpcreateindex/src/biddingall.go

@@ -218,6 +218,24 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 									}
 								}
 								newTmp[v] = newmap
+								attachments := mp["attachments"]
+								con := ""
+								if attachments != nil {
+									am, _ := attachments.(map[string]interface{})
+									if am != nil {
+										for _, v1 := range am {
+											vm, _ := v1.(map[string]interface{})
+											if vm != nil {
+												c, _ := vm["content"].(string)
+												con += c
+											}
+										}
+									}
+								}
+								if con != "" {
+									con = FilterDetailSpace(con)
+									newTmp["attachments"] = con
+								}
 							}
 						} else {
 							if v == "detail" {

udpcreateindex/src/biddingdata.go

@@ -245,6 +245,24 @@ func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
 									}
 								}
 								newTmp[v] = newmap
+								attachments := mp["attachments"]
+								con := ""
+								if attachments != nil {
+									am, _ := attachments.(map[string]interface{})
+									if am != nil {
+										for _, v1 := range am {
+											vm, _ := v1.(map[string]interface{})
+											if vm != nil {
+												c, _ := vm["content"].(string)
+												con += c
+											}
+										}
+									}
+								}
+								if con != "" {
+									con = FilterDetailSpace(con)
+									newTmp["attachments"] = con
+								}
 							}
 						} else {
 							if v == "detail" {

udpcreateindex/src/biddingindex.go

@@ -222,6 +222,24 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 								}
 							}
 							newTmp[v] = newmap
+							attachments := mp["attachments"]
+							con := ""
+							if attachments != nil {
+								am, _ := attachments.(map[string]interface{})
+								if am != nil {
+									for _, v1 := range am {
+										vm, _ := v1.(map[string]interface{})
+										if vm != nil {
+											c, _ := vm["content"].(string)
+											con += c
+										}
+									}
+								}
+							}
+							if con != "" {
+								con = FilterDetailSpace(con)
+								newTmp["attachments"] = con
+							}
 						}
 					} else {
 						if v == "detail" {

udpcreateindex/src/biddingindexback.go

@@ -171,7 +171,12 @@ func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
 }
 
 var filterReg = regexp.MustCompile("<[^>]+>")
+var filterSpace = regexp.MustCompile("<[^>]*?>|[\\s\u3000\u2003\u00a0]")
 
 func FilterDetail(text string) string {
 	return filterReg.ReplaceAllString(text, "")
 }
+
+func FilterDetailSpace(text string) string {
+	return filterSpace.ReplaceAllString(text, "")
+}

udpcreateindex/src/biddingindexback2.go

@@ -0,0 +1,168 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	qutil "qfw/util"
+	elastic "qfw/util/elastic"
+	"strings"
+	//elastic "qfw/util/elastic_v5"
+	//	"strings"
+	"sync"
+	"time"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+func biddingBackTask2(data []byte, mapInfo map[string]interface{}) {
+	defer qutil.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  qutil.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": qutil.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	c, _ := mapInfo["coll"].(string)
+	if c == "" {
+		c, _ = biddingback["collect"].(string)
+	}
+	cs := strings.Split(c, ",")
+	for _, c := range cs {
+		//bidding库
+		session := mgo.GetMgoConn(86400)
+		defer mgo.DestoryMongoConn(session)
+		//连接信息
+
+		db, _ := biddingback["db"].(string)
+		index, _ := biddingback["index"].(string)
+		itype, _ := biddingback["type"].(string)
+		count, _ := session.DB(db).C(c).Find(&q).Count()
+		//线程池
+		UpdatesLock := sync.Mutex{}
+		log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+		query := session.DB(db).C(c).Find(q).Select(bson.M{
+			"contenthtml": 0,
+			"s_sha":       0,
+		}).Sort("_id").Iter()
+		//查询抽取结果
+		n := 0
+		//更新数组
+		arrEs := []map[string]interface{}{}
+		//对比两张表数据，减少查询次数
+		thread := qutil.IntAll(mapInfo["thread"])
+		//不传为0只生成招标索引,1生成招标+预览,2只生成预览
+		if thread < 1 {
+			thread = 3
+		}
+		log.Println("es线程数:", thread)
+		espool := make(chan bool, thread)
+		now1 := time.Now().Unix()
+		for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+			if qutil.IntAll(tmp["extracttype"]) == -1 {
+				tmp = make(map[string]interface{})
+				continue
+			}
+			ct := qutil.Int64All(tmp["comeintime"])
+			pt := qutil.Int64All(tmp["publishtime"])
+			if pt > ct+86400 || pt > now1 { //时间问题,需要更新
+				if ct > now1 {
+					ct = now1
+				}
+				tmp["publishtime"] = ct
+			}
+
+			ps, _ := tmp["projectscope"].(string)
+			if ps == "" {
+				tmp["projectscope"] = "" //= tmp["detail"]
+			}
+			if len(ps) > ESLEN {
+				tmp["projectscope"] = string(([]rune(ps))[:4000])
+			}
+			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
+				tmp["budget"] = nil
+			} else if sbd, ok := tmp["budget"].(string); ok {
+				tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+			}
+			if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
+				tmp["bidamount"] = nil
+			} else if sbd, ok := tmp["bidamount"].(string); ok {
+				tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+			}
+			UpdatesLock.Lock()
+			newTmp := map[string]interface{}{}
+			for _, v := range biddingIndexFields {
+				if tmp[v] != nil {
+					if "projectinfo" == v {
+						//处理附件 content
+						mp, _ := tmp[v].(map[string]interface{})
+						if mp != nil {
+							newmap := map[string]interface{}{}
+							for _, v1 := range projectinfoFields {
+								if mp[v1] != nil {
+									newmap[v1] = mp[v1]
+								}
+							}
+							if len(newmap) > 0 {
+								newTmp[v] = newmap
+							}
+							attachments := mp["attachments"]
+							con := ""
+							if attachments != nil {
+								am, _ := attachments.(map[string]interface{})
+								if am != nil {
+									for _, v1 := range am {
+										vm, _ := v1.(map[string]interface{})
+										if vm != nil {
+											c, _ := vm["content"].(string)
+											con += c
+										}
+									}
+								}
+							}
+							if con != "" {
+								con = FilterDetailSpace(con)
+								newTmp["attachments"] = con
+							}
+						}
+					} else {
+						if v == "detail" {
+							detail, _ := tmp[v].(string)
+							newTmp[v] = FilterDetail(detail)
+						} else {
+							newTmp[v] = tmp[v]
+						}
+					}
+				} else if v == "budget" || v == "bidamount" {
+					newTmp[v] = nil
+				}
+			}
+			arrEs = append(arrEs, newTmp)
+			if len(arrEs) >= BulkSizeBack {
+				tmps := arrEs
+				espool <- true
+				go func(tmps []map[string]interface{}) {
+					defer func() {
+						<-espool
+					}()
+					elastic.BulkSave(index, itype, &tmps, true)
+				}(tmps)
+				arrEs = []map[string]interface{}{}
+			}
+			UpdatesLock.Unlock()
+			if n%1000 == 0 {
+				log.Println("current:", n, qutil.BsonIdToSId(tmp["_id"]))
+			}
+			tmp = make(map[string]interface{})
+		}
+		UpdatesLock.Lock()
+		if len(arrEs) > 0 {
+			tmps := arrEs
+			elastic.BulkSave(index, itype, &tmps, true)
+		}
+		UpdatesLock.Unlock()
+		log.Println(mapInfo, "create biddingback2 index...over", c, n)
+	}
+}

udpfilterdup/src/datamap.go

@@ -14,29 +14,31 @@ import (
 
 type Info struct {
 	id                 string
-	title              string
-	area               string
-	city               string
-	subtype            string
-	buyer              string
-	agency             string //代理机构
-	winner             string //中标单位
-	budget             float64 //预算金额
-	bidamount		   float64//中标金额
-	projectname        string
-	projectcode        string
-	publishtime        int64
-	comeintime         int64
-	bidopentime        int64 //开标时间
-	agencyaddr		   string//开标地点
-	detail		   	   string//招标内容
-	site			   string//站点
-	href			   string//正文的url
-	titleSpecialWord bool //标题特殊次
-	specialWord bool	  //再次判断的特殊次
+	title              string		//标题
+	area               string		//省份
+	city               string		//城市
+	subtype            string		//信息类型
+	buyer              string		//采购单位
+	agency             string 		//代理机构
+	winner             string 		//中标单位
+	budget             float64 		//预算金额
+	bidamount		   float64		//中标金额
+	projectname        string		//项目名称
+	projectcode        string		//项目编号
+	publishtime        int64		//发布时间
+	comeintime         int64		//采集时间
+	bidopentime        int64 		//开标时间
+	agencyaddr		   string		//开标地点
+	detail		   	   string		//招标内容
+	site			   string		//站点
+	href			   string		//正文的url
+	titleSpecialWord bool 			//标题特殊词
+	specialWord bool	 			//再次判断的特殊词
+	mergemap           map[string]interface{}   //合并记录
 }
 
 var datelimit = float64(432000)
+var reason string //判重记录
 
 type datamap struct {
 	lock   sync.Mutex //锁
@@ -61,11 +63,10 @@ func NewDatamap(days int, lastid string) *datamap {
 	now1 := int64(0)
 	n, continuSum := 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
-		//|| qutil.ObjToString(tmp["subtype"]) == "变更"
 		if qutil.IntAll(tmp["repeat"]) == 1 || qutil.IntAll(tmp["repeat"]) == -1{
 			continuSum++
 		} else {
-			cm := tmp["comeintime"] //时间单位
+			cm := tmp["comeintime"] //时间单位?
 			//cm := tmp["publishtime"]
 			comeintime := qutil.Int64All(cm)
 			if comeintime == 0 {
@@ -77,7 +78,9 @@ func NewDatamap(days int, lastid string) *datamap {
 			}
 			if qutil.Float64All(now1-comeintime) < datelimit {
 				info := NewInfo(tmp)
+				//时间字符串
 				dkey := qutil.FormatDateWithObj(&cm, qutil.Date_yyyyMMdd)
+				//拼接的一个时间字符串 xxxx_类型_省份
 				k := fmt.Sprintf("%s_%s_%s", dkey, info.subtype, info.area)
 				data := dm.data[k]
 				if data == nil {
@@ -130,15 +133,20 @@ func NewInfo(tmp map[string]interface{}) *Info {
 	info.detail		= qutil.ObjToString(tmp["detail"])
 	info.site	 = qutil.ObjToString(tmp["site"])
 	info.href	 = qutil.ObjToString(tmp["href"])
-	return info
-}
+	info.mergemap = *qutil.ObjToMap(tmp["merge_map"])
 
 
-func (d *datamap) check(info *Info) (b bool, id string) {
+	return info
+}
+// 486 396 315
+func (d *datamap) check(info *Info) (b bool,  source *Info,reasons string) {
+	reason = ""
 	d.lock.Lock()
 	defer d.lock.Unlock()
 	keys := []string{}
+	//不同时间段
 	for k, _ := range d.keys {
+		//...代码
 		keys = append(keys, fmt.Sprintf("%s_%s_%s", k, info.subtype, info.area))
 		if info.area != "全国" { //这个后续可以不要
 			keys = append(keys, fmt.Sprintf("%s_%s_%s", k, info.subtype, "全国"))
@@ -147,221 +155,69 @@ func (d *datamap) check(info *Info) (b bool, id string) {
 L:
 	for _, k := range keys {
 		data := d.data[k]
-		if len(data) > 0 { //对比
+		if len(data) > 0 { //对比v   找到同类型，同省或全国的数据作对比
 			for _, v := range data {
-				//正常重复
-				if v.id == info.id {
-
-					return false, v.id
+				if v.id == info.id {//正常重复
+					return false, v,""
 				}
 				if math.Abs(qutil.Float64All(v.publishtime-info.publishtime)) > datelimit {
-					continue
-				}
-				if v.agency != "" && info.agency != "" && v.agency != info.agency {
-					continue
+					continue   //是否为5天内数据
 				}
 
-				if info.subtype==v.subtype {
-					if info.subtype == "变更" {
-						//以下为新增方法 ， 变更数据判重处理   v为原数据  info为目标数据
-						if info.publishtime<v.publishtime{
-							continue
-						}
-						if info.titleSpecialWord&&info.title!=v.title&&v.title!="" {
-							continue
-						}
-						if v.projectcode != info.projectcode&&len([]rune(info.projectcode)) >=10&&v.projectcode!=""{
-							continue
-						}
+				//if v.agency != "" && info.agency != "" && v.agency != info.agency {
+				//	continue
+				//}
 
-						//同城判定有效
-						first_judge:= false
-						if (v.projectcode != ""&&v.projectcode==info.projectcode&&v.projectname != ""&&v.projectname==info.projectname)||
-							(v.projectcode != ""&&v.projectcode==info.projectcode&&v.bidopentime != 0&&v.bidopentime==info.bidopentime)||
-							(v.title != ""&&v.title==info.title&&v.agencyaddr != ""&&v.agencyaddr==info.agencyaddr&&v.bidopentime != 0&&v.bidopentime==info.bidopentime) {
-							first_judge = true
-
-						}
-						//3/6等判断
-						n := 0
-						if v.title != "" && v.title == info.title {
-							n++
-						}
-						if v.projectname != "" && v.projectname == info.projectname {
-							n++
-						}
-						if v.projectcode != "" && v.projectcode == info.projectcode {
-							n++
-						}
-						if v.bidopentime != 0 && v.bidopentime == info.bidopentime {
-							n++
-						}
-						if v.agencyaddr != "" && v.agencyaddr == info.agencyaddr {
-							n++
-						}
-						if v.detail != "" && v.detail == info.detail {
-							n++
-						}
-						t:= judgeCityType(v.area,info.area,v.city,info.city)
+				//指定该范围内数据判重  jsondata
+				if info.agency=="" {
+					if v.agency=="" {
+						continue
+					}
+				}
 
-						//同站点url判重
-						if info.site != "" && v.site == info.site {
-							if v.href==info.href&&info.href!="" {
-								b = true
-								id = v.id
-								log.Println("href满足过滤:",info.id)
-								break L
-							}else {
+				//是否走站内判重
+				if info.site != "" && v.site == info.site {
+					//独有判重... 待定  jsondata配置
+				}
+				//类型分组-相同类型继续
+				if info.subtype==v.subtype {
+					//代理机构相同-非空相等
+					if v.agency != "" && info.agency != "" && v.agency == info.agency {
+						reason = fmt.Sprintf(reason,"同机构,")
+						if info.agency=="" {
+							reason = fmt.Sprintf(reason,"指定范围,")
+							//指定该范围内数据判重  jsondata
+							if v.agency=="" {
 								continue
 							}
-						}
-
-
-						if n>=3||first_judge==true {
-							if t==2 {//同城
-								if conditionAgainRepeat(v,info) {
-									continue
-								}
+						}else {
+							reason = fmt.Sprintf(reason,"非指定范围,")
+							if quickHeavyMethodTwo(v,info) {
 								b = true
-								id = v.id
+								source = v
+								reasons = reason
 								break L
 							}
 						}
 					}else {
-						n:=0 //三要素
-						m:=0 //二要素
-						x:=0 //四要素
-						if info.buyer != "" &&v.buyer == info.buyer {
-							n++
-							x++
-						}
-						if info.projectname != ""&&v.projectname == info.projectname {
-							n++
-							m++
-							x++
-						}
-						if info.projectcode != ""&&v.projectcode == info.projectcode {
-							n++
-							m++
-							x++
-						}
-						if info.title != ""&&v.title == info.title {
-							x++
-						}
-
-						t:= judgeCityType(v.area,info.area,v.city,info.city)
-						c_1 :=conditionTitle(v.title,info.title) //标题满足
-						c_2 :=conditionNum(v.projectcode,info.projectcode) //编号满足
-						c_3 :=conditionTitleBuyer(v.title,info.title,v.buyer,info.buyer) //标题+采购单位
-						c_4 :=conditionCodeTime(v.bidopentime,info.bidopentime,v.projectcode,info.projectcode) //编号+开标时间
-
-
-						//同站点url判重
-						if info.site != "" && v.site == info.site {
-							if v.href==info.href&&info.href!="" {
-								b = true
-								id = v.id
-								log.Println("href满足过滤:",info.id)
-								break L
-							}else {
+						reason = fmt.Sprintf(reason,"非同机构,")
+						if info.agency=="" {
+							reason = fmt.Sprintf(reason,"指定范围,")
+							//指定该范围内数据判重  jsondata
+							if v.agency=="" {
 								continue
 							}
-						}
-
-						
-
-						site_b := false
-						if info.site != "" {
-							for _,v := range siteArr {
-								if info.site==v["addr"] {
-									site_b=true
-									break
-								}
-							}
-						}
-						if info.site != "" && v.site == info.site&&site_b {
-							if n>1||c_1||c_2 {
-								if conditionAgainRepeat(v,info) {
-									continue
-								}
-								b = true
-								id = v.id
-								log.Println("站点满足过滤")
-								break L
-							}
 						}else {
-							if info.titleSpecialWord&&info.title!=v.title&&v.title!="" {
-								continue
-							}
-							if v.projectcode != info.projectcode&&len([]rune(info.projectcode)) >=10&&v.projectcode!=""{
-								continue
-							}
-							//先决条件满足三要素,条件4
-							if n==3||c_4{
-								if conditionAgainRepeat(v,info) {
-									continue
-								}
+							reason = fmt.Sprintf(reason,"非指定范围,")
+							if quickHeavyMethodOne(v,info) {
 								b = true
-								id = v.id
+								source = v
+								reasons = reason
 								break L
 							}
-
-							//城市判断
-							if t==0||t==1 { //最少一个全国
-								if c_1 && (c_2||n>1) {
-
-									if conditionAgainRepeat(v,info) {
-										continue
-									}
-									b = true
-									id = v.id
-									break L
-								}
-								if c_2&&x>2{
-
-									if conditionAgainRepeat(v,info) {
-										continue
-									}
-									b = true
-									id = v.id
-									break L
-								}
-							}else if t==2 { //	省-市
-								if c_1||c_2||n>1 {
-									//新增二次判断逻辑
-									if conditionAgainRepeat(v,info) {
-										continue
-									}
-									b = true
-									id = v.id
-									break L
-								}
-							}else if t==3 {//	!省 !市
-								if (c_1&&n>1)||(c_2&&x>2){
-
-									if conditionAgainRepeat(v,info) {
-										continue
-									}
-									b = true
-									id = v.id
-									break L
-								}
-
-							}else if t==4 {//	省 !市
-								if m>1||(c_1&&m>0)||(c_2&&x>1)||(c_3&&n>1){
-
-									if conditionAgainRepeat(v,info) {
-										continue
-									}
-									b = true
-									id = v.id
-									break L
-								}
-							}else  {
-
-							}
 						}
 					}
+
 				}
 			}
 		}
@@ -389,102 +245,392 @@ L:
 }
 
 
-//判断是否同城等情况
-func judgeCityType(v string, info string,v_c string,info_c string) (t int) {
-
-	t=0
-	if  (v=="全国"||v=="")&&(info=="全国"||info=="") {//均为全国
-		t=0
-	}else if  v!="全国"&&info!="全国"&&v!=""&&info!=""&&
-		v_c!="全国"&&info_c!="全国"&&v_c!=""&&info_c!=""{//均非全国
-		if v==info &&v_c==info_c { //同省同城
-			t=2
-		}else if v!=info&&v_c!=info_c{//非同省非同城
-			t=3
-		}else {//同省非同城
-			t=4
+func (d *datamap) replaceSourceData(replaceData *Info , replaceId string) {
+	ct, _ := strconv.ParseInt(replaceId[:8], 16, 64)
+	dkey := qutil.FormatDateByInt64(&ct, qutil.Date_yyyyMMdd)
+	k := fmt.Sprintf("%s_%s_%s", dkey, replaceData.subtype, replaceData.area)
+	data := d.data[k]
+	if data == nil {
+		data = []*Info{replaceData}
+		d.data[k] = data
+		if !d.keys[dkey] {
+			d.keys[dkey] = true
+			d.update(ct)
+		}
+	} else {
+		//遍历替换
+		for k,v:=range data{
+			if v.id==replaceId{
+				data[k] = replaceData
+				break
+			}
 		}
-	}else {//有且一个全国 ，包含多种情况，
-		t=1
+
+
+
+		d.data[k] = data
 	}
-	return t
+
 }
 
-//条件一 标题
-func conditionTitle(t1 string, t2 string) bool {
 
-	if len([]rune(t1))>10 && len([]rune(t2))>10&&
-		(strings.Contains(t1, t2)||strings.Contains(t2, t1)) {
-		return true
+//判重方法1
+func quickHeavyMethodOne(v *Info ,info *Info) bool {
+
+	if info.subtype=="招标"||info.subtype=="邀标"||info.subtype=="询价"||
+		info.subtype=="竞谈"||info.subtype=="单一"||info.subtype=="竞价"||
+		info.subtype=="变更"||info.subtype=="其他" {
+		//招标结果
+		if tenderRepeat_A(v,info) {
+			if tenderRepeat_C(v,info) {
+				return false
+			}else {
+				reason = fmt.Sprintf(reason,"---招标类")
+				return true
+			}
+		}else {
+			return false
+		}
+
+	}else if info.subtype=="中标"||info.subtype=="成交"||info.subtype=="废标"||info.subtype=="流标"{
+		//中标结果
+		if winningRepeat_A(v,info) {
+			if winningRepeat_C(v,info) {
+				return false
+			}else {
+				reason = fmt.Sprintf(reason,"---中标类")
+				return true
+			}
+		}else {
+			return false
+		}
+
+	}else if info.subtype=="合同"||info.subtype=="验收"||info.subtype=="违规"{
+		//合同
+		if contractRepeat_A(v,info) {
+			if contractRepeat_C(v,info) {
+				return false
+			}else {
+				reason = fmt.Sprintf(reason,"---合同类")
+				return true
+			}
+		}else {
+			return false
+		}
+	}else {
+
 	}
+
 	return false
 }
-//条件二 项目编号
-func conditionNum(c1 string ,c2 string) bool {
+//判重方法2
+func quickHeavyMethodTwo(v *Info ,info *Info) bool {
 
-	if c1 == c2&&len([]rune(c1)) >=10 {
-		return true
+	//相同
+	if v.agency==info.agency &&v.agency!=""&&info.agency!="" {
+
+		if info.subtype=="招标"||info.subtype=="邀标"||info.subtype=="询价"||
+			info.subtype=="竞谈"||info.subtype=="单一"||info.subtype=="竞价"||
+			info.subtype=="变更"||info.subtype=="其他" {
+			//招标结果
+			if tenderRepeat_B(v,info) {
+				if tenderRepeat_C(v,info) { //有不同
+					return false
+				}else {
+					reason = fmt.Sprintf(reason,"---招标类")
+					return true
+				}
+			}else{
+				return false
+			}
+
+		}else if info.subtype=="中标"||info.subtype=="成交"||info.subtype=="废标"||info.subtype=="流标"{
+			//中标结果
+			if winningRepeat_B(v,info) {
+				if winningRepeat_C(v,info) { //有不同
+					return false
+				}else {
+					reason = fmt.Sprintf(reason,"---中标类")
+					return true
+				}
+			}else{
+				return false
+			}
+
+		}else if info.subtype=="合同"||info.subtype=="验收"||info.subtype=="违规"{
+			//合同
+			if contractRepeat_B(v,info) {
+				if contractRepeat_C(v,info) { //有不同
+					return false
+				}else {
+					reason = fmt.Sprintf(reason,"---合同类")
+					return true
+				}
+			}else {
+				return false
+			}
+		}else {
+
+		}
+	}
+
+	//不同
+	if v.agency!=info.agency &&v.agency!=""&&info.agency!="" {
+		return false
+	}
+	//机构最少一个为空
+	if v.agency==""||info.agency=="" {
+		if quickHeavyMethodOne(v,info) {
+			reason = fmt.Sprintf(reason,"---机构最少一个空")
+			return true
+		}else {
+			return false
+		}
 	}
 	return false
 }
-//条件三 采购单位+标题
-func conditionTitleBuyer(t1 string ,t2 string,b1 string,b2 string) bool {
 
-	if t1==t2&&b1==b2 {
+
+
+//招标_A
+func tenderRepeat_A(v *Info ,info *Info) bool {
+
+	var ss string
+	p1,p2,p3,p4,p9,p10,p11 := false,false,false,false,false,false,false
+	if v.projectname!=""&&v.projectname==info.projectname {
+		ss = fmt.Sprintf(ss,"p1(名称)-")
+		p1 = true
+	}
+	if v.buyer!=""&&v.buyer==info.buyer {
+		ss = fmt.Sprintf(ss,"p2(单位)-")
+		p2 = true
+	}
+	if v.projectcode!=""&&v.projectcode==info.projectcode {
+		ss = fmt.Sprintf(ss,"p3(编号)-")
+		p3 = true
+	}
+	if v.budget!=0&&v.budget==info.budget {
+		ss = fmt.Sprintf(ss,"p4(预算)-")
+		p4 = true
+	}
+	if v.bidopentime!=0&&v.bidopentime==info.bidopentime {
+		ss = fmt.Sprintf(ss,"p9(开标时间)-")
+		p9 = true
+	}
+	if v.agencyaddr!=""&&v.agencyaddr==info.agencyaddr {
+		ss = fmt.Sprintf(ss,"p10(开标地点)-")
+		p10 = true
+	}
+	if len([]rune(v.title))>10 && len([]rune(info.title))>10&&
+		(strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title)) {
+		ss = fmt.Sprintf(ss,"p11(标题)-")
+		p11 = true
+	}
+
+	if (p1&&p2&&p3)||(p1&&p2&&p4)||(p1&&p2&&p9)||
+		(p1&&p2&&p10)||(p1&&p3&&p9)||(p1&&p3&&p10)||
+		(p1&&p4&&p9)||(p1&&p4&&p10)||(p2&&p3&&p4)||
+		(p2&&p3&&p9)||(p2&&p3&&p10)||(p2&&p3&&p11)||
+		(p2&&p4&&p9)||(p2&&p4&&p10)||(p2&&p4&&p11)||
+		(p3&&p4&&p9)||(p3&&p4&&p10)||(p3&&p4&&p11)||
+		(p4&&p9&&p10)||(p4&&p9&&p11)||(p9&&p10&&p11) {
+		reason = fmt.Sprintf(reason,"满足招标A，3要素组合-",ss,",")
 		return true
 	}
 	return false
 }
-//条件四 编号+开标时间
-func conditionCodeTime(t1 int64 ,t2 int64,c1 string,c2 string) bool {
 
-	if c1 != ""&&c1==c2&&t1 != 0&&t1==t2&&len([]rune(c1)) >=10 {
-		return true
+//招标_B
+func tenderRepeat_B(v *Info ,info *Info) bool {
+
+	m,n :=0,0
+	if v.projectname!=""&&v.projectname==info.projectname {
+		m++
+		n++
+	}
+	if v.buyer!=""&&v.buyer==info.buyer {
+		m++
+	}
+	if v.projectcode!=""&&v.projectcode==info.projectcode {
+		m++
+	}
+	if v.budget!=0&&v.budget==info.budget {
+		m++
+	}
+	if v.bidopentime!=0&&v.bidopentime==info.bidopentime {
+		m++
+	}
+	if v.agencyaddr!=""&&v.agencyaddr==info.agencyaddr {
+		m++
+	}
+	if len([]rune(v.title))>10 && len([]rune(info.title))>10&&
+		(strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title)) {
+		m++
+		n++
+	}
+	if m>=2 {
+		if n==2 &&m==2 {
+			return false
+		}else {
+			reason = fmt.Sprintf(reason,"满足招标B，七选二,")
+			return true
+		}
 	}
 	return false
 }
+//招标_C
+func tenderRepeat_C(v *Info ,info *Info) bool {
+
+	if v.budget!=0&&info.budget!=0&&v.budget!=info.budget {
+
+		return true
+	}
+	//原始地址...
 
-func conditionAgainRepeat(v *Info ,info *Info) bool {
-	//同省情况下
-	//if v.area==info.area&&v.area!="全国"&&v.area!=""&&v.city==info.city {
-	//
-	//}
+	if v.bidopentime!=0&&info.bidopentime!=0&&v.bidopentime!=info.bidopentime {
+		return true
+	}
+	if v.agencyaddr!=""&&info.agencyaddr!=""&&v.agencyaddr!=info.agencyaddr {
+		return true
+	}
+	return false
+}
 
-	// 编号过短+金额  不等时，暂存
+//中标_A
+func winningRepeat_A(v *Info ,info *Info) bool {
 
+	var ss string
+	p1,p2,p3,p5,p6,p11 := false,false,false,false,false,false
+	if v.projectname!=""&&v.projectname==info.projectname {
+		ss = fmt.Sprintf(ss,"p1(标题)-")
+		p1 = true
+	}
+	if v.buyer!=""&&v.buyer==info.buyer {
+		ss = fmt.Sprintf(ss,"p2(单位)-")
+		p2 = true
+	}
+	if v.projectcode!=""&&v.projectcode==info.projectcode {
+		ss = fmt.Sprintf(ss,"p3(编号)-")
+		p3 = true
+	}
+	if v.bidamount!=0&&v.bidamount==info.bidamount {
+		ss = fmt.Sprintf(ss,"p5(中标金)-")
+		p5 = true
+	}
+	if v.winner!=""&&v.winner==info.winner {
+		ss = fmt.Sprintf(ss,"p6(中标人)-")
+		p6 = true
+	}
+	if len([]rune(v.title))>10 && len([]rune(info.title))>10&&
+		(strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title)) {
+		ss = fmt.Sprintf(ss,"p11(标题)-")
+		p11 = true
+	}
 
+	if (p1&&p2&&p3)||(p1&&p2&&p5)||(p1&&p2&&p6)||
+		(p1&&p3&&p5)||(p1&&p3&&p6)||(p1&&p5&&p6)||
+		(p2&&p3&&p5)||(p2&&p3&&p6)||(p2&&p3&&p11)||
+		(p2&&p5&&p6)||(p2&&p5&&p11)||(p2&&p6&&p11)||
+		(p3&&p5&&p6)||(p3&&p5&&p11)||(p3&&p6&&p11)||
+		(p5&&p6&&p11){
+		reason = fmt.Sprintf(reason,"满足中标A，3要素组合-",ss,",")
+		return true
+	}
 
+	return false
+}
+//中标_B
+func winningRepeat_B(v *Info ,info *Info) bool {
 
-	//相同采购单位下
-	if info.buyer != "" &&v.buyer == info.buyer {
-		//满足标题
-		if  len([]rune(v.title)) >=10&&len([]rune(info.title)) >=10&&v.title!=info.title&&(info.specialWord||v.specialWord){
+	m,n :=0,0
+	if v.projectname!=""&&v.projectname==info.projectname {
+		m++
+		n++
+	}
+	if v.buyer!=""&&v.buyer==info.buyer {
+		m++
+	}
+	if v.projectcode!=""&&v.projectcode==info.projectcode {
+		m++
+	}
+	if v.bidamount!=0&&v.bidamount==info.bidamount {
+		m++
+	}
+	if v.winner!=""&&v.winner==info.winner {
+		m++
+	}
+	if len([]rune(v.title))>10 && len([]rune(info.title))>10&&
+		(strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title)) {
+		m++
+		n++
+	}
+	if m>=2 {
+		if n==2 &&m==2 {
+			return false
+		}else {
+			reason = fmt.Sprintf(reason,"满足中标B，六选二,")
 			return true
 		}
+	}
+	return false
+}
 
-		if info.subtype=="招标"||info.subtype=="邀标"||info.subtype=="询价"||
-			info.subtype=="竞谈"||info.subtype=="单一"||info.subtype=="竞价"||
-			info.subtype=="其他"||info.subtype=="变更" {
-			//预算金额满足条件
-			if v.budget!=info.budget&&v.budget!=0&&info.budget!=0 {
-				return true
-			}
-		}else if info.subtype=="中标"||info.subtype=="成交"||info.subtype=="废标"||
-			info.subtype=="流标"||info.subtype=="合同"||info.subtype=="验收"||
-			info.subtype=="违规"{
-			//中标金额单位满足条件
-			if (v.bidamount!=info.bidamount&&v.bidamount!=0&&info.bidamount!=0)||
-				(v.winner!=info.winner&&v.winner!=""&&info.winner!=""){
-				return true
-			}
-		}else {
+//中标_C
+func winningRepeat_C(v *Info ,info *Info) bool {
 
-		}
+	if v.bidamount!=0&&info.bidamount!=0&&v.bidamount!=info.bidamount {
+		return true
+	}
+	if v.winner!=""&&info.winner!=""&&v.winner!=info.winner {
+		return true
+	}
+	//原始地址...
+
+
+	return false
+}
+
+//合同_A
+func contractRepeat_A(v *Info ,info *Info) bool {
+
+	if tenderRepeat_A(v,info) {
+		return true
+	}
+	if winningRepeat_A(v,info) {
+		return true
+	}
+	return false
+}
+
+//合同_B
+func contractRepeat_B(v *Info ,info *Info) bool {
+
+	if tenderRepeat_B(v,info) {
+		return true
+	}
+	if winningRepeat_B(v,info) {
+		return true
 	}
+	return false
+}
+
+//合同_C
+func contractRepeat_C(v *Info ,info *Info) bool {
 
+	if tenderRepeat_C(v,info) {
+		return true
+	}
+	if winningRepeat_C(v,info) {
+		return true
+	}
 	return false
 }
 
+
+
+
+
+
 func (d *datamap) update(t int64) {
 	//每天0点清除历史数据
 	d.keymap = d.GetLatelyFiveDay(t)

udpfilterdup/src/main.go

@@ -6,7 +6,6 @@ package main
 
 import (
 	"encoding/json"
-	"flag"
 	"fmt"
 	"gopkg.in/mgo.v2/bson"
 	"log"
@@ -33,8 +32,9 @@ var (
 	nextNode     []map[string]interface{} //下节点数组
 	dupdays      = 5                      //初始化判重范围
 	DM           *datamap                 //判重数据
-	lastid       = ""
+	lastid       = "5da3f2c5a5cb26b9b79847fe"
 	//5da3f2c5a5cb26b9b79847fc
+
 	//正则筛选相关
 	FilterRegTitle = regexp.MustCompile("^_$")
 	FilterRegTitle_1 = regexp.MustCompile("^_$")
@@ -46,8 +46,8 @@ var (
 )
 
 func init() {
-	flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
-	flag.Parse()
+	//flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
+	//flag.Parse()
 	//172.17.145.163:27080
 	util.ReadConfig(&Sysconfig)
 	nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
@@ -66,12 +66,14 @@ func init() {
 
 
 	//测试临时注释
-	//dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
-	////加载数据
-	//DM = NewDatamap(dupdays, lastid)
-	//FilterRegTitle = regexp.MustCompile(util.ObjToString(Sysconfig["specialwords"]))
-	//FilterRegTitle_1 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_1"]))
-	//FilterRegTitle_2 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_2"]))
+	dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
+	//加载数据
+	DM = NewDatamap(dupdays, lastid)
+	fmt.Println(DM.keys)
+	fmt.Println(DM.data)
+	FilterRegTitle = regexp.MustCompile(util.ObjToString(Sysconfig["specialwords"]))
+	FilterRegTitle_1 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_1"]))
+	FilterRegTitle_2 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_2"]))
 
 
 
@@ -90,7 +92,7 @@ func init() {
 }
 
 //新增一个方法 判断
-func main()  {
+func mainTest()  {
 
 	//log.Println("1")
 	//代码copy数据
@@ -221,7 +223,7 @@ func main()  {
 
 
 
-func mainTest() {
+func main() {
 	go checkMapJob()
 
 	updport := Sysconfig["udpport"].(string)
@@ -243,6 +245,14 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 		if err != nil {
 			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
 		} else if mapInfo != nil {
+
+			//更新流程
+
+
+
+
+
+			//判重流程
 			go task(data, mapInfo)
 			key, _ := mapInfo["key"].(string)
 			if key == "" {
@@ -284,11 +294,6 @@ func task(data []byte, mapInfo map[string]interface{}) {
 
 	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
 
-
-		if util.ObjToString(tmp["subtype"]) == "变更" {
-			//continue
-		}
-
 		if n%10000 == 0 {
 			log.Println("current:", n, tmp["_id"],"repeateN:",repeateN)
 		}
@@ -322,28 +327,128 @@ func task(data []byte, mapInfo map[string]interface{}) {
 				}
 				mapLock.Unlock()
 			}else  {
-				b, id := DM.check(info)
+				//判重原因 reason
+				// tmp["_id"] 对比id   id原始id
+				b, source,reason := DM.check(info)
 				if b { //有重复,生成更新语句，更新抽取和更新招标
-
 					repeateN++
 					mapLock.Lock()
-					updateExtract = append(updateExtract, []map[string]interface{}{
-						map[string]interface{}{
-							"_id": tmp["_id"],
-						},
-						map[string]interface{}{
-							"$set": map[string]interface{}{
-								"repeat":   1,
-								"repeatid": id,
+
+					var mergeArr []int64 	//更改合并数组记录
+					var newData *Info		//更换新的数据池数据
+
+					var id_map  = map[string]interface{}{}
+					//合并操作--评功权重打分-合并完替换原始数据池
+					basic_bool := basicDataScore(source,info)
+					if basic_bool {
+						//已原始数据为标准-对比数据打判重标签
+						newData,mergeArr= mergeDataFields(source,info)
+						DM.replaceSourceData(newData,source.id) //替换
+						id_map["_id"]= util.StringTOBsonId(source.id)
+
+						//对比的数据打判重标签
+						updateExtract = append(updateExtract, []map[string]interface{}{
+							map[string]interface{}{
+								"_id": tmp["_id"],
+							},
+							map[string]interface{}{
+								"$set": map[string]interface{}{
+									"repeat":   1,
+									"repeatid": source.id,
+								},
+							},
+						})
+
+
+
+					}else {
+						//已对比数据为标准 ，数据池的数据打判重标签
+						newData,mergeArr= mergeDataFields(info,source)
+						DM.replaceSourceData(newData,source.id)//替换
+						id_map["_id"]= util.StringTOBsonId(info.id)
+
+						//数据池的数据打判重标签
+						updateExtract = append(updateExtract, []map[string]interface{}{
+							map[string]interface{}{
+								"_id": util.StringTOBsonId(source.id),
 							},
+							map[string]interface{}{
+								"$set": map[string]interface{}{
+									"repeat":   1,
+									"repeatid": info.id,
+								},
+							},
+						})
+
+					}
+
+
+					//
+					var update_map  = map[string]interface{}{
+						"$set": map[string]interface{}{
+							"reason":reason,
+							"merge":newData.mergemap,
 						},
-					})
+					}
+					//更新合并后的数据
+					for _,value :=range mergeArr {
+						if value==1 {
+							update_map["$set"].(map[string]interface{})["area"] = newData.area
+							update_map["$set"].(map[string]interface{})["city"] = newData.city
+						}else if value==2 {
+							update_map["$set"].(map[string]interface{})["projectname"] = newData.projectname
+						}else if value==3 {
+							update_map["$set"].(map[string]interface{})["projectcode"] = newData.projectcode
+						}else if value==4 {
+							update_map["$set"].(map[string]interface{})["buyer"] = newData.buyer
+						}else if value==5 {
+							update_map["$set"].(map[string]interface{})["budget"] = newData.budget
+						}else if value==6 {
+							update_map["$set"].(map[string]interface{})["winner"] = newData.winner
+						}else if value==7 {
+							update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
+						}else if value==8 {
+							update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
+						}else {
+
+						}
+					}
 
+					//构建数据库更新用到的
+					updateExtract = append(updateExtract, []map[string]interface{}{
+						id_map,
+						update_map,
+					})
 					if len(updateExtract) > 500 {
 						mgo.UpdateBulk(extract, updateExtract...)
 						updateExtract = [][]map[string]interface{}{}
 					}
 					mapLock.Unlock()
+
+
+
+					//log.Println("判重具体原因：",reason)
+					//repeateN++
+					//mapLock.Lock()
+					//updateExtract = append(updateExtract, []map[string]interface{}{
+					//	map[string]interface{}{
+					//		"_id": tmp["_id"],
+					//	},
+					//	map[string]interface{}{
+					//		"$set": map[string]interface{}{
+					//			"repeat":   1,
+					//			"repeatid": id,
+					//		},
+					//	},
+					//})
+					//
+					//if len(updateExtract) > 500 {
+					//	mgo.UpdateBulk(extract, updateExtract...)
+					//	updateExtract = [][]map[string]interface{}{}
+					//}
+					//mapLock.Unlock()
+
+
 				} else {
 					//IS.Add("new")
 				}
@@ -381,6 +486,185 @@ func task(data []byte, mapInfo map[string]interface{}) {
 	}
 }
 
+
+//合并字段
+func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
+
+	var mergeArr []int64
+	mergeArr = make([]int64,0)
+	//1、城市
+	if (source.area==""||source.area=="全国")&&info.area!="全国"&&info.area!=""{
+		var arrA []string
+		if source.mergemap["area"]==nil {
+			arrA = make([]string, 0)
+		}else {
+			arrA = source.mergemap["area"].([]string)
+		}
+		arrA = append(arrA,source.area)
+		source.mergemap["area"] = arrA
+
+
+		var arrC []string
+		if source.mergemap["city"]==nil {
+			arrC = make([]string, 0)
+		}else {
+			arrC = source.mergemap["city"].([]string)
+		}
+		arrC = append(arrC,source.city)
+		source.mergemap["city"] = arrC
+
+		source.area = info.area
+		source.city = info.city
+		mergeArr = append(mergeArr,1)
+	}
+	//2、项目名称
+	if source.projectname==""&&info.projectname!=""{
+		var arr []string
+		if source.mergemap["projectname"]==nil {
+			arr = make([]string, 0)
+		}else {
+			arr = source.mergemap["projectname"].([]string)
+		}
+		arr = append(arr,source.projectname)
+		source.mergemap["projectname"] = arr
+
+		source.projectname = info.projectname
+		mergeArr = append(mergeArr,2)
+	}
+	//3、项目编号
+	if source.projectcode==""&&info.projectcode!=""{
+		var arr []string
+		if source.mergemap["projectcode"]==nil {
+			arr = make([]string, 0)
+		}else {
+			arr = source.mergemap["projectcode"].([]string)
+		}
+		arr = append(arr,source.projectcode)
+		source.mergemap["projectcode"] = arr
+
+		source.projectcode = info.projectcode
+		mergeArr = append(mergeArr,3)
+	}
+	//4、采购单位
+	if source.buyer==""&&info.buyer!=""{
+		var arr []string
+		if source.mergemap["buyer"]==nil {
+			arr = make([]string, 0)
+		}else {
+			arr = source.mergemap["buyer"].([]string)
+		}
+		arr = append(arr,source.buyer)
+		source.mergemap["buyer"] = arr
+
+		source.buyer = info.buyer
+		mergeArr = append(mergeArr,4)
+	}
+	//5、预算
+	if source.budget==0&&info.budget!=0{
+		var arr []float64
+		if source.mergemap["budget"]==nil {
+			arr = make([]float64, 0)
+		}else {
+			arr = source.mergemap["budget"].([]float64)
+		}
+		arr = append(arr,source.budget)
+		source.mergemap["budget"] = arr
+
+		source.budget = info.budget
+		mergeArr = append(mergeArr,5)
+	}
+	//6、中标单位
+	if source.winner==""&&info.winner!=""{
+		var arr []string
+		if source.mergemap["winner"]==nil {
+			arr = make([]string, 0)
+		}else {
+			arr = source.mergemap["winner"].([]string)
+		}
+		arr = append(arr,source.winner)
+		source.mergemap["winner"] = arr
+
+		source.winner = info.winner
+		mergeArr = append(mergeArr,6)
+	}
+	//7、中标金额
+	if source.bidamount==0&&info.bidamount!=0{
+		var arr []float64
+		if source.mergemap["bidamount"]==nil {
+			arr = make([]float64, 0)
+		}else {
+			arr = source.mergemap["bidamount"].([]float64)
+		}
+		arr = append(arr,source.bidamount)
+		source.mergemap["bidamount"] = arr
+
+		source.bidamount = info.bidamount
+		mergeArr = append(mergeArr,7)
+	}
+	//8、开天时间-地点
+	if source.bidopentime==0&&info.bidopentime!=0{
+		var arr []int64
+		if source.mergemap["bidopentime"]==nil {
+			arr = make([]int64, 0)
+		}else {
+			arr = source.mergemap["bidopentime"].([]int64)
+		}
+		arr = append(arr,source.bidopentime)
+		source.mergemap["bidopentime"] = arr
+
+		source.bidopentime = info.bidopentime
+		mergeArr = append(mergeArr,8)
+	}
+
+
+
+
+
+	//以上合并过于简单，待进一步优化
+
+	return source,mergeArr
+}
+
+
+//权重评估
+func basicDataScore(v *Info, info *Info) bool  {
+	m,n:=0,0
+	if v.projectname!="" {m++}
+	if v.buyer!="" {m++}
+	if v.projectcode!="" {m++}
+	if v.budget!=0 {m++}
+	if v.bidamount!=0 {m++}
+	if v.winner!="" {m++}
+	if v.bidopentime!=0 {m++}
+	if v.agencyaddr!="" {m++}
+	if v.agency!="" {m=m+2}
+	if v.city!="" {m=m+2}
+
+	if info.projectname!="" {n++}
+	if info.buyer!="" {n++}
+	if info.projectcode!="" {n++}
+	if info.budget!=0 {n++}
+	if info.bidamount!=0 {n++}
+	if info.winner!="" {n++}
+	if info.bidopentime!=0 {n++}
+	if info.agencyaddr!="" {n++}
+	if info.agency!="" {n=m+2}
+	if info.city!="" {n=m+2}
+
+	if m>n {
+		return true
+	}else if m==n {
+		if v.comeintime>=info.comeintime {
+			return true
+		}else {
+			return false
+		}
+	}else {
+		return false
+	}
+}
+
+
 //无效数据
 func invalidData(d1 string,d2 string,d3 string)  bool{
 	var n int