Merge branch 'dev3.2' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.2

src/jy/admin/clearversion.go

@@ -63,7 +63,7 @@ func init() {
 		_id, _ := c.GetPostForm("_id")
 		isuse, _ := c.GetPostForm("isuse")
 		if isuse == "true" { //启用确保只有一个版本
-			l, _ := Mgo.Find("clearversion", `{"isuse":true}`, nil, nil, false, -1, -1)
+			l, _ := Mgo.Find("clearversion", `{"delete":false,"isuse":true}`, nil, nil, false, -1, -1)
 			if len(*l) > 0 {
 				c.JSON(200, gin.H{"rep": false})
 				return

src/jy/admin/user.go

@@ -10,10 +10,11 @@ import (
 	qu "qfw/util"
 	"time"
 
+	"fmt"
+
 	"github.com/gin-contrib/sessions"
 	"github.com/gin-gonic/gin"
 	"gopkg.in/mgo.v2/bson"
-	"fmt"
 )
 
 func init() {
@@ -45,28 +46,34 @@ func init() {
 		_id := c.Query("id")
 		c.HTML(http.StatusOK, "secondmenu.html", gin.H{"_id": _id})
 	})
-	Admin.POST("/menu", Menu)
-	Admin.POST("/menu/save", MenuSave)
-	Admin.POST("/menu/data", MenuData)
-	Admin.POST("/menu/searchbyid", MenuSearchById)
-	Admin.POST("/menu/del", MenuDel)
-	Admin.POST("/infotype/data", InfotypeData)
-	Admin.POST("/fields/data", FieldsData)
-	Admin.POST("/topclass/data", TopclassData)
-	Admin.POST("/subclass/data", SubclassData)
-	Admin.POST("/infotype/save", InfotypeSave)
-	Admin.POST("/infotype/select", InfotypeSelect)
-	Admin.POST("/infotype/del", InfotypeDel)
-	Admin.POST("/role/menu/data", RoleMenuData)
-	Admin.POST("/role/menu/save", RoleMenuSave)
-	Admin.POST("/role/select", RoleSelect)
-	Admin.POST("/role/secondmenu/data", RoleSecondMenuData)
-	Admin.POST("/role/menu/del", RoleMenuDel)
-	Admin.POST("/role/secondmenu/del", RoleSecondMenuDel)
-	Admin.POST("/secondmenu/data", SecondMenuData)
-	Admin.POST("/secondmenu/save", SecondMenuSave)
-	Admin.POST("/secondmenu/searchbyid", SecondMenuSearchById)
-	Admin.POST("/secondmenu/del", SecondMenuDel)
+
+	Admin.POST("/menu", Menu) //左侧菜单栏
+	//一级菜单
+	Admin.POST("/menu/save", MenuSave)             //保存一级菜单
+	Admin.POST("/menu/data", MenuData)             //查询一级菜单
+	Admin.POST("/menu/searchbyid", MenuSearchById) //编辑一级菜单
+	Admin.POST("/menu/del", MenuDel)               //删除一级菜单
+	//二级菜单
+	Admin.POST("/secondmenu/data", SecondMenuData)             //查询二级菜单
+	Admin.POST("/secondmenu/save", SecondMenuSave)             //保存二级菜单
+	Admin.POST("/secondmenu/searchbyid", SecondMenuSearchById) //编辑二级菜单
+	Admin.POST("/secondmenu/del", SecondMenuDel)               //删除二级菜单
+	//分类抽取
+	Admin.POST("/infotype/data", InfotypeData)     //查询分类信息
+	Admin.POST("/fields/data", FieldsData)         //查询所有属性
+	Admin.POST("/topclass/data", TopclassData)     //查询所有一级分类
+	Admin.POST("/subclass/data", SubclassData)     //查询二级分类
+	Admin.POST("/infotype/save", InfotypeSave)     //保存分类信息
+	Admin.POST("/infotype/select", InfotypeSelect) //编辑分类信息
+	Admin.POST("/infotype/del", InfotypeDel)       //删除分类信息
+	//角色
+	Admin.POST("/role/menu/data", RoleMenuData) //查询角色的所有菜单
+	Admin.POST("/role/menu/save", RoleMenuSave) //保存角色的所有菜单
+	Admin.POST("/role/select", RoleSelect)      //编辑一级菜单下的二级菜单
+	//Admin.POST("/role/secondmenu/data", RoleSecondMenuData) //查询一级菜单下的二级菜单
+	Admin.POST("/role/menu/del", RoleMenuDel) //删除一级菜单
+	//Admin.POST("/role/secondmenu/del", RoleSecondMenuDel)   //
+
 	Admin.POST("/user/data", User)
 	Admin.POST("/user/save", UserSave)
 	Admin.POST("/user/searchbyid", UserSearchById)
@@ -97,51 +104,51 @@ func SecondMenuData(c *gin.Context) {
 }
 func InfotypeData(c *gin.Context) {
 	datas, _ := Mgo.Find("infotype", `{}`, nil, nil, false, -1, -1)
-	list:=[]map[string]interface{}{}
-	for _,data:=range *datas{
-		if data["subclass"]==nil{
-			data["subclass"]=""
-			list=append(list,data)
-		}else{
-			for k,_:=range data["subclass"].(map[string]interface{}){
-				value:=map[string]interface{}{
-					"_id":data["_id"],
-					"topclass":data["topclass"],
-					"subclass":k,
+	list := []map[string]interface{}{}
+	for _, data := range *datas {
+		if data["subclass"] == nil {
+			data["subclass"] = ""
+			list = append(list, data)
+		} else {
+			for k, _ := range data["subclass"].(map[string]interface{}) {
+				value := map[string]interface{}{
+					"_id":      data["_id"],
+					"topclass": data["topclass"],
+					"subclass": k,
 				}
-				list=append(list,value)
+				list = append(list, value)
 			}
 		}
 	}
 	c.JSON(200, gin.H{"data": list})
 }
-func FieldsData(c *gin.Context){
+func FieldsData(c *gin.Context) {
 	datas, _ := Mgo.Find("fields", `{}`, nil, nil, false, -1, -1)
 	c.JSON(200, gin.H{"data": datas})
 }
-func TopclassData(c *gin.Context){
+func TopclassData(c *gin.Context) {
 	datas, _ := Mgo.Find("infoclass", `{}`, nil, nil, false, -1, -1)
 	c.JSON(200, gin.H{"data": datas})
 }
-func SubclassData(c *gin.Context){
+func SubclassData(c *gin.Context) {
 	topclass, _ := c.GetPostForm("top")
-	datas, _ := Mgo.FindOneByField("infoclass",`{"topclass":"`+topclass+`"}`,`{"subclass":"1"}`)
-	data:=*datas
-	if data["subclass"]!=nil{
+	datas, _ := Mgo.FindOneByField("infoclass", `{"topclass":"`+topclass+`"}`, `{"subclass":"1"}`)
+	data := *datas
+	if data["subclass"] != nil {
 		c.JSON(200, gin.H{"data": datas})
 	}
 }
-func InfotypeSave(c *gin.Context){
+func InfotypeSave(c *gin.Context) {
 	topclass, _ := c.GetPostForm("topclass")
 	subclass, _ := c.GetPostForm("subclass")
 	fieldsStr, _ := c.GetPostForm("fields")
 	fieldsStr2, _ := c.GetPostForm("fields2")
-	_id,_:=c.GetPostForm("_id")
+	_id, _ := c.GetPostForm("_id")
 	fields := make([]string, 0)
 	err := json.Unmarshal([]byte(fieldsStr), &fields)
 	fields2 := make([]string, 0)
 	err2 := json.Unmarshal([]byte(fieldsStr2), &fields2)
-	if _id=="" {
+	if _id == "" {
 		b := Mgo.Count("infotype", map[string]interface{}{
 			"topclass": topclass,
 		})
@@ -153,7 +160,7 @@ func InfotypeSave(c *gin.Context){
 				},
 			})
 		}
-		if (b == 0)||(b!=0&&b2==0&&subclass!=""){
+		if (b == 0) || (b != 0 && b2 == 0 && subclass != "") {
 			if err == nil {
 				if subclass == "" {
 					maps := map[string]interface{}{
@@ -165,8 +172,7 @@ func InfotypeSave(c *gin.Context){
 					}
 					Mgo.Save("infotype", maps)
 				} else {
-					maps := map[string]interface{}{
-					}
+					maps := map[string]interface{}{}
 					for _, field := range fields {
 						maps = map[string]interface{}{
 							"subclass." + subclass + "." + field: true,
@@ -183,25 +189,25 @@ func InfotypeSave(c *gin.Context){
 				c.JSON(200, gin.H{"rep": true})
 			}
 		}
-	}else{
-		if err == nil && err2==nil{
-			set:=map[string]interface{}{
-				"_id":bson.ObjectIdHex(_id),
+	} else {
+		if err == nil && err2 == nil {
+			set := map[string]interface{}{
+				"_id": bson.ObjectIdHex(_id),
 			}
 			if subclass == "" {
 				maps := map[string]interface{}{
-					"fields":   map[string]interface{}{},
+					"fields": map[string]interface{}{},
 				}
 				for _, field := range fields {
 					maps["fields"].(map[string]interface{})[field] = true
 				}
-				maps2:=map[string]interface{}{
-					"$set":maps,
+				maps2 := map[string]interface{}{
+					"$set": maps,
 				}
 				Mgo.Update("infotype", set, maps2, false, false)
 			} else {
 				maps := map[string]interface{}{
-					"subclass."+subclass:map[string]interface{}{},
+					"subclass." + subclass: map[string]interface{}{},
 				}
 				for _, field := range fields {
 					maps["subclass."+subclass].(map[string]interface{})[field] = true
@@ -219,75 +225,73 @@ func InfotypeSave(c *gin.Context){
 func InfotypeSelect(c *gin.Context) {
 	_id, _ := c.GetPostForm("_id")
 	subclass, _ := c.GetPostForm("subclass")
-	fmt.Println(_id)
-	fmt.Println(subclass)
-	one1,_:=Mgo.FindById("infotype",_id,`{}`)
-	one:=*one1
+	one1, _ := Mgo.FindById("infotype", _id, `{}`)
+	one := *one1
 	datas, _ := Mgo.Find("fields", `{}`, nil, nil, false, -1, -1)
-	datas2:=*datas
+	datas2 := *datas
 	//list2:=[]map[string]interface{}{}
-	if one["subclass"]==nil{
-		list:=[]interface{}{}
-		for k,v:=range one["fields"].(map[string]interface{}){
-			if v==true{
-				s_name1,_:=Mgo.FindOneByField("fields",map[string]interface{}{"s_field":k},`{}`)
-				s_name:=*s_name1
+	if one["subclass"] == nil {
+		list := []interface{}{}
+		for k, v := range one["fields"].(map[string]interface{}) {
+			if v == true {
+				s_name1, _ := Mgo.FindOneByField("fields", map[string]interface{}{"s_field": k}, `{}`)
+				s_name := *s_name1
 				//已存在属性
-				list=append(list,s_name)
-				for key,value:=range datas2{
-					if value["s_field"]==k{
+				list = append(list, s_name)
+				for key, value := range datas2 {
+					if value["s_field"] == k {
 						datas2 = append(datas2[:key], datas2[key+1:]...)
 					}
 				}
 			}
 		}
-		fmt.Println(one["topclass"].(string),list)
-		c.JSON(200, gin.H{"topclass": one["topclass"].(string), "subclass": "","fields2":datas2, "fields": list})
-	}else{
-		maps:=one["subclass"].(map[string]interface{})[subclass]
-		list:=[]interface{}{}
-		for k,v:=range maps.(map[string]interface{}){
-			if v==true{
-				s_name1,_:=Mgo.FindOneByField("fields",map[string]interface{}{"s_field":k},`{}`)
-				s_name:=*s_name1
-				list=append(list,s_name)
-				for key,value:=range datas2{
-					if value["s_field"]==k{
+		fmt.Println(one["topclass"].(string), list)
+		c.JSON(200, gin.H{"topclass": one["topclass"].(string), "subclass": "", "fields2": datas2, "fields": list})
+	} else {
+		maps := one["subclass"].(map[string]interface{})[subclass]
+		list := []interface{}{}
+		for k, v := range maps.(map[string]interface{}) {
+			if v == true {
+				s_name1, _ := Mgo.FindOneByField("fields", map[string]interface{}{"s_field": k}, `{}`)
+				s_name := *s_name1
+				list = append(list, s_name)
+				for key, value := range datas2 {
+					if value["s_field"] == k {
 						datas2 = append(datas2[:key], datas2[key+1:]...)
 					}
 				}
 			}
 		}
-		fmt.Println(one["topclass"].(string),subclass,list)
-		c.JSON(200, gin.H{"topclass": one["topclass"].(string), "subclass":subclass,"fields2":datas2, "fields": list})
+		fmt.Println(one["topclass"].(string), subclass, list)
+		c.JSON(200, gin.H{"topclass": one["topclass"].(string), "subclass": subclass, "fields2": datas2, "fields": list})
 	}
 
 }
 func InfotypeDel(c *gin.Context) {
 	_id, _ := c.GetPostForm("_id")
 	subclass, _ := c.GetPostForm("subclass")
-	set:=map[string]interface{}{
-		"_id":bson.ObjectIdHex(_id),
+	set := map[string]interface{}{
+		"_id": bson.ObjectIdHex(_id),
 	}
-	if subclass==""{
-		b:=Mgo.Del("infotype",set)
+	if subclass == "" {
+		b := Mgo.Del("infotype", set)
 		c.JSON(200, gin.H{"rep": b})
-	}else{
-		info1,_:=Mgo.FindById("infotype",_id,`{}`)
-		info:=*info1
-		maps:=info["subclass"].(map[string]interface{})
-		delete(maps,subclass)
-		if len(maps)==0{
-			b:=Mgo.Del("infotype",set)
+	} else {
+		info1, _ := Mgo.FindById("infotype", _id, `{}`)
+		info := *info1
+		maps := info["subclass"].(map[string]interface{})
+		delete(maps, subclass)
+		if len(maps) == 0 {
+			b := Mgo.Del("infotype", set)
 			c.JSON(200, gin.H{"rep": b})
-		}else{
-			maps2:=map[string]interface{}{
-				"subclass":maps,
+		} else {
+			maps2 := map[string]interface{}{
+				"subclass": maps,
 			}
-			maps3:=map[string]interface{}{
-				"$set":maps2,
+			maps3 := map[string]interface{}{
+				"$set": maps2,
 			}
-			b:=Mgo.Update("infotype",set,maps3,false,false)
+			b := Mgo.Update("infotype", set, maps3, false, false)
 			c.JSON(200, gin.H{"rep": b})
 		}
 	}
@@ -405,17 +409,18 @@ func RoleSelect(c *gin.Context) {
 	}
 
 }
-func RoleSecondMenuData(c *gin.Context) {
-	role, _ := c.GetPostForm("role")
-	_id, _ := c.GetPostForm("_id")
-	maps := map[string]interface{}{
-		"menuid":       _id,
-		"role." + role: true,
-	}
-	datas, _ := Mgo.Find("menusecond", maps, nil, nil, false, -1, -1)
 
-	c.JSON(200, gin.H{"data": datas})
-}
+//func RoleSecondMenuData(c *gin.Context) {
+//	role, _ := c.GetPostForm("role")
+//	_id, _ := c.GetPostForm("_id")
+//	maps := map[string]interface{}{
+//		"menuid":       _id,
+//		"role." + role: true,
+//	}
+//	datas, _ := Mgo.Find("menusecond", maps, nil, nil, false, -1, -1)
+
+//	c.JSON(200, gin.H{"data": datas})
+//}
 func Menu(c *gin.Context) {
 	session := sessions.Default(c)
 	u := session.Get("user").(*map[string]interface{})
@@ -572,21 +577,22 @@ func RoleMenuDel(c *gin.Context) {
 	}
 	c.JSON(200, gin.H{"rep": b})
 }
-func RoleSecondMenuDel(c *gin.Context) {
-	_id, _ := c.GetPostForm("_id")
-	role, _ := c.GetPostForm("role")
-	maps := map[string]interface{}{
-		"_id": bson.ObjectIdHex(_id),
-	}
-	data := map[string]interface{}{
-		"role." + role: false,
-	}
-	data2 := map[string]interface{}{
-		"$set": data,
-	}
-	b := Mgo.Update("menusecond", maps, data2, true, false)
-	c.JSON(200, gin.H{"rep": b})
-}
+
+//func RoleSecondMenuDel(c *gin.Context) {
+//	_id, _ := c.GetPostForm("_id")
+//	role, _ := c.GetPostForm("role")
+//	maps := map[string]interface{}{
+//		"_id": bson.ObjectIdHex(_id),
+//	}
+//	data := map[string]interface{}{
+//		"role." + role: false,
+//	}
+//	data2 := map[string]interface{}{
+//		"$set": data,
+//	}
+//	b := Mgo.Update("menusecond", maps, data2, true, false)
+//	c.JSON(200, gin.H{"rep": b})
+//}
 func UserUppwd(c *gin.Context) {
 	_id, _ := c.GetPostForm("_id")
 	pwd, _ := c.GetPostForm("pwd")

src/jy/extract/clearesult.go

@@ -45,6 +45,7 @@ func (c *ClearTask) ClearProcess(doc *map[string]interface{}) {
 				}
 			}
 			tmpmaparr[field] = tmparr
+			//	qu.Debug("tmpmaparr--", tmpmaparr)
 			set[field] = (*doc)[field]
 		}
 		tmpmaparr["resultid"] = _id
@@ -97,7 +98,7 @@ func SaveCltLog() {
 	time.AfterFunc(10*time.Second, SaveCltLog)
 }
 
-//批量更新抽取结果的值(todo)
+//批量更新抽取结果的值
 func (c *ClearTask) UpdateResultVal() {
 	defer qu.Catch()
 	c.ClearChannel = make(chan bool, 5)
@@ -108,7 +109,7 @@ func (c *ClearTask) UpdateResultVal() {
 			arr := c.UpdateResult[:500]
 			go func(tmp *[][]map[string]interface{}) {
 				qu.Try(func() {
-					db.Mgo.UpdateBulk(c.ClearTaskInfo.FromColl, *tmp...)
+					c.ClearTaskInfo.FDB.UpdateBulk(c.ClearTaskInfo.FromColl, *tmp...)
 					<-c.ClearChannel
 				}, func(err interface{}) {
 					log.Println(err)
@@ -119,9 +120,9 @@ func (c *ClearTask) UpdateResultVal() {
 		} else {
 			c.ClearChannel <- true
 			arr := c.UpdateResult
-			go func(tmp *[][]map[string]interface{}) {
+			func(tmp *[][]map[string]interface{}) {
 				qu.Try(func() {
-					db.Mgo.UpdateBulk(c.ClearTaskInfo.FromColl, *tmp...)
+					c.ClearTaskInfo.FDB.UpdateBulk(c.ClearTaskInfo.FromColl, *tmp...)
 					<-c.ClearChannel
 				}, func(err interface{}) {
 					log.Println(err)

src/jy/extract/extractInit.go

@@ -84,9 +84,9 @@ type ExtractTask struct {
 	AuditFields   []string                            //需要审核的字段名称
 
 	ProvinceMap       map[string]string
-	CityBrief         map[string]*City     //只加载一次即可
-	ProvinceBrief     map[string]*Province //只加载一次
-	AreaToCity        map[string][]*City   //两个文件共用
+	CityBrief         map[string]*City     //市简称(只加载一次即可)
+	ProvinceBrief     map[string]*Province //省简称(只加载一次)
+	AreaToCity        map[string][]*City   //市，省全称简称(两个文件共用)
 	DistrictCityMap   map[string]*City
 	StreetDistrictMap map[string]*District
 	AreaGet           *ju.DFA //市全称
@@ -730,7 +730,7 @@ func (e *ExtractTask) InitDFA() {
 		pb := v["brief"].(string)
 		p := e.ProvinceBrief[pb]
 		//加载
-		for _, ss := range []string{k, pb} {
+		for _, ss := range []string{k, pb} { //省全称和省简称
 			cs := e.AreaToCity[ss]
 			if cs != nil {
 				cs = append(cs, p.Captial)
@@ -738,7 +738,7 @@ func (e *ExtractTask) InitDFA() {
 				cs = []*City{p.Captial}
 			}
 			e.AreaToCity[ss] = cs
-			e.AreaSimGet.AddWord(ss) //省全称和省简称
+			e.AreaSimGet.AddWord(ss)
 		}
 		city, _ := v["city"].(map[string]interface{})
 		for k1, v1 := range city {

src/jy/extract/extractcity.go

@@ -53,11 +53,11 @@ func (e *ExtractTask) TransmitData(resulttmp map[string]interface{}, id string)
 			fieldval = append(fieldval, fmt.Sprint(val))
 		}
 	}
-	//log.Println("field========", fieldval)
+	//qu.Debug("fieldval========", fieldval)
 	bres, c, p = e.ExtractProvinceCity(province, city, id, fieldval) //抽取省和市
-	//log.Println("b--------", bres, "p---------", p, "c-------------", c)
+	//qu.Debug("b--------", bres, "p---------", p, "c-------------", c)
 	bres, p, c, d = e.ExtractDistrict(fieldval, bres, c, p, id) //抽取区或县
-	//log.Println("bres========", bres, "p===========", p, "c=========", c, "d=============", d)
+	//qu.Debug("bres========", bres, "p===========", p, "c=========", c, "d=============", d)
 	return
 }
 
@@ -65,10 +65,8 @@ func (e *ExtractTask) TransmitData(resulttmp map[string]interface{}, id string)
 func (e *ExtractTask) ExtractDistrict(field []string, bres bool, c, p, id string) (bool, string, string, string) {
 	d := ""
 	for _, str := range field {
-		//log.Println("field===========", str)
 		for pos, GET := range []*ju.DFA{e.AreaDistrict, e.AreaStreet} { //先匹配区或县再匹配街道
 			word := GET.CheckSensitiveWord(str)
-			//log.Println("word================", word)
 			if word != "" {
 				if pos == 0 { //区或县匹配
 					//log.Println("县直接匹配到====", word)
@@ -80,7 +78,7 @@ func (e *ExtractTask) ExtractDistrict(field []string, bres bool, c, p, id string
 						d = word
 						ctmp := city.Brief
 						ptmp := city.P.Brief
-						//log.Println("ctmpptmp================", ptmp, ctmp)
+						//log.Println("ctmpptmp================", ptmp, ctmp, bres)
 						if !bres { //城市省份没有抽到，通过区或县定位市和省
 							c = ctmp
 							p = ptmp
@@ -88,7 +86,6 @@ func (e *ExtractTask) ExtractDistrict(field []string, bres bool, c, p, id string
 						} else { //对比抽到的城市省份是否一致
 							if c != ctmp || p != ptmp {
 								//log.Println("str---", str, "====", word)
-								//log.Println("district: City And Province, Inconsistent Before And After,Id:", id, c, p, ctmp, ptmp, d)
 								c = ctmp
 								p = ptmp
 							}
@@ -111,7 +108,6 @@ func (e *ExtractTask) ExtractDistrict(field []string, bres bool, c, p, id string
 							bres = true
 						} else { //对比抽到的城市省份是否一致
 							if c != ctmp || p != ptmp {
-								//log.Println("street: City And Province, Inconsistent Before And After,Id:", id, c, p, ctmp, ptmp, d)
 								c = ctmp
 								p = ptmp
 							}

src/jy/pretreated/analystep.go

@@ -21,36 +21,27 @@ func AnalyStart(job *util.Job) {
 	//计算表格占比,返回表格数组、占比
 	tabs, ration := ComputeConRatio(con, 1)
 	if len(tabs) > 0 {
-		newcon, newtabs, newration := FindBigText(con, ration, tabs)//查找大文本，5次
-		if newcon != "" && newration == 0 {
+		newcon, newtabs, newration := FindBigText(con, ration, tabs)
+		if newcon != "" {
 			con = newcon
 			tabs = newtabs
 			ration = newration
 		}
 	}
-	blockArrays, _ := DivideBlock(con, 1, job.RuleBlock)//分块
-	if len(blockArrays) > 0 { //有分块
+	blockArrays, _ := DivideBlock(con, 1, job.RuleBlock) //分块
+	if len(blockArrays) > 0 {                            //有分块
 		//从块里面找分包
-		job.BlockPackage = FindPackageFromBlocks(&blockArrays, job.Title)//从块里面找分包
+		job.BlockPackage = FindPackageFromBlocks(&blockArrays, job.Title) //从块里面找分包
 		for _, bl := range blockArrays {
 			if len([]rune(bl.Text)) > 80 {
-				ba1, _ := DivideBlock(bl.Text, 1, job.RuleBlock)
-				if len(ba1) > 0 {
-					t := ""
-					for _, t1 := range ba1 {
-						t += t1.Text
-					}
-					bl.Text = t
-					bl.ColonKV = GetKVAll(t, bl.Title, 1)
-					bl.SpaceKV = SspacekvEntity.Entrance(t, bl.Title)
-				}
+				bl.Block, _ = DivideBlock(bl.Text, 1, job.RuleBlock)
 			}
 			//块中再查找表格(块,处理完把值赋到块)
 			t1, _ := ComputeConRatio(bl.Text, 2)
 			if len(t1) > 0 {
-				job.HasTable = 1 //添加标识:文本中有table
-				tabres := AnalyTableV2(t1, job.Category, bl.Title, bl.Text, 2, job.SourceMid, job.RuleBlock)//解析表格入口 返回：汇总表格对象
-				processTableResult(tabres, bl, job)//分析table解析结果
+				job.HasTable = 1                                                                             //添加标识:文本中有table
+				tabres := AnalyTableV2(t1, job.Category, bl.Title, bl.Text, 2, job.SourceMid, job.RuleBlock) //解析表格入口 返回：汇总表格对象
+				processTableResult(tabres, bl, job)                                                          //分析table解析结果
 				if bl.Title == "" && tabres.BlockTag != "" {
 					bl.Title = tabres.BlockTag
 				}
@@ -87,8 +78,8 @@ func AnalyStart(job *util.Job) {
 		}
 
 		//调用kv解析
-		bl.ColonKV = GetKVAll(newCon, "", 1)
-		bl.SpaceKV = SspacekvEntity.Entrance(newCon, "")
+		bl.ColonKV = GetKVAll(newCon, "", nil, 1)
+		bl.SpaceKV = SspacekvEntity.Entrance(newCon, "", nil)
 		job.Block = append(job.Block, bl)
 	}
 }
@@ -278,7 +269,7 @@ func tableDivideBlock(con string, ration float32, tabs []*goquery.Selection) str
 			content = regEndWrap.ReplaceAllString(content, "")
 			doc, _ := goquery.NewDocumentFromReader(strings.NewReader(con))
 			doc.Find("table").Eq(0).ReplaceWithHtml(content)
-			con, _ = doc.Html()
+			con, _ = doc.Find("body").Html()
 		}
 	}
 	return con
@@ -294,9 +285,11 @@ func FindBigText(con string, r float32, t []*goquery.Selection) (content string,
 		if content != "" {
 			tabs, ration = ComputeConRatio(content, 1)
 			if len(tabs) > 0 {
-				content = tableDivideBlock(content, ration, tabs)
-				if content == "" {
+				con := tableDivideBlock(content, ration, tabs)
+				if con == "" {
 					return
+				} else {
+					content = con
 				}
 			} else {
 				doc, _ := goquery.NewDocumentFromReader(strings.NewReader(con))

src/jy/pretreated/colonkv.go

@@ -63,8 +63,11 @@ func (ce *ColonkvEntity) divisionMoreKV(con string) string {
 }
 
 //获取冒号kv入口
-func (ce *ColonkvEntity) entrance(con, title string, from int) ([]*Kv, map[string]string) {
+func (ce *ColonkvEntity) entrance(con, title string, contactFormat *ContactFormat, from int) ([]*Kv, map[string]string) {
 	kvs := ce.GetKvs(con, title, from)
+	if from == 1 {
+		FormatContactKv(&kvs, title, nil, contactFormat)
+	}
 	kv := map[string]string{}
 	for _, v := range kvs {
 		if strings.TrimSpace(v.Value) == "" {
@@ -238,8 +241,7 @@ func IsContactKvHandle(value string, m map[string]bool) bool {
 
 //kv关于联系人信息的处理
 //采购人>集中采购机构
-/*
-func FormatContactKv(kvs *[]*Kv, title string, buyers []string) {
+func FormatContactKv(kvs *[]*Kv, title string, buyers []string, contactFormat *ContactFormat) {
 	////////////////////////////
 	//处理联系人信息
 	var indexMap map[int]string
@@ -565,7 +567,6 @@ func FormatContactKv(kvs *[]*Kv, title string, buyers []string) {
 	//	}
 	//Debug("totalIndexMap", len(totalIndexMap))
 }
-*/
 func ContactTypeTitleMatch(title string) string {
 	matchType := ""
 	if title != "" && len([]rune(title)) < 15 {
@@ -614,9 +615,9 @@ func HasOrderContactType(text string) []string {
 
 //两种冒号kv结合到一起
 //from 1--全文 2--table td 3--table td解析采购单位联系人 4--分包
-func GetKVAll(content, title string, from int) *JobKv {
+func GetKVAll(content, title string, contactFormat *ContactFormat, from int) *JobKv {
 	content = formatText(content, "kv")
-	m1Kvs, _ := colonkvEntity.entrance(content, title, from)
+	m1Kvs, _ := colonkvEntity.entrance(content, title, contactFormat, from)
 	m1, m1Weight := KvTagsToKV(m1Kvs, title, nil, from)
 	if m1 == nil {
 		m1 = map[string]string{}

src/jy/pretreated/division.go

@@ -48,12 +48,16 @@ var (
 	regDivision        = regexp.MustCompile("[:：]")
 	regSpliteSegment   = regexp.MustCompile("[\r\n]")
 	regFilterNumber    = regexp.MustCompile("^[\\d一二三四五六七八九十]+")
-	regSplit           = regexp.MustCompile("和|以?及|与|、")
+	regSplit           = regexp.MustCompile("或|和|以?及|与|、|或")
 	regStartWrap       = regexp.MustCompile("^[\r\n]")
 	regEndWrap         = regexp.MustCompile("[\r\n]$")
 	regMoreWrap        = regexp.MustCompile("[\r\n]{2,}")
 	replSerial         = regexp.MustCompile("(\r\n|^)([\\d一二三四五六七八九十][、．.:：，])+\\d")
 	moreColonReg       = regexp.MustCompile("[:：]+")
+	regFilter          = regexp.MustCompile("等$")
+	confusion          = map[string]string{
+		"参与": "canyu",
+	}
 	//查找分包之前，先对内容进行预处理
 	/*
 		第一包：采购设备清单
@@ -165,7 +169,6 @@ func DivideBlock(content string, from int, ruleBlock *util.RuleBlock) ([]*util.B
 		}
 		//获取块中除了序号和标题的内容
 		blockText := regTrimSpace.ReplaceAllString(content[end:nextStart], "")
-		var titles = []string{}
 		if title != "" {
 			blockTextTemp := regReplAllSpace.ReplaceAllString(blockText, "")
 			//特殊情况处理
@@ -184,6 +187,7 @@ func DivideBlock(content string, from int, ruleBlock *util.RuleBlock) ([]*util.B
 						十二、开标时间：2017年3月20日9时30分
 					*/
 					blockText = title
+					title = ""
 				}
 			} else if blockTextTemp != "" && regDivision.MatchString(title) {
 				/*
@@ -196,34 +200,16 @@ func DivideBlock(content string, from int, ruleBlock *util.RuleBlock) ([]*util.B
 				divisionIndexs := regDivision.FindStringIndex(title)
 				titleBefore := regReplAllSpace.ReplaceAllString(title[:divisionIndexs[0]], "")
 				titleAfter := regReplAllSpace.ReplaceAllString(title[divisionIndexs[1]:], "")
+				blockText = title + "\n" + blockText
 				if titleAfter != "" {
-					titles = append(titles, titleBefore)
-					//分段 去每一个冒号前面的key
-					segments := regSpliteSegment.Split(blockText, -1)
-					for _, sv := range segments {
-						divisionIndexs = regDivision.FindStringIndex(sv)
-						if len(divisionIndexs) == 0 {
-							continue
-						}
-						titleTemp := regReplAllSpace.ReplaceAllString(sv[:divisionIndexs[0]], "")
-						if titleTemp == "" {
-							continue
-						}
-						titles = append(titles, titleTemp)
-					}
-					blockText = title + "\n" + blockText
 					title = ""
 				} else {
-					blockText = title + "\n" + blockText
 					title = titleBefore
 				}
 			} else {
 				blockText = title + "\n" + blockText
 			}
 		}
-		if len(titles) == 0 {
-			titles = append(titles, title)
-		}
 		//没有内容的块，不打标签，不分段
 		if blockText == "" {
 			continue
@@ -233,29 +219,29 @@ func DivideBlock(content string, from int, ruleBlock *util.RuleBlock) ([]*util.B
 			continue
 		}
 		blockText = hasMergeKV(title, blockText)
-		block := &util.Block{
-			Index: index,     //序号
-			Text:  blockText, //内容
-			Title: title,     //标题
-			Start: start,
-			End:   nextStart,
-		}
 		//
 		titleIsExists := map[string]bool{} //去重
-		for _, tv := range titles {
-			tv = filterTitle(tv)
-			//分割标题 [和及]。。。
-			splitTitles := regSplit.Split(tv, -1)
-			for _, sv := range splitTitles {
-				if sv == "" || titleIsExists[sv] {
-					continue
-				}
-				titleIsExists[sv] = true
-				//标题过短过长不打标签
-				if len([]rune(sv)) >= 2 && len([]rune(sv)) <= 10 {
-					//打标签
-					block.Tags = append(block.Tags, util.GetBlockTags(sv))
-				}
+		title = filterTitle(title)
+		//分割标题 [和及]。。。 参与
+		splitTitles := ProcTitle(title)
+		block := &util.Block{
+			Index:  index,     //序号
+			Text:   blockText, //内容
+			Title:  title,     //标题
+			Titles: splitTitles,
+			Start:  start,
+			End:    nextStart,
+		}
+
+		for _, sv := range splitTitles {
+			if sv == "" || titleIsExists[sv] {
+				continue
+			}
+			titleIsExists[sv] = true
+			//标题过短过长不打标签
+			if len([]rune(sv)) >= 2 && len([]rune(sv)) <= 10 {
+				//打标签
+				block.Tags = append(block.Tags, util.GetBlockTags(sv))
 			}
 		}
 		tagsToBlocks(blocks, block)
@@ -279,18 +265,62 @@ func DivideBlock(content string, from int, ruleBlock *util.RuleBlock) ([]*util.B
 			returnValue = 1
 		}
 	}
-
+	contactFormat := &util.ContactFormat{
+		IndexMap: map[int]string{},
+		MatchMap: map[string]map[string]bool{},
+	}
 	for _, bl := range returnBlocks {
 		//解析kv
 		newText := TextAfterRemoveTable(bl.Text)
-		bl.ColonKV = GetKVAll(newText, bl.Title, from)
-		bl.SpaceKV = SspacekvEntity.Entrance(newText, bl.Title)
+		bl.ColonKV = GetKVAll(newText, bl.Title, contactFormat, from)
+		bl.SpaceKV = SspacekvEntity.Entrance(newText, bl.Title, contactFormat)
 		//正则抽取的时候有时需要匹配换行或者句号，这里在解析完kv之后，在块结尾添加换行和句号
 		bl.Text = appendWarpStop(bl.Text)
 	}
 	return returnBlocks, returnValue
 }
 
+//块标题处理
+func ProcTitle(title string) []string {
+	if title == "" {
+		return []string{}
+	}
+	for k, v := range confusion {
+		title = strings.Replace(title, k, v, -1)
+	}
+	direct := 1
+	prev := ""
+	ara := regSplit.Split(title, -1)
+	for kk, vv := range ara {
+		for kkk, vvv := range confusion {
+			vv = strings.Replace(vv, vvv, kkk, -1)
+		}
+		ara[kk] = vv
+		if len([]rune(vv)) == 2 {
+			if kk == 0 {
+				direct = -1
+			} else {
+				start := ""
+				if len([]rune(prev)) > 3 {
+					start = string([]rune(prev)[:len([]rune(prev))-2])
+				}
+				ara[kk] = start + vv
+			}
+		}
+		if len([]rune(vv)) > 3 {
+			if direct == -1 {
+				end := string([]rune(vv)[len([]rune(vv))-2:])
+				for i := 0; i < kk; i++ {
+					ara[i] = ara[i] + end
+				}
+				break
+			}
+			prev = vv
+		}
+	}
+	return ara
+}
+
 //有合并kv的 例如项目名称及编号
 func hasMergeKV(title, text string) string {
 	title = regDivision.ReplaceAllString(title, "")
@@ -424,6 +454,12 @@ func tagsToBlocks(blocks []*util.Block, block *util.Block) {
 }
 
 func filterTitle(title string) string {
+	if strings.Contains(title, "，") && strings.Contains(title, "。") {
+		return ""
+	}
+	if len([]rune(title)) > 30 {
+		return ""
+	}
 	//清理空格
 	title = regReplAllSpace.ReplaceAllString(title, "")
 	//清理成对出现的符号中的内容
@@ -432,6 +468,7 @@ func filterTitle(title string) string {
 	title = regReplAllSymbol.ReplaceAllString(title, "")
 	//清理序号
 	title = regFilterNumber.ReplaceAllString(title, "")
+	title = regFilter.ReplaceAllString(title, "")
 	return title
 }
 
@@ -449,8 +486,8 @@ func FindPackageFromBlocks(blocks *[]*util.Block, title string) (blockPackage ma
 		//把分包内容摘除掉有问题 有的项目名称中包含二标段
 		if ok && false {
 			v.Text = surplusText
-			v.ColonKV = GetKVAll(surplusText, v.Title, 1)
-			v.SpaceKV = SspacekvEntity.Entrance(surplusText, v.Title)
+			v.ColonKV = GetKVAll(surplusText, v.Title, nil, 1)
+			v.SpaceKV = SspacekvEntity.Entrance(surplusText, v.Title, nil)
 		}
 	}
 	return
@@ -599,9 +636,9 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 				//合并文本
 				(*blockPackage)[index].Text += "\n" + text
 				//合并冒号kv
-				colonJobKv := GetKVAll(strings.TrimLeft(text, headKey), "", 1)
+				colonJobKv := GetKVAll(strings.TrimLeft(text, headKey), "", nil, 1)
 				if headKey != "" {
-					kvAgain := GetKVAll(text, "", 4)
+					kvAgain := GetKVAll(text, "", nil, 4)
 					for kv_k, kv_v := range kvAgain.Kv {
 						if colonJobKv.Kv[kv_k] == "" {
 							colonJobKv.Kv[kv_k] = kv_v
@@ -619,7 +656,7 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 					(*blockPackage)[index].ColonKV.Kv[kv_k] = kv_v
 				}
 				//合并空格kv
-				spaceJobKv := SspacekvEntity.Entrance(text, "")
+				spaceJobKv := SspacekvEntity.Entrance(text, "", nil)
 				for kv_k, kv_v := range spaceJobKv.Kv {
 					if kv_v == "" {
 						continue
@@ -637,9 +674,9 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 					Type:     bv[1],
 					Accuracy: accuracy,
 				}
-				finalKv := GetKVAll(strings.TrimLeft(text, headKey), "", 4)
+				finalKv := GetKVAll(strings.TrimLeft(text, headKey), "", nil, 4)
 				if headKey != "" {
-					kvAgain := GetKVAll(text, "", 4)
+					kvAgain := GetKVAll(text, "", nil, 4)
 					for kv_k, kv_v := range kvAgain.Kv {
 						if finalKv.Kv[kv_k] == "" {
 							finalKv.Kv[kv_k] = kv_v
@@ -648,7 +685,7 @@ func divisionPackageChild(blockPackage *map[string]*util.BlockPackage, content,
 					}
 				}
 				newBpkg.ColonKV = finalKv
-				newBpkg.SpaceKV = SspacekvEntity.Entrance(text, "")
+				newBpkg.SpaceKV = SspacekvEntity.Entrance(text, "", nil)
 				(*blockPackage)[index] = newBpkg
 			}
 		}

src/jy/pretreated/spacekv.go

@@ -16,7 +16,7 @@ var (
 	excludeSpaceKey = regexp.MustCompile("[.、�\\[【{｛〔<《\\]】}｝〕>》]")
 )
 
-func (se *SpacekvEntity) Entrance(text, title string) *util.JobKv {
+func (se *SpacekvEntity) Entrance(text, title string, contactFormat *util.ContactFormat) *util.JobKv {
 	lines := se.getLines(text)
 	kvMaps := []*util.Kv{}
 	for _, line := range lines {
@@ -26,7 +26,7 @@ func (se *SpacekvEntity) Entrance(text, title string) *util.JobKv {
 		}
 		kvMaps = append(kvMaps, kvMap...)
 	}
-	//FormatContactKv(&kvMaps, title, nil, contactFormat)
+	FormatContactKv(&kvMaps, title, nil, contactFormat)
 	kv, tagKv := KvTagsToKV(kvMaps, title, nil, 1)
 	return &util.JobKv{
 		Kvs:   kvMaps,

src/jy/pretreated/tablev2.go

@@ -128,8 +128,8 @@ func NewTD(Goquery *goquery.Selection, tr *TR, table *Table) *TD {
 	text := dwReg.ReplaceAllString(u.TrimLRAll(txt, ""), "$1")
 	td.Val = text //值
 	td.Text = txt //原始串
-	//对td单元格值判断是否是key和根据td内容长度进行分块处理
-	td.tdIsKey(tr, table, bsontable)
+	//对td单元格值判断是否是表头和根据td内容长度进行分块处理
+	td.tdIsHb(tr, table, bsontable)
 	bhead := false
 	if td.TR.RowPos == 0 { //第一行
 		if td.Goquery.Closest("thead").Size() == 1 && !bsontable { //如果是thead确定为k值表头
@@ -228,16 +228,17 @@ func (td *TD) tdHasTable(bsontable *bool, tr *TR, table *Table) {
 	}
 }
 
-//对td单元格值判断是否是key和根据td内容长度进行分块处理
-func (td *TD) tdIsKey(tr *TR, table *Table, bsontable bool) {
+//对td单元格值判断是否是表头和根据td内容长度进行分块处理
+func (td *TD) tdIsHb(tr *TR, table *Table, bsontable bool) {
 	lenval := len([]rune(td.Val)) //经过处理的td内容长度
 	//if lentxt > 9 {
 	//td.KV = GetKVAll(txt, "")
 	ub := []*u.Block{}
-	//经过处理的td内容长度大于50
-	if lenval > 50 {
+	//经过处理的td内容长度大于50，划块，分包
+	if lenval > 50 { //看是否划块
+		//u.Debug(txt)
+		ub, _ = DivideBlock(td.Text, 2, table.TableResult.RuleBlock) //对td的原始值
 		//看是否划块
-		ub, _ = DivideBlock(td.Text, 2, nil) //对td的原始值
 		if len(ub) > 0 {
 			colonKvWeight := map[string]int{}
 			spaceKvWeight := map[string]int{}
@@ -271,15 +272,15 @@ func (td *TD) tdIsKey(tr *TR, table *Table, bsontable bool) {
 		}*/
 		if len(tr.TDs) > 0 {
 			tdleft := tr.TDs[len(tr.TDs)-1]
-			if tdleft.BH && excludeKey.MatchString(tdleft.Text) {
+			if tdleft.BH && excludeKey.MatchString(tdleft.Text) { //(涉及包号|包件号?|项目标号|规格|型号|招标范围|业绩|废标)|(^编号$)|([^包段标]编号)
 				isFindPkg = false
 			}
 		}
 		if isFindPkg {
 			if len(ub) > 0 {
-				blockPackage = FindPackageFromBlocks(&ub, "")
+				blockPackage = FindPackageFromBlocks(&ub, "") //从块里面找分包
 			} else {
-				blockPackage = FindPackageFromText("", td.Val)
+				blockPackage = FindPackageFromText("", td.Val) //从正文里面找分包
 			}
 		}
 		if len(blockPackage) > 0 {
@@ -309,26 +310,26 @@ func (td *TD) tdIsKey(tr *TR, table *Table, bsontable bool) {
 			}
 		}
 	}
-	//经过处理的td内容长度小于等于50
-	if lenval <= 50 {
+	//经过处理的td内容长度小于50，冒号kv，td表头
+	if lenval < 50 {
 		//		td.SortKV = FindKv(text, "")
 		kvTitle := ""
 		if len(td.TR.TDs) > 0 {
 			kvTitle = td.TR.TDs[len(td.TR.TDs)-1].Val
 		}
 		/*
-				预算总价
-		(人民币：元）
-		 */
+					预算总价
+			(人民币：元）
+		*/
 		if td.Text != "" && strings.Contains(td.Text, "预算总价") && (strings.Contains(td.Text, "(") || strings.Contains(td.Text, "（")) {
 			tagindex := 0
 			if tagindex = strings.Index(td.Text, "（"); tagindex <= 0 {
 				tagindex = strings.Index(td.Text, "(")
 			}
-			td.SortKV.AddKey(strings.TrimSpace(td.Text [:tagindex]), strings.TrimSpace(td.Text [tagindex:])) //存放kv值
+			td.SortKV.AddKey(strings.TrimSpace(td.Text[:tagindex]), strings.TrimSpace(td.Text[tagindex:])) //存放kv值
 			td.BH = true
 		}
-		_, resm := colonkvEntity.entrance(td.Val, kvTitle, 3) //td冒号kv
+		_, resm := colonkvEntity.entrance(td.Val, kvTitle, nil, 3) //td冒号kv
 		for k, v := range resm {
 			td.SortKV.AddKey(k, v) //存放kv值
 		}
@@ -381,7 +382,7 @@ func (td *TD) tdIsKey(tr *TR, table *Table, bsontable bool) {
 		if len(td.TR.TDs) > 0 {
 			kvTitle = td.TR.TDs[len(td.TR.TDs)-1].Val
 		}
-		_, resm := colonkvEntity.entrance(td.Val, kvTitle, 2)
+		_, resm := colonkvEntity.entrance(td.Val, kvTitle, nil, 2) //获取冒号kv入口
 		for k, v := range resm {
 			td.SortKV.AddKey(k, v)
 		}

src/jy/util/article.go

@@ -53,6 +53,7 @@ type RuleBlock struct {
 type Block struct {
 	Tags        []Tags                   //对块做的标签，可以作为数据抽取的依据
 	Title       string                   //块标题
+	Titles      []string                 //拆分以后多个块标题
 	Index       int                      //块索引
 	Text        string                   //块内容
 	Start       int                      //开始索引
@@ -63,6 +64,7 @@ type Block struct {
 	BPackage    *BlockPackage            //分包信息
 	Tag         map[string]bool          //块标签
 	Block       []*Block                 //子块
+	Category    string                   //块分类
 	Winnerorder []map[string]interface{} //块中，中标候选人排序
 }
 

src/res/blocktagdb/中标单位.txt

@@ -39,7 +39,6 @@
 推荐中标候选人排序
 确定入围评标投标人
 第一中标侯选人
-第一中标候选人
 第一包中标人
 预中标人
 询价小组推荐的成交候选人

src/res/formattext.json

@@ -20,11 +20,6 @@
             "separator": " ",
             "desc": "替换掉无效的kv"
         },
-        {
-            "reg": "[^\\n:：]{2,18}[:：]\\s*详见[^,。，.：:\\s]{2,18}",
-            "separator": "",
-            "desc": "替换掉无效的kv"
-        },
         {
             "reg": "(\\d+[，,.]+)+\\d+((百|千)?元|(百|千)?(万|亿)元?)",
             "separator": "[，,]__",
@@ -182,6 +177,11 @@
             "reg": "\n[\\d.\u3000\u2003\u00a0\\s]*(联系人)及(电话)[:：](.+?)[\u3000\u2003\u00a0\\s]+(.+)",
             "separator": "\n$1：$3\n$2：$4",
             "desc": ""
+        },
+        {
+            "reg": "[^\\n:：]{2,18}[:：]\\s*详见[^,。，.：:\\s]{2,18}",
+            "separator": "",
+            "desc": "替换掉无效的kv"
         }
     ]
 }

src/udpfileserver/main.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
+	"github.com/go-gomail/gomail"
 	"gopkg.in/mgo.v2/bson"
 	"jy/mongodbutil"
 	"log"
@@ -9,8 +11,11 @@ import (
 	"net"
 	"net/rpc"
 	"path"
+	"qfw/common/src/qfw/util"
 	qu "qfw/util"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -61,7 +66,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 		lid := strings.TrimSpace(mapInfo["lteid"].(string))
 		if bson.IsObjectIdHex(gid) && bson.IsObjectIdHex(lid) {
 			var jsq int64
-			query := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(gid),"$lte": bson.ObjectIdHex(lid),}}
+			query := bson.M{"_id": bson.M{"$gt": bson.ObjectIdHex(gid),"$lte": bson.ObjectIdHex(lid),}}
 			log.Println("query---:", query)
 			sum :=mongodbutil.Mgo.Count(MgoC,query)
 			log.Println("sum:", sum)
@@ -71,7 +76,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 				limit = sum
 			}
 			for i := 0; i < pageNum; i++ {
-				query = bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(gid), "$lte": bson.ObjectIdHex(lid)}}
+				query = bson.M{"_id": bson.M{"$gt": bson.ObjectIdHex(gid), "$lte": bson.ObjectIdHex(lid)}}
 				log.Println("page=", i+1,"query=", query,limit)
 				list, b := mongodbutil.Mgo.Find(MgoC,query,nil,bson.M{"_id": 1,MgoFileFiled:1},false,0, limit)
 				if !b{
@@ -103,6 +108,11 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 										log.Println(mid, "mgo ", MgoFileFiled,"没有fid ")
 										continue
 									}
+									//if qu.ObjToString(fileinfo["update"]) ==""{
+									//	<-ChanB
+									//	log.Println(mid, "mgo ", MgoFileFiled,"没有update ")
+									//	continue
+									//}
 									save(mid,attk, qmap, &fileinfo,&updateNum)
 									<-ChanB
 								}
@@ -111,7 +121,10 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					}
 				}
 			}
+			//识别完以后再次查询数据库，进行下一轮识别
 			log.Println("处理查询数据结束...",jsq,time.Now().Sub(stime))
+			//进行下一轮识别
+			forfunc(lid)
 		} else {
 			log.Println("开始id或结束id参数错误:", string(data))
 		}
@@ -185,6 +198,7 @@ func save(mid interface{},attk string, qmap, fileinfo *map[string]interface{},up
 			(*fileinfo)["content"] = rdata["context"]
 		}
 		(*fileinfo)["expend"] = rdata["expend"]
+		delete(*fileinfo,"update")
 		//log.Println((*fileinfo))
 
 		(*qmap)[MgoFileFiled].(map[string]interface{})["attachments"].(map[string]interface{})[attk]=*fileinfo
@@ -208,8 +222,81 @@ func save(mid interface{},attk string, qmap, fileinfo *map[string]interface{},up
 		}else {
 			log.Println(mid, "mongo更新数据失败",qu.ObjToString((*fileinfo)["fid"]))
 		}
+		nowHour := time.Now().Hour()
+		rdlock.Lock()
+		if nowHour != hourNum{
+			log.Println("send email:",SendMail(fmt.Sprint(updateBool,mid)))
+			hourNum = nowHour
+		}
+		rdlock.Unlock()
 	} else {
 		log.Println(mid, "调用rpc服务解析异常：",qu.ObjToString((*fileinfo)["fid"]), rdata["err"])
 	}
 
 }
+var hourNum int
+var rdlock sync.RWMutex
+func SendMail( body string ) error {
+	//定义邮箱服务器连接信息，如果是阿里邮箱 pass填密码，qq邮箱填授权码
+	mailConn := map[string]string {
+		"user": "550838476@qq.com",
+		"pass": "",
+		"host": "smtp.qq.com",
+		"port": "465",
+	}
+
+	port, _ := strconv.Atoi(mailConn["port"]) //转换端口类型为int
+
+	m := gomail.NewMessage()
+	m.SetHeader("From","Get to" + "<" + mailConn["user"] + ">")  //这种方式可以添加别名，即“XD Game”， 也可以直接用<code>m.SetHeader("From",mailConn["user"])</code> 读者可以自行实验下效果
+	m.SetHeader("To", []string{"550838476@qq.com"}...)  //发送给多个用户
+	m.SetHeader("Subject", "MongoId")  //设置邮件主题
+	m.SetBody("text/html", body)     //设置邮件正文
+
+	d := gomail.NewDialer(mailConn["host"], port, mailConn["user"], mailConn["pass"])
+
+	err := d.DialAndSend(m)
+	return err
+
+}
+
+func forfunc(lid string) {
+	for {
+		//查询最后一个id
+		lastObjectId, _ := mongodbutil.Mgo.Find(MgoC,nil,"-_id",bson.M{"_id":1},true,-1,-1)
+		lastId,ok := (*lastObjectId)[0]["_id"].(bson.ObjectId)
+		log.Println("lastID:",lastId)
+		//查询最后一个id出错重新查询
+		if!ok{//转换失败
+			log.Println("查询异常",*lastObjectId)
+			time.Sleep(time.Minute)
+			continue
+		}
+		//查询最后一个id等于上一轮的id就重新查询
+		if lastId.Hex() == lid {
+			log.Println("没有新数据",lastId.Hex())
+			SendMail(time.Now().String()+"没有最新数据,当前最后一条数据id:"+lastId.Hex())
+			time.Sleep(time.Hour)
+			continue
+		}
+		//不相等说明有新数据，进行下次处理
+		m := map[string]string{
+			"gtid":lid,//上一轮结束的最后id
+			"lteid":lastId.Hex(),//新一轮查询出来的id
+		}
+		bytes, _ := json.Marshal(m)
+		//发送udp
+		err := udpclient.WriteUdp(bytes,mu.OP_TYPE_DATA,&net.UDPAddr{
+			IP:   net.ParseIP( util.ObjToString(Sysconfig["udpip"])),
+			Port:  util.IntAll(Sysconfig["udpport"]),
+		})
+		if err != nil{
+			log.Println("发送udp失败",err,string(bytes))
+			time.Sleep(time.Minute)
+			continue
+		}
+		SendMail(time.Now().String()+fmt.Sprint("发送udp成功,gtid：",lid,"，lteid:",lastId.Hex()))
+		log.Println("发送udp成功,gtid：",lid,"，lteid:",lastId.Hex())
+		break//发送完后终止循环
+	}
+}

src/web/templates/admin/clearversion.html

@@ -168,7 +168,7 @@ function use(_id,isuse){
 			data:{"_id":_id,"isuse":isuse},
 			success:function(r){
 				if(r.rep){				
-					window.location.href="/admin/clearversion"	
+					window.location.href="/admin/versionclear"	
 				}else{
 					showTip("启用失败", 1000, function() {});
 				}