Merge branch 'dev3.2' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.2

src/jy/admin/user.go

@@ -2,21 +2,22 @@
 package admin
 
 import (
+	"encoding/json"
+	"jy/front"
 	. "jy/mongodbutil"
 	. "jy/util"
 	"net/http"
+	qu "qfw/util"
 	"time"
-	"github.com/gin-gonic/gin"
+
 	"github.com/gin-contrib/sessions"
-	"strconv"
+	"github.com/gin-gonic/gin"
 	"gopkg.in/mgo.v2/bson"
-	qu "qfw/util"
-	"encoding/json"
 )
 
 func init() {
 	Admin.GET("/index", func(c *gin.Context) {
-		c.HTML(http.StatusOK, "index.html",nil)
+		c.HTML(http.StatusOK, "index.html", nil)
 	})
 	Admin.GET("/user", func(c *gin.Context) {
 		c.HTML(http.StatusOK, "user.html", gin.H{})
@@ -28,31 +29,31 @@ func init() {
 		c.HTML(http.StatusOK, "role.html", gin.H{})
 	})
 	Admin.GET("/role/menu", func(c *gin.Context) {
-		role:=c.Query("role")
-		c.HTML(http.StatusOK, "rolemenu.html", gin.H{"role":role})
+		role := c.Query("role")
+		c.HTML(http.StatusOK, "rolemenu.html", gin.H{"role": role})
 	})
 	Admin.GET("/role/secondmenu", func(c *gin.Context) {
-		role:=c.Query("role")
-		_id:=c.Query("_id")
-		c.HTML(http.StatusOK, "rolesecondmenu.html", gin.H{"role":role,"_id":_id})
+		role := c.Query("role")
+		_id := c.Query("_id")
+		c.HTML(http.StatusOK, "rolesecondmenu.html", gin.H{"role": role, "_id": _id})
 	})
 	Admin.GET("/secondmenu", func(c *gin.Context) {
 		_id := c.Query("id")
-		c.HTML(http.StatusOK, "secondmenu.html", gin.H{"_id":_id})
+		c.HTML(http.StatusOK, "secondmenu.html", gin.H{"_id": _id})
 	})
 	Admin.POST("/menu", Menu)
-	Admin.POST("/menu/save",MenuSave)
-	Admin.POST("/menu/data",MenuData)
+	Admin.POST("/menu/save", MenuSave)
+	Admin.POST("/menu/data", MenuData)
 	Admin.POST("/menu/searchbyid", MenuSearchById)
 	Admin.POST("/menu/del", MenuDel)
-	Admin.POST("/role/menu/data",RoleMenuData)
-	Admin.POST("/role/menu/save",RoleMenuSave)
-	Admin.POST("/role/select",RoleSelect)
-	Admin.POST("/role/secondmenu/data",RoleSecondMenuData)
-	Admin.POST("/role/menu/del",RoleMenuDel)
-	Admin.POST("/role/secondmenu/del",RoleSecondMenuDel)
-	Admin.POST("/secondmenu/data",SecondMenuData)
-	Admin.POST("/secondmenu/save",SecondMenuSave)
+	Admin.POST("/role/menu/data", RoleMenuData)
+	Admin.POST("/role/menu/save", RoleMenuSave)
+	Admin.POST("/role/select", RoleSelect)
+	Admin.POST("/role/secondmenu/data", RoleSecondMenuData)
+	Admin.POST("/role/menu/del", RoleMenuDel)
+	Admin.POST("/role/secondmenu/del", RoleSecondMenuDel)
+	Admin.POST("/secondmenu/data", SecondMenuData)
+	Admin.POST("/secondmenu/save", SecondMenuSave)
 	Admin.POST("/secondmenu/searchbyid", SecondMenuSearchById)
 	Admin.POST("/secondmenu/del", SecondMenuDel)
 	Admin.POST("/user/data", User)
@@ -63,7 +64,12 @@ func init() {
 }
 
 func User(c *gin.Context) {
-	data, _ := Mgo.Find("user", `{}`, nil, nil, false, -1, -1)
+	maps := map[string]interface{}{
+		"role": map[string]interface{}{
+			"$ne": "3",
+		},
+	}
+	data, _ := Mgo.Find("user", maps, nil, nil, false, -1, -1)
 	c.JSON(200, gin.H{"data": data})
 }
 func MenuData(c *gin.Context) {
@@ -71,152 +77,173 @@ func MenuData(c *gin.Context) {
 	c.JSON(200, gin.H{"data": data})
 }
 func SecondMenuData(c *gin.Context) {
-	_id,_:= c.GetPostForm("_id")
-	maps:=map[string]interface{}{
-		"menuid":_id,
+	_id, _ := c.GetPostForm("_id")
+	maps := map[string]interface{}{
+		"menuid": _id,
 	}
-	data, _ := Mgo.Find("secondmenu",maps, nil, nil, false, -1, -1)
+	data, _ := Mgo.Find("menusecond", maps, nil, nil, false, -1, -1)
 	c.JSON(200, gin.H{"data": data})
 }
 func RoleMenuData(c *gin.Context) {
-	role,_:=c.GetPostForm("role")
+	role, _ := c.GetPostForm("role")
 	maps := map[string]interface{}{
-		"role."+role: true,
+		"role." + role: true,
 	}
-	datas, _ := Mgo.Find("menu",maps, nil, nil, false, -1, -1)
-	list:=[]map[string]interface{}{}
-	for _,value:= range *datas{
-		_id:=qu.BsonIdToSId(value["_id"])
-		maps:=map[string]interface{}{
+	datas, _ := Mgo.Find("menu", maps, nil, nil, false, -1, -1)
+	list := []map[string]interface{}{}
+	for _, value := range *datas {
+		_id := qu.BsonIdToSId(value["_id"])
+		maps := map[string]interface{}{
 			"menuid": _id,
 		}
-		count, _ := Mgo.Find("secondmenu",maps, nil, nil, false, -1, -1)
-		if len(*count)!=0{
-			value["secondmenu"]=true
-		}else{
-			value["secondmenu"]=false
+		count, _ := Mgo.Find("menusecond", maps, nil, nil, false, -1, -1)
+		if len(*count) != 0 {
+			value["secondmenu"] = true
+		} else {
+			value["secondmenu"] = false
 		}
-		list=append(list,value)
+		list = append(list, value)
 	}
 	c.JSON(200, gin.H{"data": list})
 }
-func RoleMenuSave(c *gin.Context){
-	menu,_:=c.GetPostForm("menu")
+func RoleMenuSave(c *gin.Context) {
+	menu, _ := c.GetPostForm("menu")
 	secondmenuStr, _ := c.GetPostForm("secondmenuStr")
-	role,_:=c.GetPostForm("role")
+	secondmenuStr2, _ := c.GetPostForm("secondmenuStr2")
+	role, _ := c.GetPostForm("role")
 	secondmenus := make([]string, 0)
+	secondmenus2 := make([]string, 0)
 	err := json.Unmarshal([]byte(secondmenuStr), &secondmenus)
-	if err == nil && len(secondmenus) > 0 {
+	err2 := json.Unmarshal([]byte(secondmenuStr2), &secondmenus2)
+	if err == nil && err2 == nil {
 		for _, v := range secondmenus {
-			maps:=map[string]interface{}{
-				"_id":bson.ObjectIdHex(v),
+			maps := map[string]interface{}{
+				"_id": bson.ObjectIdHex(v),
 			}
-			data:=map[string]interface{}{
-				"role."+role:true,
+			data := map[string]interface{}{
+				"role." + role: true,
 			}
-			data2:=map[string]interface{}{
-				"$set":data,
+			data2 := map[string]interface{}{
+				"$set": data,
 			}
-			Mgo.Update("secondmenu", maps, data2, true, false)
+			Mgo.Update("menusecond", maps, data2, true, false)
+		}
+		for _, v := range secondmenus2 {
+			maps := map[string]interface{}{
+				"_id": bson.ObjectIdHex(v),
+			}
+			data := map[string]interface{}{
+				"role." + role: false,
+			}
+			data2 := map[string]interface{}{
+				"$set": data,
+			}
+			Mgo.Update("menusecond", maps, data2, true, false)
 		}
 	}
-	maps:=map[string]interface{}{
-		"_id":bson.ObjectIdHex(menu),
-	}
-	data:=map[string]interface{}{
-		"role."+role:true,
+	b := false
+	if (len(secondmenus) == 0 && len(secondmenus2) == 0) || (len(secondmenus) > 0) {
+		maps := map[string]interface{}{
+			"_id": bson.ObjectIdHex(menu),
+		}
+		data := map[string]interface{}{
+			"role." + role: true,
+		}
+		data2 := map[string]interface{}{
+			"$set": data,
+		}
+		b = Mgo.Update("menu", maps, data2, true, false)
 	}
-	data2:=map[string]interface{}{
-		"$set":data,
+	if len(secondmenus) == 0 && len(secondmenus2) > 0 {
+		maps := map[string]interface{}{
+			"_id": bson.ObjectIdHex(menu),
+		}
+		data := map[string]interface{}{
+			"role." + role: false,
+		}
+		data2 := map[string]interface{}{
+			"$set": data,
+		}
+		b = Mgo.Update("menu", maps, data2, true, false)
 	}
-	b:=Mgo.Update("menu", maps, data2, true, false)
+
 	c.JSON(200, gin.H{"rep": b})
 }
-func RoleSelect(c *gin.Context){
-	menuid,_:=c.GetPostForm("_id")
-	maps := map[string]interface{}{
-		"menuid":menuid,
+func RoleSelect(c *gin.Context) {
+	menuid, _ := c.GetPostForm("_id")
+	mark, _ := c.GetPostForm("mark")
+	role, _ := c.GetPostForm("role")
+	if mark == "" {
+		maps := map[string]interface{}{
+			"menuid": menuid,
+		}
+		datas, _ := Mgo.Find("menusecond", maps, nil, nil, false, -1, -1)
+		c.JSON(200, gin.H{"data": datas})
+	} else {
+		maps := map[string]interface{}{
+			"menuid":       menuid,
+			"role." + role: true,
+		}
+		data, _ := Mgo.Find("menusecond", maps, nil, nil, false, -1, -1)
+		maps2 := map[string]interface{}{
+			"menuid":       menuid,
+			"role." + role: false,
+		}
+		names, _ := Mgo.FindById("menu", menuid, `{name:1}`)
+		name := *names
+		name1 := name["name"]
+		data2, _ := Mgo.Find("menusecond", maps2, nil, nil, false, -1, -1)
+		c.JSON(200, gin.H{"name": name1, "data": data, "data2": data2})
 	}
-	datas, _ := Mgo.Find("secondmenu",maps, nil, nil, false, -1, -1)
-	c.JSON(200, gin.H{"data": datas})
+
 }
 func RoleSecondMenuData(c *gin.Context) {
-	role,_:=c.GetPostForm("role")
-	_id,_:=c.GetPostForm("_id")
+	role, _ := c.GetPostForm("role")
+	_id, _ := c.GetPostForm("_id")
 	maps := map[string]interface{}{
-		"menuid":_id,
-		"role."+role: true,
+		"menuid":       _id,
+		"role." + role: true,
 	}
-	datas, _ := Mgo.Find("secondmenu",maps, nil, nil, false, -1, -1)
+	datas, _ := Mgo.Find("menusecond", maps, nil, nil, false, -1, -1)
 
 	c.JSON(200, gin.H{"data": datas})
 }
 func Menu(c *gin.Context) {
-	//管理员0，审核员1.开发员2，超级管理3
 	session := sessions.Default(c)
-	role:=session.Get("role").(string)
-	maps:=map[string]interface{}{
-	}
-	if role=="3"{
-		maps=map[string]interface{}{
-		}
-	}else {
-		maps = map[string]interface{}{
-			"role." + role: true,
-		}
-	}
-	data, _ := Mgo.Find("menu", maps, nil, nil, false, -1, -1)
-	list:=[]map[string]interface{}{}
-	for _,value:=range *data{
-		_id:=value["_id"]
-		if role=="3"{
-			maps=map[string]interface{}{
-				"menuid":qu.BsonIdToSId(_id),
-			}
-		}else {
-			maps = map[string]interface{}{
-				"role." + role: true,
-				"menuid":qu.BsonIdToSId(_id),
-			}
-		}
-		secdatas, _ := Mgo.Find("secondmenu", maps, nil, nil, false, -1, -1)
-		secmenumap:=map[string]interface{}{}
-		for index,secdata:=range *secdatas{
-			secmenumap[strconv.Itoa(index+1)]=secdata
-		}
-		if len(secmenumap)!=0{
-			value["secondmenu"]=secmenumap
-		}
-		list=append(list,value)
-	}
-	c.JSON(200, gin.H{"data": list,"role":role})
+	u := session.Get("user").(*map[string]interface{})
+	list := front.UserMenu[qu.ObjToString((*u)["email"])]
+	c.JSON(200, gin.H{"data": list})
 }
 func MenuSave(c *gin.Context) {
 	name, _ := c.GetPostForm("name")
 	href, _ := c.GetPostForm("href")
 	pic, _ := c.GetPostForm("pic")
-	_id,_:=c.GetPostForm("_id")
-	data := map[string]interface{}{
-	}
-	if _id==""{
+	_id, _ := c.GetPostForm("_id")
+	data := map[string]interface{}{}
+	if _id == "" {
 		data = map[string]interface{}{
 			"name": name,
-			"href":   href,
+			"href": href,
 			"pic":  pic,
+			"role": map[string]interface{}{
+				"0": false,
+				"1": false,
+				"2": false,
+			},
 		}
-		b:=Mgo.Save("menu",data)
+		b := Mgo.Save("menu", data)
 		c.JSON(200, gin.H{"rep": b})
-	}else{
+	} else {
 		data = map[string]interface{}{
 			"name": name,
-			"href":   href,
+			"href": href,
 			"pic":  pic,
 		}
-		data2:=map[string]interface{}{
-			"$set":data,
+		data2 := map[string]interface{}{
+			"$set": data,
 		}
-		maps:=map[string]interface{}{
-			"_id":bson.ObjectIdHex(_id),
+		maps := map[string]interface{}{
+			"_id": bson.ObjectIdHex(_id),
 		}
 		b := Mgo.Update("menu", maps, data2, true, false)
 		c.JSON(200, gin.H{"rep": b})
@@ -229,30 +256,34 @@ func SecondMenuSave(c *gin.Context) {
 	pic, _ := c.GetPostForm("pic")
 	_id, _ := c.GetPostForm("_id")
 	menuid, _ := c.GetPostForm("menuid")
-	data := map[string]interface{}{
-	}
-	if _id==""{
+	data := map[string]interface{}{}
+	if _id == "" {
 		data = map[string]interface{}{
-			"name": name,
+			"name":   name,
 			"href":   href,
-			"pic":  pic,
-			"menuid":menuid,
+			"pic":    pic,
+			"menuid": menuid,
+			"role": map[string]interface{}{
+				"0": false,
+				"1": false,
+				"2": false,
+			},
 		}
-		b:=Mgo.Save("secondmenu",data)
+		b := Mgo.Save("menusecond", data)
 		c.JSON(200, gin.H{"rep": b})
-	}else {
+	} else {
 		data = map[string]interface{}{
-			"name":   name,
-			"href":   href,
-			"pic":    pic,
+			"name": name,
+			"href": href,
+			"pic":  pic,
 		}
-		data2:=map[string]interface{}{
-			"$set":data,
+		data2 := map[string]interface{}{
+			"$set": data,
 		}
-		maps:=map[string]interface{}{
-			"_id":bson.ObjectIdHex(_id),
+		maps := map[string]interface{}{
+			"_id": bson.ObjectIdHex(_id),
 		}
-		b := Mgo.Update("secondmenu",maps, data2, true, false)
+		b := Mgo.Update("menusecond", maps, data2, true, false)
 		c.JSON(200, gin.H{"rep": b})
 	}
 
@@ -286,7 +317,7 @@ func MenuSearchById(c *gin.Context) {
 }
 func SecondMenuSearchById(c *gin.Context) {
 	_id, _ := c.GetPostForm("_id")
-	data, _ := Mgo.FindById("secondmenu", _id, nil)
+	data, _ := Mgo.FindById("menusecond", _id, nil)
 	c.JSON(200, gin.H{"rep": data})
 }
 
@@ -302,32 +333,32 @@ func MenuDel(c *gin.Context) {
 }
 func SecondMenuDel(c *gin.Context) {
 	_id, _ := c.GetPostForm("_id")
-	b := Mgo.Del("secondmenu", `{"_id":"`+_id+`"}`)
+	b := Mgo.Del("menusecond", `{"_id":"`+_id+`"}`)
 	c.JSON(200, gin.H{"rep": b})
 }
 func RoleMenuDel(c *gin.Context) {
 	_id, _ := c.GetPostForm("_id")
 	role, _ := c.GetPostForm("role")
-	maps:=map[string]interface{}{
-		"_id":bson.ObjectIdHex(_id),
+	maps := map[string]interface{}{
+		"_id": bson.ObjectIdHex(_id),
 	}
-	data:=map[string]interface{}{
-		"role."+role:false,
+	data := map[string]interface{}{
+		"role." + role: false,
 	}
-	data2:=map[string]interface{}{
-		"$set":data,
+	data2 := map[string]interface{}{
+		"$set": data,
 	}
 	b := Mgo.Update("menu", maps, data2, true, false)
-	maps=map[string]interface{}{
-		"menuid":_id,
+	maps = map[string]interface{}{
+		"menuid": _id,
 	}
-	count,_:=Mgo.Find("secondmenu",maps,nil,nil,false,-1,-1)
-	if len(*count)!=0{
-		for _,c:=range *count{
-			maps=map[string]interface{}{
-				"_id":c["_id"],
+	count, _ := Mgo.Find("menusecond", maps, nil, nil, false, -1, -1)
+	if len(*count) != 0 {
+		for _, c := range *count {
+			maps = map[string]interface{}{
+				"_id": c["_id"],
 			}
-			Mgo.Update("secondmenu", maps, data2, true, false)
+			Mgo.Update("menusecond", maps, data2, true, false)
 		}
 
 	}
@@ -336,16 +367,16 @@ func RoleMenuDel(c *gin.Context) {
 func RoleSecondMenuDel(c *gin.Context) {
 	_id, _ := c.GetPostForm("_id")
 	role, _ := c.GetPostForm("role")
-	maps:=map[string]interface{}{
-		"_id":bson.ObjectIdHex(_id),
+	maps := map[string]interface{}{
+		"_id": bson.ObjectIdHex(_id),
 	}
-	data:=map[string]interface{}{
-		"role."+role:false,
+	data := map[string]interface{}{
+		"role." + role: false,
 	}
-	data2:=map[string]interface{}{
-		"$set":data,
+	data2 := map[string]interface{}{
+		"$set": data,
 	}
-	b := Mgo.Update("secondmenu", maps, data2, true, false)
+	b := Mgo.Update("menusecond", maps, data2, true, false)
 	c.JSON(200, gin.H{"rep": b})
 }
 func UserUppwd(c *gin.Context) {

src/jy/extract/extract.go

@@ -8,7 +8,6 @@ import (
 	db "jy/mongodbutil"
 	"jy/pretreated"
 	ju "jy/util"
-	"log"
 	qu "qfw/util"
 	"qfw/util/redis"
 	"reflect"
@@ -18,6 +17,7 @@ import (
 	"time"
 	"unicode/utf8"
 
+	log "github.com/donnie4w/go-logger/logger"
 	"gopkg.in/mgo.v2/bson"
 )
 
@@ -134,8 +134,8 @@ func StartExtractTaskId(taskId string) bool {
 	ext.InitFile()
 
 	ext.IsRun = true
-	go ext.ResultSave()
-	go ext.BidSave()
+	go ext.ResultSave(true)
+	go ext.BidSave(true)
 	if isgo {
 		go RunExtractTask(taskId)
 	}
@@ -167,17 +167,17 @@ func RunExtractTask(taskId string) {
 	if count < PageSize {
 		limit = count
 	}
-	log.Printf("count=%d,pageNum=%d,query=%v", count, pageNum, query)
+	fmt.Printf("count=%d,pageNum=%d,query=%v", count, pageNum, query)
 	for i := 0; i < pageNum; i++ {
 		query = bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(ext.TaskInfo.LastExtId)}}
-		log.Printf("page=%d,query=%v", i+1, query)
+		fmt.Printf("page=%d,query=%v", i+1, query)
 		list, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
 		for _, v := range *list {
 			if qu.ObjToString(v["sensitive"]) != "" { //去除含敏感词数据
 				continue
 			}
 			_id := qu.BsonIdToSId(v["_id"])
-			log.Println(_id)
+			log.Debug(_id)
 			if !ext.IsRun {
 				break
 			}
@@ -266,7 +266,7 @@ func PreInfo(doc map[string]interface{}) (j, jf *ju.Job) {
 			pretreated.AnalyStart(jf)
 		}
 	}, func(err interface{}) {
-		log.Println("pretreated.AnalyStart", err)
+		log.Debug("pretreated.AnalyStart", err)
 	})
 	return j, jf
 }
@@ -332,13 +332,13 @@ func (e *ExtractTask) ExtractDetail(j *ju.Job) {
 			for _, v := range vc.RulePres {
 				tmp = ExtRegPre(tmp, j, v, e.TaskInfo)
 			}
-			//log.Println("抽取-前置规则", tmp)
+			// log.Debug("抽取-前置规则", tmp)
 
 			//抽取-规则
 			for _, v := range vc.RuleCores {
 				ExtRegCore(vc.ExtFrom, tmp, j, v, e)
 			}
-			//log.Println("抽取-规则", tmp)
+			// log.Debug("抽取-规则", tmp)
 
 			//项目名称未能抽取到，标题来凑
 			if vc.Field == "projectname" {
@@ -351,7 +351,7 @@ func (e *ExtractTask) ExtractDetail(j *ju.Job) {
 			for _, v := range vc.RuleBacks {
 				ExtRegBack(j, v, e.TaskInfo)
 			}
-			//log.Println("抽取-后置规则", tmp)
+			// log.Debug("抽取-后置规则", tmp)
 		}
 
 		//全局后置规则
@@ -403,9 +403,9 @@ func (e *ExtractTask) ExtractDetail(j *ju.Job) {
 		}
 		PackageDetail(j, e) //处理分包信息
 		//		bs, _ := json.Marshal(j.Result)
-		//		log.Println("抽取结果", j.Title, j.SourceMid, string(bs))
+		//		 log.Debug("抽取结果", j.Title, j.SourceMid, string(bs))
 	}, func(err interface{}) {
-		log.Println("ExtractProcess err", err)
+		log.Debug("ExtractProcess err", err)
 	})
 }
 func (e *ExtractTask) ExtractFile(j *ju.Job) {
@@ -413,7 +413,7 @@ func (e *ExtractTask) ExtractFile(j *ju.Job) {
 		doc := *j.Data
 		//全局前置规则,结果覆盖doc属性
 		for _, v := range e.RulePres {
-			if e.FileFields[v.Field] > 0 {
+			if value, ok := e.FileFields.Load(v.Field);ok && qu.IntAllDef(value,1) >0{
 				doc = ExtRegPre(doc, j, v, e.TaskInfo)
 			}
 		}
@@ -426,32 +426,32 @@ func (e *ExtractTask) ExtractFile(j *ju.Job) {
 			}
 			//抽取-前置规则
 			for _, v := range vc.RulePres {
-				if e.FileFields[vc.Field] > 0 {
+				if value, ok := e.FileFields.Load(v.Field);ok && qu.IntAllDef(value,1) >0{
 					tmp = ExtRegPre(tmp, j, v, e.TaskInfo)
 				}
 			}
-			//log.Println("抽取-前置规则", tmp)
+			// log.Debug("抽取-前置规则", tmp)
 
 			//抽取-规则
 			for _, v := range vc.RuleCores {
-				if e.FileFields[vc.Field] > 0 {
+				if value, ok := e.FileFields.Load(v.Field);ok && qu.IntAllDef(value,1) >0{
 					ExtRegCore(vc.ExtFrom, tmp, j, v, e)
 				}
 			}
-			//log.Println("抽取-规则", tmp)
+			// log.Debug("抽取-规则", tmp)
 
 			//抽取-后置规则
 			for _, v := range vc.RuleBacks {
-				if e.FileFields[vc.Field] > 0 {
+				if value, ok := e.FileFields.Load(v.Field);ok && qu.IntAllDef(value,1) >0{
 					ExtRegBack(j, v, e.TaskInfo)
 				}
 			}
-			//log.Println("抽取-后置规则", tmp)
+			// log.Debug("抽取-后置规则", tmp)
 		}
 
 		//全局后置规则
 		for _, v := range e.RuleBacks {
-			if e.FileFields[v.Field] > 0 {
+			if value, ok := e.FileFields.Load(v.Field);ok && qu.IntAllDef(value,1) >0{
 				ExtRegBack(j, v, e.TaskInfo)
 			}
 		}
@@ -501,9 +501,9 @@ func (e *ExtractTask) ExtractFile(j *ju.Job) {
 
 		PackageDetail(j, e) //处理分包信息
 		//		bs, _ := json.Marshal(j.Result)
-		//		log.Println("抽取结果", j.Title, j.SourceMid, string(bs))
+		//		 log.Debug("抽取结果", j.Title, j.SourceMid, string(bs))
 	}, func(err interface{}) {
-		log.Println("ExtractProcess err", err)
+		log.Debug("ExtractProcess err", err)
 	})
 }
 
@@ -591,8 +591,8 @@ func getKvByLuaFields(extfrom string, j *ju.Job, in *RegLuaInfo, t map[string][]
 			if bl.ColonKV != nil {
 				kvs := bl.ColonKV.Kvs
 				kvs2 := bl.ColonKV.Kvs_2
-				//log.Println("ColonKV1", kvs)
-				//log.Println("ColonKV2", kvs2)
+				// log.Debug("ColonKV1", kvs)
+				// log.Debug("ColonKV2", kvs2)
 				for _, tag := range tags {
 					for _, kv := range kvs {
 						if tag.Type == "string" {
@@ -669,7 +669,7 @@ func getKvByLuaFields(extfrom string, j *ju.Job, in *RegLuaInfo, t map[string][]
 			//空格kv
 			if bl.SpaceKV != nil {
 				kvs := bl.SpaceKV.Kvs
-				//log.Println("SpaceKV", kvs)
+				// log.Debug("SpaceKV", kvs)
 				for _, tag := range tags {
 					for _, kv := range kvs {
 						if tag.Type == "string" {
@@ -711,7 +711,7 @@ func getKvByLuaFields(extfrom string, j *ju.Job, in *RegLuaInfo, t map[string][]
 			//表格kv
 			if bl.TableKV != nil {
 				tkv := bl.TableKV
-				//log.Println("tkv", tkv)
+				// log.Debug("tkv", tkv)
 				for k, v := range tkv.Kv {
 					if k == fieldname {
 						if len(tags) > -tkv.KvIndex[fieldname] {
@@ -731,7 +731,7 @@ func getKvByLuaFields(extfrom string, j *ju.Job, in *RegLuaInfo, t map[string][]
 								"matchtype": "tag_string",
 							})
 						} else { //涉及其他待处理
-							//log.Println(tags)
+							// log.Debug(tags)
 						}
 					}
 				}
@@ -1031,7 +1031,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 		}
 		if e.IsExtractCity { //城市抽取
 			b, p, c, d := e.TransmitData(tmp, _id) //抽取省份城市
-			//log.Println("省份---", p, "城市---", c, "区---", d)
+			// log.Debug("省份---", p, "城市---", c, "区---", d)
 			tmp["district"] = d
 			if b {
 				tmp["city"] = c
@@ -1049,7 +1049,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 			if len(j.BrandData) > 0 {
 				tmp["tablebrand"] = j.BrandData
 			}
-			//log.Println("============", j.HasBrand, j.HasGoods, j.HasKey, j.HasTable, j.BrandData)
+			// log.Debug("============", j.HasBrand, j.HasGoods, j.HasKey, j.HasTable, j.BrandData)
 		}
 		if e.TaskInfo.TestColl == "" {
 			if len(tmp) > 0 { //保存抽取结果
@@ -1090,11 +1090,11 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 			tmp["resultf"] = resultf
 			b := db.Mgo.Update(e.TaskInfo.TestColl, `{"_id":"`+_id+`"}`, map[string]interface{}{"$set": tmp}, true, false)
 			if !b {
-				log.Println(e.TaskInfo.TestColl, _id)
+				log.Debug(e.TaskInfo.TestColl, _id)
 			}
 		}
 	}, func(err interface{}) {
-		log.Println("AnalysisSaveResult err", err)
+		log.Debug("AnalysisSaveResult err", err)
 	})
 }
 

src/jy/extract/extractInit.go

@@ -4,13 +4,15 @@ package extract
 import (
 	db "jy/mongodbutil"
 	ju "jy/util"
-	"log"
 	qu "qfw/util"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
+
+	log "github.com/donnie4w/go-logger/logger"
 )
 
 type RegLuaInfo struct { //正则或脚本信息
@@ -66,8 +68,8 @@ type ExtractTask struct {
 	IsExtractCity bool                //是否开启城市抽取
 	Fields        map[string]int      //抽取属性组
 
-	IsFileField       bool      //是否开启附件抽取
-	FileFields        map[string]int      //抽取附件属性组
+	IsFileField bool           //是否开启附件抽取
+	FileFields  *sync.Map //抽取附件属性组
 
 	ResultChanel chan bool                  //抽取结果详情
 	ResultArr    [][]map[string]interface{} //抽取结果详情
@@ -155,11 +157,11 @@ func (e *ExtractTask) InitTestTaskInfo(resultcoll, trackcoll string) {
 //加载任务信息
 func (e *ExtractTask) InitTaskInfo() {
 	task, _ := db.Mgo.FindById("task", e.Id, nil)
-	log.Println("task", task)
+	log.Debug("task", task)
 	if len(*task) > 1 {
 		v, _ := db.Mgo.FindOne("version", `{"version":"`+(*task)["s_version"].(string)+`","delete":false}`)
 		strs := strings.Split((*task)["s_mgosavecoll"].(string), "/")
-		log.Println("s_mgosavecoll", strs)
+		log.Debug("s_mgosavecoll", strs)
 		if len(strs) < 3 {
 			return
 		} else {
@@ -182,7 +184,7 @@ func (e *ExtractTask) InitTaskInfo() {
 				e.IsExtractCity = (*v)["isextractcity"].(bool)
 			}
 		}
-		log.Println(e.TaskInfo.Name, "thread:", qu.IntAllDef((*task)["i_process"], 1))
+		log.Debug(e.TaskInfo.Name, "thread:", qu.IntAllDef((*task)["i_process"], 1))
 	} else {
 		return
 	}
@@ -219,7 +221,7 @@ func (e *ExtractTask) InitRulePres() {
 				}
 				e.RulePres = append(e.RulePres, rinfo)
 			}, func(err interface{}) {
-				log.Println(rinfo.Code, rinfo.Field, err)
+				log.Debug(rinfo.Code, rinfo.Field, err)
 			})
 		}
 	}
@@ -256,7 +258,7 @@ func (e *ExtractTask) InitRuleBacks() {
 				}
 				e.RuleBacks = append(e.RuleBacks, rinfo)
 			}, func(err interface{}) {
-				log.Println(rinfo.Code, rinfo.Field, err)
+				log.Debug(rinfo.Code, rinfo.Field, err)
 			})
 		}
 	}
@@ -313,7 +315,7 @@ func (e *ExtractTask) InitRuleCore() {
 						}
 						rulePres = append(rulePres, rinfo)
 					}, func(err interface{}) {
-						log.Println(rinfo.Code, rinfo.Field, err)
+						log.Debug(rinfo.Code, rinfo.Field, err)
 					})
 				}
 			}
@@ -349,7 +351,7 @@ func (e *ExtractTask) InitRuleCore() {
 						}
 						ruleBacks = append(ruleBacks, rinfo)
 					}, func(err interface{}) {
-						log.Println(rinfo.Code, rinfo.Field, err)
+						log.Debug(rinfo.Code, rinfo.Field, err)
 					})
 				}
 			}
@@ -402,7 +404,7 @@ func (e *ExtractTask) InitRuleCore() {
 						}
 						ruleCores = append(ruleCores, rinfo)
 					}, func(err interface{}) {
-						log.Println(rinfo.Code, rinfo.Field, err)
+						log.Debug(rinfo.Code, rinfo.Field, err)
 					})
 				}
 			}
@@ -463,7 +465,7 @@ func (e *ExtractTask) InitPkgCore() {
 						}
 						ruleBacks = append(ruleBacks, rinfo)
 					}, func(err interface{}) {
-						log.Println(rinfo.Code, rinfo.Field, err)
+						log.Debug(rinfo.Code, rinfo.Field, err)
 					})
 				}
 			}
@@ -745,82 +747,83 @@ func (e *ExtractTask) InitDFA() {
 }
 
 //保存抽取详情数据
-func (e *ExtractTask) ResultSave() {
+func (e *ExtractTask) ResultSave(init bool) {
 	defer qu.Catch()
-	e.ResultChanel = make(chan bool, 5)
-	e.ResultArr = [][]map[string]interface{}{}
-	for {
-		if len(e.ResultArr) > 500 {
-			e.ResultChanel <- true
-			arr := e.ResultArr[:500]
-			go func(tmp *[][]map[string]interface{}) {
-				qu.Try(func() {
-					db.Mgo.UpSertBulk("extract_result", *tmp...)
-					<-e.ResultChanel
-				}, func(err interface{}) {
-					log.Println(err)
-					<-e.ResultChanel
-				})
-			}(&arr)
-			e.ResultArr = e.ResultArr[500:]
-		} else {
-			e.ResultChanel <- true
-			arr := e.ResultArr
-			go func(tmp *[][]map[string]interface{}) {
-				qu.Try(func() {
-					db.Mgo.UpSertBulk("extract_result", *tmp...)
-					<-e.ResultChanel
-				}, func(err interface{}) {
-					log.Println(err)
-					<-e.ResultChanel
-				})
-			}(&arr)
-			e.ResultArr = [][]map[string]interface{}{}
-			time.Sleep(10 * time.Second)
-		}
-		if !e.IsRun {
-			break
-		}
+	if e.ResultArr == nil {
+		e.ResultArr = [][]map[string]interface{}{}
+	}
+	if init {
+		go func() {
+			for {
+				if len(e.ResultArr) > 500 {
+					arr := e.ResultArr[:500]
+					qu.Try(func() {
+						db.Mgo.UpSertBulk("extract_result", arr...)
+					}, func(err interface{}) {
+						log.Debug(err)
+					})
+					e.ResultArr = e.ResultArr[500:]
+				} else {
+					arr := e.ResultArr
+					qu.Try(func() {
+						db.Mgo.UpSertBulk("extract_result", arr...)
+					}, func(err interface{}) {
+						log.Debug(err)
+					})
+					e.ResultArr = [][]map[string]interface{}{}
+				}
+				time.Sleep(10 * time.Second)
+			}
+		}()
+	} else {
+		arr := e.ResultArr
+		qu.Try(func() {
+			e.TaskInfo.TDB.UpSertBulk(e.TaskInfo.ToColl, arr...)
+		}, func(err interface{}) {
+			log.Debug(err)
+		})
+		e.ResultArr = [][]map[string]interface{}{}
 	}
 }
 
 //保存抽取数据
-func (e *ExtractTask) BidSave() {
+func (e *ExtractTask) BidSave(init bool) {
 	defer qu.Catch()
-	e.BidChanel = make(chan bool, 5)
-	e.BidArr = [][]map[string]interface{}{}
-	for {
-		if len(e.BidArr) > 500 {
-			e.BidChanel <- true
-			arr := e.BidArr[:500]
-			go func(tmp *[][]map[string]interface{}) {
-				qu.Try(func() {
-					db.Mgo.UpSertBulk(e.TaskInfo.ToColl, *tmp...)
-					<-e.BidChanel
-				}, func(err interface{}) {
-					log.Println(err)
-					<-e.BidChanel
-				})
-			}(&arr)
-			e.BidArr = e.BidArr[500:]
-		} else {
-			e.BidChanel <- true
-			arr := e.BidArr
-			go func(tmp *[][]map[string]interface{}) {
-				qu.Try(func() {
-					db.Mgo.UpSertBulk(e.TaskInfo.ToColl, *tmp...)
-					<-e.BidChanel
-				}, func(err interface{}) {
-					log.Println(err)
-					<-e.BidChanel
-				})
-			}(&arr)
-			e.BidArr = [][]map[string]interface{}{}
-		}
-		if !e.IsRun {
-			break
-		}
-		time.Sleep(10 * time.Second)
+	if e.BidArr == nil {
+		e.BidArr = [][]map[string]interface{}{}
+	}
+	if init {
+		go func() {
+			for {
+				if len(e.BidArr) > 500 {
+					arr := e.BidArr[:500]
+					qu.Try(func() {
+						e.TaskInfo.TDB.UpSertBulk(e.TaskInfo.ToColl, arr...)
+					}, func(err interface{}) {
+						log.Debug(err)
+					})
+					e.BidArr = e.BidArr[500:]
+				} else {
+					arr := e.BidArr
+					qu.Try(func() {
+						e.TaskInfo.TDB.UpSertBulk(e.TaskInfo.ToColl, arr...)
+					}, func(err interface{}) {
+						log.Debug(err)
+					})
+					e.BidArr = [][]map[string]interface{}{}
+				}
+				time.Sleep(10 * time.Second)
+			}
+		}()
+	} else {
+		arr := e.BidArr
+		qu.Try(func() {
+			e.TaskInfo.TDB.UpSertBulk(e.TaskInfo.ToColl, arr...)
+		}, func(err interface{}) {
+			log.Debug(err)
+		})
+		e.BidArr = [][]map[string]interface{}{}
+		time.Sleep(1 * time.Second)
 	}
 }
 
@@ -867,7 +870,7 @@ func (e *ExtractTask) InitAuditRule() {
 				ru = string(rs[1 : len(rs)-1])
 				rureg, err = regexp.Compile(ru)
 				if err != nil {
-					log.Println("error---rule:", r)
+					log.Debug("error---rule:", r)
 					continue
 				}
 				i_rule = append(i_rule, []interface{}{rureg}...)
@@ -914,19 +917,20 @@ func (e *ExtractTask) InitFile() {
 	//query:=bson.M{"version":e.TaskInfo.Version,"delete":false}
 	ve, _ := db.Mgo.FindOne("version", `{"version":"`+e.TaskInfo.Version+`","delete":false}`)
 	//ve, _ := db.Mgo.FindOne("version", query)
-	if ve == nil{
+	if ve == nil {
 		return
 	}
-	if (*ve)["isfiles"]!=nil && (*ve)["isfiles"].(bool){
-		e.IsFileField =true
+	if (*ve)["isfiles"] != nil && (*ve)["isfiles"].(bool) {
+		e.IsFileField = true
 	}
-	efiled := make(map[string]int,0)
-	if (*ve)["s_filefileds"] != nil{
-		for _,vff :=range (*ve)["s_filefileds"].([]interface{}) {
-			efiled[vff.(string)]=1
+	syscefiled := new(sync.Map)
+
+	if (*ve)["s_filefileds"] != nil {
+		for _, vff := range (*ve)["s_filefileds"].([]interface{}) {
+			syscefiled.Store(vff.(string),1)
 		}
 	}
-	e.FileFields = efiled
+	e.FileFields = syscefiled
 }
 
 //加载清理任务信息
@@ -944,7 +948,7 @@ func (c *ClearTask) InitClearTaskInfo() {
 			IsCltLog:    ju.Config["iscltlog"].(bool),
 			ProcessPool: make(chan bool, qu.IntAllDef((*cleartask)["i_process"], 1)),
 		}
-		log.Println(c.ClearTaskInfo.Name, "thread:", qu.IntAllDef((*cleartask)["i_process"], 1))
+		log.Debug(c.ClearTaskInfo.Name, "thread:", qu.IntAllDef((*cleartask)["i_process"], 1))
 	} else {
 		return
 	}

src/jy/extract/extractudp.go

@@ -3,13 +3,15 @@ package extract
 
 import (
 	"encoding/json"
+	"fmt"
 	db "jy/mongodbutil"
 	ju "jy/util"
-	"log"
 	mu "mfw/util"
 	"net"
 	qu "qfw/util"
+	"sync"
 
+	log "github.com/donnie4w/go-logger/logger"
 	"gopkg.in/mgo.v2/bson"
 )
 
@@ -29,44 +31,49 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 		var rep map[string]interface{}
 		err := json.Unmarshal(data, &rep)
 		if err != nil {
-			log.Println(err)
+			log.Debug(err)
 		} else {
 			sid, _ := rep["gtid"].(string)
 			eid, _ := rep["lteid"].(string)
 			stype, _ := rep["stype"].(string)
-			if stype == "distributed" { //分布式抽取分支
-				log.Println("分布式抽取id段", sid, eid)
-				InstanceId := qu.ObjToString(rep["InstanceId"])
-				db.Mgo.Update("ecs", `{"InstanceId":"`+InstanceId+`"}`,
-					map[string]interface{}{
-						"$set": map[string]interface{}{
-							"extstatus": "running",
-						},
-					}, true, false)
-				ExtractByUdp(sid, eid, qu.ObjToString(rep["InstanceId"]))
-				db.Mgo.Update("ecs", `{"InstanceId":"`+InstanceId+`"}`,
-					map[string]interface{}{
-						"$set": map[string]interface{}{
-							"extstatus": "ok",
-						},
-					}, true, false)
-				log.Println("分布式抽取完成", sid, eid, "释放esc实例", qu.ObjToString(rep["ip"]))
+			if sid == "" || eid == "" {
+				log.Debug("err", "sid=", sid, "eid=", eid)
 			} else {
-				log.Println("udp通知抽取id段", sid, eid)
-				ExtractByUdp(sid, eid)
-				log.Println("udp通知抽取完成,eid=", eid)
-				for _, m := range nextNodes {
-					by, _ := json.Marshal(map[string]interface{}{
-						"gtid":  sid,
-						"lteid": eid,
-						"stype": qu.ObjToString(m["stype"]),
-					})
-					err := Udpclient.WriteUdp(by, mu.OP_TYPE_DATA, &net.UDPAddr{
-						IP:   net.ParseIP(m["addr"].(string)),
-						Port: qu.IntAll(m["port"]),
-					})
-					if err != nil {
-						log.Println(err)
+				go Udpclient.WriteUdp([]byte("udpok"), mu.OP_NOOP, ra)
+				if stype == "distributed" { //分布式抽取分支
+					log.Debug("分布式抽取id段", sid, eid)
+					InstanceId := qu.ObjToString(rep["InstanceId"])
+					db.Mgo.Update("ecs", `{"InstanceId":"`+InstanceId+`"}`,
+						map[string]interface{}{
+							"$set": map[string]interface{}{
+								"extstatus": "running",
+							},
+						}, true, false)
+					ExtractByUdp(sid, eid, qu.ObjToString(rep["InstanceId"]))
+					db.Mgo.Update("ecs", `{"InstanceId":"`+InstanceId+`"}`,
+						map[string]interface{}{
+							"$set": map[string]interface{}{
+								"extstatus": "ok",
+							},
+						}, true, false)
+					log.Debug("分布式抽取完成", sid, eid, "释放esc实例", qu.ObjToString(rep["ip"]))
+				} else {
+					log.Debug("udp通知抽取id段", sid, eid)
+					ExtractByUdp(sid, eid)
+					log.Debug("udp通知抽取完成,eid=", eid)
+					for _, m := range nextNodes {
+						by, _ := json.Marshal(map[string]interface{}{
+							"gtid":  sid,
+							"lteid": eid,
+							"stype": qu.ObjToString(m["stype"]),
+						})
+						err := Udpclient.WriteUdp(by, mu.OP_TYPE_DATA, &net.UDPAddr{
+							IP:   net.ParseIP(m["addr"].(string)),
+							Port: qu.IntAll(m["port"]),
+						})
+						if err != nil {
+							log.Debug(err)
+						}
 					}
 				}
 			}
@@ -75,44 +82,46 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 		var rep map[string]interface{}
 		err := json.Unmarshal(data, &rep)
 		if err != nil {
-			log.Println(err)
+			log.Debug(err)
 		} else {
-			log.Println(rep)
+			log.Debug(rep)
 		}
 	}
 }
 
+var ext *ExtractTask
+
 //根据id区间抽取
 func ExtractByUdp(sid, eid string, instanceId ...string) {
 	defer qu.Catch()
-	ext := &ExtractTask{}
-	ext.Id = qu.ObjToString(ju.Config["udptaskid"])
-	ext.InitTaskInfo()
-	ext.TaskInfo.FDB = db.MgoFactory(3, 5, 600, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
-	ext.InitRulePres()
-	ext.InitRuleBacks()
-	ext.InitRuleCore()
-	ext.InitTag()
-	ext.InitClearFn()
-	if ext.IsExtractCity { //版本上控制是否开始城市抽取
-		//初始化城市DFA信息
-		ext.InitDFA()
-	}
-	//质量审核
-	ext.InitAuditFields()
-	ext.InitAuditRule()
-	ext.InitAuditClass()
-	ext.InitAuditRecogField()
-
-	//品牌抽取是否开启
-	ju.IsBrandGoods, _ = ju.Config["brandgoods"].(bool)
-	//附件抽取是否开启
-	ext.InitFile()
+	if ext == nil {
+		ext = &ExtractTask{}
+		ext.Id = qu.ObjToString(ju.Config["udptaskid"])
+		ext.InitTaskInfo()
+		ext.TaskInfo.FDB = db.MgoFactory(3, 5, 600, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
+		ext.TaskInfo.TDB = db.MgoFactory(3, 5, 600, ext.TaskInfo.ToDbAddr, ext.TaskInfo.ToDB)
+		ext.InitRulePres()
+		ext.InitRuleBacks()
+		ext.InitRuleCore()
+		ext.InitTag()
+		ext.InitClearFn()
+		if ext.IsExtractCity { //版本上控制是否开始城市抽取
+			//初始化城市DFA信息
+			ext.InitDFA()
+		}
+		//质量审核
+		ext.InitAuditFields()
+		ext.InitAuditRule()
+		ext.InitAuditClass()
+		ext.InitAuditRecogField()
 
-	go ext.ResultSave()
-	go ext.BidSave()
-	ext.IsRun = true
+		//品牌抽取是否开启
+		ju.IsBrandGoods, _ = ju.Config["brandgoods"].(bool)
 
+		ext.ResultSave(true)
+		ext.BidSave(true)
+		ext.IsRun = true
+	}
 	if len(instanceId) > 0 { //分布式抽取进度
 		query := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sid), "$lte": bson.ObjectIdHex(eid)}}
 		count1 := ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl, query)
@@ -123,7 +132,7 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 		if count < PageSize {
 			limit = count
 		}
-		log.Printf("count=%d,pageNum=%d,query=%v", count, pageNum, query)
+		fmt.Printf("count=%d,pageNum=%d,query=%v", count, pageNum, query)
 
 		startI := 0 //接着上次任务执行
 		sidback := sid
@@ -138,7 +147,7 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 
 		for i := startI; i < pageNum; i++ {
 			query = bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sid), "$lte": bson.ObjectIdHex(eid)}}
-			log.Printf("page=%d,query=%v", i+1, query)
+			fmt.Printf("page=%d,query=%v", i+1, query)
 			if ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl, query) > 0 {
 				list, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
 				for _, v := range *list {
@@ -146,7 +155,7 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 						continue
 					}
 					_id := qu.BsonIdToSId(v["_id"])
-					log.Println(_id)
+					log.Debug(_id)
 					var j, jf *ju.Job
 					if ext.IsFileField && v["projectinfo"] != nil {
 						v["isextFile"] = true
@@ -164,7 +173,7 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 					}}, true, false)
 			}
 			queryback := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sidback)}}
-			log.Printf("page=%d,queryback=%v", i+1, queryback)
+			fmt.Printf("page=%d,queryback=%v", i+1, queryback)
 			if ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl+"_back", queryback) > 0 {
 				list2, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl+"_back", queryback, nil, Fields, false, 0, limit)
 				for _, v := range *list2 {
@@ -172,7 +181,7 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 						continue
 					}
 					_id := qu.BsonIdToSId(v["_id"])
-					log.Println(_id)
+					log.Debug(_id)
 					var j, jf *ju.Job
 					if ext.IsFileField && v["projectinfo"] != nil {
 						v["isextFile"] = true
@@ -203,16 +212,16 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 		if count < PageSize {
 			limit = count
 		}
+		wg := sync.WaitGroup{}
 		for i := 0; i < pageNum; i++ {
 			query = bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sid)}}
-			log.Printf("page=%d,query=%v", i+1, query)
+			fmt.Printf("page=%d,query=%v", i+1, query)
 			list, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
-			for _, v := range *list {
+			for k, v := range *list {
 				if qu.ObjToString(v["sensitive"]) != "" { //去除含敏感词数据
 					continue
 				}
 				_id := qu.BsonIdToSId(v["_id"])
-				log.Println(_id)
 				var j, jf *ju.Job
 				if ext.IsFileField && v["projectinfo"] != nil {
 					v["isextFile"] = true
@@ -221,10 +230,18 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 					j, _ = PreInfo(v)
 				}
 				ext.TaskInfo.ProcessPool <- true
-				go ext.ExtractProcess(j, jf)
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					ext.ExtractProcess(j, jf)
+				}()
+				if k%1000 == 0 {
+					log.Debug(i, k, _id)
+				}
 				sid = _id
 			}
-
 		}
+		wg.Wait()
+		ext.BidSave(false)
 	}
 }

src/jy/front/front.go

@@ -7,12 +7,16 @@ import (
 	. "jy/util"
 	"net/http"
 	qu "qfw/util"
+	"strconv"
 
 	"github.com/gin-contrib/sessions"
 	"github.com/gin-gonic/gin"
 )
 
+var UserMenu map[string][]map[string]interface{}
+
 func init() {
+	UserMenu = make(map[string][]map[string]interface{})
 	front := Router.Group("/")
 	{
 		front.Static("/front", "./web/templates/front")
@@ -27,6 +31,7 @@ func Login(c *gin.Context) {
 	pwd := c.PostForm("pwd")
 	res, _ := Mgo.FindOne("user", fmt.Sprintf(`{"email":"%s","pwd":"%s"}`, email, Se.EncodeString(pwd)))
 	if res != nil && len(*res) > 0 {
+		UserMenu[qu.ObjToString((*res)["email"])] = GetUserMenu(qu.ObjToString((*res)["role"]))
 		session := sessions.Default(c)
 		(*res)["pwd"] = pwd
 		(*res)["_id"] = qu.BsonIdToSId((*res)["_id"])
@@ -44,3 +49,39 @@ func Logout(c *gin.Context) {
 	session.Delete("user")
 	c.Redirect(http.StatusMovedPermanently, "/front/login.html")
 }
+
+func GetUserMenu(role string) []map[string]interface{} {
+	list := []map[string]interface{}{}
+	maps := map[string]interface{}{}
+	if role == "3" {
+		maps = map[string]interface{}{}
+	} else {
+		maps = map[string]interface{}{
+			"role." + role: true,
+		}
+	}
+	data, _ := Mgo.Find("menu", maps, nil, nil, false, -1, -1)
+	for _, value := range *data {
+		_id := value["_id"]
+		if role == "3" {
+			maps = map[string]interface{}{
+				"menuid": qu.BsonIdToSId(_id),
+			}
+		} else {
+			maps = map[string]interface{}{
+				"role." + role: true,
+				"menuid":       qu.BsonIdToSId(_id),
+			}
+		}
+		secdatas, _ := Mgo.Find("menusecond", maps, nil, nil, false, -1, -1)
+		secmenumap := map[string]interface{}{}
+		for index, secdata := range *secdatas {
+			secmenumap[strconv.Itoa(index+1)] = secdata
+		}
+		if len(secmenumap) != 0 {
+			value["secondmenu"] = secmenumap
+		}
+		list = append(list, value)
+	}
+	return list
+}

src/jy/util/logger.go

@@ -0,0 +1,318 @@
+package util
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"strconv"
+	"sync"
+	"time"
+)
+
+const (
+	_VER string = "1.0.2"
+)
+
+type LEVEL int32
+
+var logLevel LEVEL = 1
+var maxFileSize int64
+var maxFileCount int32
+var dailyRolling bool = true
+var consoleAppender bool = true
+var RollingFile bool = false
+var logObj *_FILE
+
+const DATEFORMAT = "2006-01-02"
+
+type UNIT int64
+
+const (
+	_       = iota
+	KB UNIT = 1 << (iota * 10)
+	MB
+	GB
+	TB
+)
+
+const (
+	ALL LEVEL = iota
+	DEBUG
+	INFO
+	WARN
+	ERROR
+	FATAL
+	OFF
+)
+
+type _FILE struct {
+	dir      string
+	filename string
+	_suffix  int
+	isCover  bool
+	_date    *time.Time
+	mu       *sync.RWMutex
+	logfile  *os.File
+	lg       *log.Logger
+}
+
+func SetConsole(isConsole bool) {
+	consoleAppender = isConsole
+}
+
+func SetLevel(_level LEVEL) {
+	logLevel = _level
+}
+
+func SetRollingFile(fileDir, fileName string, maxNumber int32, maxSize int64, _unit UNIT) {
+	maxFileCount = maxNumber
+	maxFileSize = maxSize * int64(_unit)
+	RollingFile = true
+	dailyRolling = false
+	mkdirlog(fileDir)
+	logObj = &_FILE{dir: fileDir, filename: fileName, isCover: false, mu: new(sync.RWMutex)}
+	logObj.mu.Lock()
+	defer logObj.mu.Unlock()
+	for i := 1; i <= int(maxNumber); i++ {
+		if isExist(fileDir + "/" + fileName + "." + strconv.Itoa(i)) {
+			logObj._suffix = i
+		} else {
+			break
+		}
+	}
+	if !logObj.isMustRename() {
+		logObj.logfile, _ = os.OpenFile(fileDir+"/"+fileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
+		logObj.lg = log.New(logObj.logfile, "", log.Ldate|log.Ltime|log.Lshortfile)
+	} else {
+		logObj.rename()
+	}
+	go fileMonitor()
+}
+
+func SetRollingDaily(fileDir, fileName string) {
+	RollingFile = false
+	dailyRolling = true
+	t, _ := time.Parse(DATEFORMAT, time.Now().Format(DATEFORMAT))
+	mkdirlog(fileDir)
+	logObj = &_FILE{dir: fileDir, filename: fileName, _date: &t, isCover: false, mu: new(sync.RWMutex)}
+	logObj.mu.Lock()
+	defer logObj.mu.Unlock()
+
+	if !logObj.isMustRename() {
+		logObj.logfile, _ = os.OpenFile(fileDir+"/"+fileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
+		logObj.lg = log.New(logObj.logfile, "", log.Ldate|log.Ltime|log.Lshortfile)
+	} else {
+		logObj.rename()
+	}
+}
+
+func mkdirlog(dir string) (e error) {
+	_, er := os.Stat(dir)
+	b := er == nil || os.IsExist(er)
+	if !b {
+		if err := os.MkdirAll(dir, 0666); err != nil {
+			if os.IsPermission(err) {
+				fmt.Println("create dir error:", err.Error())
+				e = err
+			}
+		}
+	}
+	return
+}
+
+func console(s ...interface{}) {
+	if consoleAppender {
+		_, file, line, _ := runtime.Caller(2)
+		short := file
+		for i := len(file) - 1; i > 0; i-- {
+			if file[i] == '/' {
+				short = file[i+1:]
+				break
+			}
+		}
+		file = short
+		log.Println(file, strconv.Itoa(line), s)
+	}
+}
+
+func catchError() {
+	if err := recover(); err != nil {
+		log.Println("err", err)
+	}
+}
+
+func Debug(v ...interface{}) {
+	if dailyRolling {
+		fileCheck()
+	}
+	defer catchError()
+	if logObj != nil {
+		logObj.mu.RLock()
+		defer logObj.mu.RUnlock()
+	}
+
+	if logLevel <= DEBUG {
+		if logObj != nil {
+			logObj.lg.Output(2, fmt.Sprintln("debug", v))
+		}
+		console("debug", v)
+	}
+}
+func Info(v ...interface{}) {
+	if dailyRolling {
+		fileCheck()
+	}
+	defer catchError()
+	if logObj != nil {
+		logObj.mu.RLock()
+		defer logObj.mu.RUnlock()
+	}
+	if logLevel <= INFO {
+		if logObj != nil {
+			logObj.lg.Output(2, fmt.Sprintln("info", v))
+		}
+		console("info", v)
+	}
+}
+func Warn(v ...interface{}) {
+	if dailyRolling {
+		fileCheck()
+	}
+	defer catchError()
+	if logObj != nil {
+		logObj.mu.RLock()
+		defer logObj.mu.RUnlock()
+	}
+
+	if logLevel <= WARN {
+		if logObj != nil {
+			logObj.lg.Output(2, fmt.Sprintln("warn", v))
+		}
+		console("warn", v)
+	}
+}
+func Error(v ...interface{}) {
+	if dailyRolling {
+		fileCheck()
+	}
+	defer catchError()
+	if logObj != nil {
+		logObj.mu.RLock()
+		defer logObj.mu.RUnlock()
+	}
+	if logLevel <= ERROR {
+		if logObj != nil {
+			logObj.lg.Output(2, fmt.Sprintln("error", v))
+		}
+		console("error", v)
+	}
+}
+func Fatal(v ...interface{}) {
+	if dailyRolling {
+		fileCheck()
+	}
+	defer catchError()
+	if logObj != nil {
+		logObj.mu.RLock()
+		defer logObj.mu.RUnlock()
+	}
+	if logLevel <= FATAL {
+		if logObj != nil {
+			logObj.lg.Output(2, fmt.Sprintln("fatal", v))
+		}
+		console("fatal", v)
+	}
+}
+
+func (f *_FILE) isMustRename() bool {
+	if dailyRolling {
+		t, _ := time.Parse(DATEFORMAT, time.Now().Format(DATEFORMAT))
+		if t.After(*f._date) {
+			return true
+		}
+	} else {
+		if maxFileCount > 1 {
+			if fileSize(f.dir+"/"+f.filename) >= maxFileSize {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func (f *_FILE) rename() {
+	if dailyRolling {
+		fn := f.dir + "/" + f.filename + "." + f._date.Format(DATEFORMAT)
+		if !isExist(fn) && f.isMustRename() {
+			if f.logfile != nil {
+				f.logfile.Close()
+			}
+			err := os.Rename(f.dir+"/"+f.filename, fn)
+			if err != nil {
+				f.lg.Println("rename err", err.Error())
+			}
+			t, _ := time.Parse(DATEFORMAT, time.Now().Format(DATEFORMAT))
+			f._date = &t
+			f.logfile, _ = os.Create(f.dir + "/" + f.filename)
+			f.lg = log.New(logObj.logfile, "\n", log.Ldate|log.Ltime|log.Lshortfile)
+		}
+	} else {
+		f.coverNextOne()
+	}
+}
+
+func (f *_FILE) nextSuffix() int {
+	return int(f._suffix%int(maxFileCount) + 1)
+}
+
+func (f *_FILE) coverNextOne() {
+	f._suffix = f.nextSuffix()
+	if f.logfile != nil {
+		f.logfile.Close()
+	}
+	if isExist(f.dir + "/" + f.filename + "." + strconv.Itoa(int(f._suffix))) {
+		os.Remove(f.dir + "/" + f.filename + "." + strconv.Itoa(int(f._suffix)))
+	}
+	os.Rename(f.dir+"/"+f.filename, f.dir+"/"+f.filename+"."+strconv.Itoa(int(f._suffix)))
+	f.logfile, _ = os.Create(f.dir + "/" + f.filename)
+	f.lg = log.New(logObj.logfile, "\n", log.Ldate|log.Ltime|log.Lshortfile)
+}
+
+func fileSize(file string) int64 {
+	fmt.Println("fileSize", file)
+	f, e := os.Stat(file)
+	if e != nil {
+		fmt.Println(e.Error())
+		return 0
+	}
+	return f.Size()
+}
+
+func isExist(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil || os.IsExist(err)
+}
+
+func fileMonitor() {
+	timer := time.NewTicker(1 * time.Second)
+	for {
+		select {
+		case <-timer.C:
+			fileCheck()
+		}
+	}
+}
+
+func fileCheck() {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Println(err)
+		}
+	}()
+	if logObj != nil && logObj.isMustRename() {
+		logObj.mu.Lock()
+		defer logObj.mu.Unlock()
+		logObj.rename()
+	}
+}

src/jy/util/util.go

@@ -2,14 +2,10 @@ package util
 
 import (
 	"fmt"
-	"io"
 	. "jy/mongodbutil"
-	"log"
-	"os"
 	qu "qfw/util"
 
 	. "gopkg.in/mgo.v2/bson"
-	"gopkg.in/natefinch/lumberjack.v2"
 )
 
 //敏感词
@@ -30,21 +26,6 @@ var BrandGet *DFA     //品牌
 var IsBrandGoods bool //是否开启品牌抽取
 
 func init() {
-	//输出日志配置,多输出源
-	filelog := &lumberjack.Logger{
-		Filename:   "./out.log",
-		MaxSize:    500, // megabytes
-		MaxBackups: 3,
-		MaxAge:     20,   //days
-		Compress:   true, // disabled by default
-	}
-	writers := []io.Writer{
-		filelog,
-		os.Stdout,
-	}
-	fileAndStdoutWriter := io.MultiWriter(writers...)
-	log.SetOutput(fileAndStdoutWriter)
-
 	syncint = make(chan bool, 1)
 }
 

src/jy/util/util2.go

@@ -256,6 +256,3 @@ func FirstKeyValueInMap(m interface{}) (string, interface{}) {
 	}
 	return "", nil
 }
-func Debug(v ...interface{}) {
-
-}

src/main.go

@@ -16,6 +16,9 @@ import (
 )
 
 func init() {
+	util.SetConsole(false)
+	util.SetLevel(util.DEBUG)
+	util.SetRollingDaily("./", "out.log")
 	qu.ReadConfig(&util.Config)
 	qu.ReadConfig("./res/brandrule.json", &util.BrandRules)
 	qu.ReadConfig("./res/goods.json", &util.GoodsConfig)

src/web/templates/admin/com_memu.html

@@ -4,7 +4,8 @@
       <ul id="menu" class="sidebar-menu" data-widget="tree">
         <li class="header">HEADER</li>
         <!-- Optionally, you can add icons to the links -->
-		<!--<li class="treeview">
+		<!--
+        <li class="treeview">
           	<a href="#"><i class="fa fa-clock-o"></i> <span>任务管理</span>
             <span class="pull-right-container">
                 <i class="fa fa-angle-left pull-right"></i>
@@ -14,8 +15,8 @@
 	            <li><a href="/admin/task"><i class="fa fa-link"></i>抽取任务</a></li>
 				<li><a href="/admin/task/export"><i class="fa fa-link"></i>导出任务</a></li>
 			</ul>
-        </li>-->
-		<!--<li><a href="/admin/version"><i class="fa fa-navicon"></i><span>版本管理</span></a></li>
+        </li>
+		<li><a href="/admin/version"><i class="fa fa-navicon"></i><span>版本管理</span></a></li>
 		<li><a href="/admin/audit/recogfield"><i class="fa fa-navicon"></i><span>质量审核</span></a></li>
 		<li class="treeview">
           	<a href="#"><i class="fa fa-laptop"></i> <span>统计</span>
@@ -39,8 +40,8 @@
 				<li><a href="/admin/rule/pre"><i class="fa fa-circle-o"></i>角色管理</a></li>
 				<li><a href="/admin/rule/pre"><i class="fa fa-circle-o"></i>菜单管理</a></li>
 			</ul>
-
-		</li>-->
+		</li>
+        -->
       </ul>
     </section>
 	<span id="role" class="hidden">{{session "role"}}</span>
@@ -59,35 +60,36 @@ $(function () {
                         str = str + '<li><a href=' + info.secondmenu[sec.toString(10)].href + '><i class="' + info.secondmenu[sec.toString(10)].pic + '"></i>' + info.secondmenu[sec.toString(10)].name + '</a></li>'
                     }
                 }
-                $('#menu').append('<li class="treeview">\n' +
-                        '          \t<a href="#"><i class="'+info.pic+'"></i> <span>'+info.name+'</span>\n' +
-                        '            <span class="pull-right-container">\n' +
-                        '                <i class="fa fa-angle-left pull-right"></i>\n' +
-                        '            </span>\n' +
-                        '          \t</a>\n' +
-                        '          \t<ul class="treeview-menu">\n' + str+
-                        '\t\t\t</ul>\n' +
+                $('#menu').append('<li class="treeview">' +
+                        '          <a href="#"><i class="'+info.pic+'"></i> <span>'+info.name+'</span>' +
+                        '            <span class="pull-right-container">' +
+                        '                <i class="fa fa-angle-left pull-right"></i>' +
+                        '            </span>' +
+                        '          </a>' +
+                        '          <ul class="treeview-menu">' + str+ '</ul>' +
                         '        </li>')
 			}else{
                 $('#menu').append('<li><a href='+info.href+'><i class="'+info.pic+'"></i> <span>'+info.name+'</span></a></li>')
 			}
-
         }
     })
 })
 function menuActive(name){
-	$(".sidebar-menu").tree();
-	$(".sidebar-menu").filter(".menu-open").removeClass("menu-open");
-	$(".sidebar-menu").filter(".active").removeClass("active");
-	var a;
-	$(".sidebar-menu a").each(function(i){
-		if($(this).attr("href").endsWith(name)){
-			a=$(this)
-		}
-	});
-	// a.parent().addClass("active");
-	// a.parent().parent().parent().addClass("active");
-	// a.parent().parent().parent().addClass("menu-open");
+    setTimeout(function(){
+        $(".sidebar-menu").tree();
+    	$(".sidebar-menu").filter(".menu-open").removeClass("menu-open");
+    	$(".sidebar-menu").filter(".active").removeClass("active");
+    	var a;
+    	$(".sidebar-menu a").each(function(i){
+    		if($(this).attr("href").endsWith(name)){
+    			a=$(this)
+    		}
+    	});
+	a.parent().addClass("active");
+	a.parent().parent().parent().addClass("active");
+	a.parent().parent().parent().addClass("menu-open");
+    },100);
 }
+ 
 </script>
 {{ end }}

src/web/templates/admin/menu.html

@@ -93,7 +93,7 @@
 {{template "footer"}}
 
 <script>
-    menuActive("user")
+    menuActive("menu")
     $(function () {
         ttable=$('#dataTable').DataTable({
             "paging"      : true,

src/web/templates/admin/role.html

@@ -49,7 +49,9 @@
         </div>
     </section>
 </div>
-
+<script>
+    menuActive("role")
+</script>
 <!-- /.modal -->
 
 <!-- footer -->

src/web/templates/admin/rolemenu.html

@@ -53,7 +53,6 @@
                             <thead>
                             <tr>
                                 <th>一级菜单</th>
-                                <th>二级菜单</th>
                                 <th>操作</th>
                             </tr>
                             </thead>
@@ -115,6 +114,7 @@
             </div>
             <!-- /.modal-content -->
         </form>
+        <input type="hidden" id="_id">
     </div>
     <!-- /.modal-dialog -->
 </div>
@@ -124,7 +124,7 @@
 {{template "footer"}}
 
 <script>
-    menuActive("user")
+    menuActive("role")
     $(function () {
         ttable=$('#dataTable').DataTable({
             "paging"      : true,
@@ -144,23 +144,15 @@
             "columns": [
                 { "data": "name",render:function(val,a,row){
                         return row.name}},
-                {"data":"_id",render:function (val,a,row) {
-                        if (row.secondmenu) {
-                            role={{.role}}
-                            return '<a class="btn btn-sm btn-success" href="/admin/role/secondmenu?_id=' + row._id + '&role='+role+'">进入二级菜单</a>'
-                        }else{
-                            return "无二级菜单"
-                        }
-                }},
                 {"data":"_id",render:function(val,a,row){
-                        return "<a href='#' onclick='del(\""+val+"\")'><i class='fa fa-fw fa-trash text-red'></i></a>"
+                        return "<a href='#' onclick='edit(\""+val+"\")'><i class='fa fa-fw fa-edit text-yellow'></i></a> &nbsp;"+
+                        "<a href='#' onclick='del(\""+val+"\")'><i class='fa fa-fw fa-trash text-red'></i></a>"
                     }}
             ]
         });
         //ttable.on('init.dt', function () {});
     })
     function menu() {
-        console.log($("#menu2").val())
         $("#select3").empty();
         $("#select4").empty();
         $.post("/admin/role/select",{"_id":$("#menu2").val()},function (data,status) {
@@ -172,20 +164,25 @@
         })
     }
     function save(){
-        menu=$("#menu2").val()
+        menuid=$("#menu2").val()
         var clearArr = [];
+        var clearArr2 = [];
         $("#select4 option").each(function(i,val){
             clearArr[i] = this.value
         })
+        $("#select3 option").each(function(i,val){
+            clearArr2[i] = this.value
+        })
         var secondmenuStr = JSON.stringify(clearArr)
-        if(menu == ""){
+        var secondmenuStr2 = JSON.stringify(clearArr2)
+        if(menuid == ""){
             alert("表单填写不完整！");
             return false;
         }
         $.ajax({
             url:"/admin/role/menu/save",
             type:"post",
-            data:{"role":{{.role}},"menu":menu,"secondmenuStr":secondmenuStr},
+            data:{"role":{{.role}},"menu":menuid,"secondmenuStr":secondmenuStr,"secondmenuStr2":secondmenuStr2},
             success:function(r){
                 if(r.rep){
                     $("#userform")[0].reset();
@@ -213,14 +210,40 @@
             })
         });
     }
+    function edit(_id){
+        $("#menu2").empty()
+        $("#select3").empty();
+        $("#select4").empty();
+        $("#_id").val(_id)
+        mark=$("#_id").val()
+        $.ajax({
+            url:"/admin/role/select",
+            type:"post",
+            data:{"_id":_id,"mark":mark,"role":{{.role}}},
+            success:function(r){
+                if(r){
+                    $("#menu2").append("<option value="+_id+">"+r.name+"</option>")
+                    $("#menu2").attr("disabled",true);
+                    for(var a=0;a<r.data2.length;a++){
+                        $("#select3").append("<option title='"+r.data2[a].name+"' value='"+r.data2[a]._id+"'>"+r.data2[a].name+"</option>");
+                    }
+                    for(var a=0;a<r.data.length;a++){
+                        $("#select4").append("<option title='"+r.data[a].name+"' value='"+r.data[a]._id+"'>"+r.data[a].name+"</option>");
+                    }
+                }
+            }
+        })
+        $("#modal-info").modal("show");
+    }
     function formReset(){
+        $("#_id").val("")
         $("#menu2").empty()
         $("#select3").empty();
         $("#select4").empty();
         $("#menu2").append("<option value=''>--请选择--</option>")
+        $("#menu2").attr("disabled",false);
         $.post("/admin/menu/data",'',function (data,status) {
             for(var a=0;a<data.data.length;a++) {
-                console.log(a)
                 $("#menu2").append("<option value="+data.data[a]._id+">"+data.data[a].name+"</option>")
             }
         })

src/web/templates/admin/rolesecondmenu.html

@@ -42,7 +42,7 @@
 {{template "footer"}}
 
 <script>
-    menuActive("user")
+    menuActive("role")
     $(function () {
         ttable=$('#dataTable').DataTable({
             "paging"      : true,

src/web/templates/admin/user.html

@@ -71,7 +71,7 @@
 				<div class="form-group">
 				    <label for="modify" class="col-sm-2 control-label">角色:</label>
 				    <div class="col-sm-10">
-				     	<select id="role" name="role" class="form-control">
+				     	<select id="role2" name="role" class="form-control">
 							<option value="2">开发员</option>
 							<option value="1">审核员</option>
 					  		<option value="0">管理员</option>
@@ -143,7 +143,7 @@ function save(){
 	email=$("#email").val()
 	pwd=$("#pwd").val()
 	na=$("#name").val()
-	role=$("#role").val()
+	role=$("#role2").val()
 	if(email==""||na==""||role==""||pwd==""){
 		alert("表单填写不完整！")
 		return false;
@@ -151,7 +151,7 @@ function save(){
 	$.ajax({
 		url:"/admin/user/save",
 		type:"post",
-		data:{"email":email,"pwd":pwd,"name":name,"role":role},
+		data:{"email":email,"pwd":pwd,"name":na,"role":role},
 		success:function(r){
 			if(r.rep){
 				$("#userform")[0].reset();

src/web/templates/admin/version.html

@@ -93,7 +93,7 @@
 						</div>
 					</div>
 					<div class="form-group">
-						<label for="code" id="fieldname" class="col-sm-2 control-label">是否附件:</label>
+						<label for="code" id="fieldname" class="col-sm-2 control-label">抽取附件:</label>
 						<div class="col-sm-10">
 							<select class="form-control" id="isfiles">
 								<option value=false>否</option>
@@ -102,7 +102,7 @@
 						</div>
 					</div>
 					<div class="form-group">
-						<label for="code" class="col-sm-2 control-label">导出属性:</label>
+						<label for="code" class="col-sm-2 control-label">附件属性:</label>
 						<div class="col-sm-10" id="selectclear">
 							<div class="doublebox">
 								<select multiple="multiple" id="select1" style="overflow-x: scroll;"></select>

udpcreateindex/src/biddingall.go

@@ -0,0 +1,276 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	qutil "qfw/util"
+	elastic "qfw/util/elastic"
+	"strings"
+	"sync"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+//对字段处理 bidamount  budget
+//招标数据表和抽取表一一对应开始更新
+var mpool = make(chan bool, 10)
+
+func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
+	defer qutil.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  qutil.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": qutil.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	//bidding库
+	session := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(session)
+	//extract库
+	extractsession := extractmgo.GetMgoConn()
+	defer extractmgo.DestoryMongoConn(extractsession)
+	//连接信息
+	c, _ := mapInfo["coll"].(string)
+	if c == "" {
+		c, _ = bidding["collect"].(string)
+	}
+
+	extractc, _ := bidding["extractcollect"].(string)
+	db, _ := bidding["db"].(string)
+	extractdb, _ := bidding["extractdb"].(string)
+	index, _ := bidding["index"].(string)
+	itype, _ := bidding["type"].(string)
+	count, _ := session.DB(db).C(c).Find(&q).Count()
+	fields := strings.Split(bidding["fields"].(string), ",")
+	//线程池
+	UpdatesLock := sync.Mutex{}
+
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	//查询招标数据
+	query := session.DB(db).C(c).Find(q).Select(bson.M{
+		"projectinfo.attachment": 0,
+		"contenthtml":            0,
+	}).Sort("_id").Iter()
+	//查询抽取结果
+	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
+
+	n := 0
+
+	//更新数组
+	arr := [][]map[string]interface{}{}
+	arrEs := []map[string]interface{}{}
+	//对比两张表数据，减少查询次数
+	var compare bson.M
+	bnil := false
+	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		update := map[string]interface{}{}
+		//对比方法----------------
+		for {
+			if compare == nil {
+				compare = make(bson.M)
+				if !extractquery.Next(compare) {
+					break
+				}
+			}
+			if compare != nil {
+				//对比
+				cid := qutil.BsonIdToSId(compare["_id"])
+				tid := qutil.BsonIdToSId(tmp["_id"])
+				if cid == tid {
+					bnil = false
+					//更新bidding表，生成索引
+					for _, k := range fields {
+						v1 := compare[k]
+						v2 := tmp[k]
+						if v2 == nil && v1 != nil {
+							update[k] = v1
+						} else if v2 != nil && v1 != nil {
+							//update[k+"_b"] = v2
+							update[k] = v1
+						} else if v2 != nil && v1 == nil {
+							//update[k+"_b"] = v2
+						}
+					}
+					if qutil.IntAll(compare["repeat"]) == 1 {
+						update["extracttype"] = -1
+					} else if qutil.IntAll(tmp["extracttype"]) == -1 {
+						update["extracttype"] = 1
+					}
+					break
+				} else {
+					if cid < tid {
+						bnil = false
+						compare = nil
+						continue
+					} else {
+						bnil = true
+						break
+					}
+				}
+			} else {
+				bnil = false
+				break
+			}
+		}
+		//下面可以多线程跑的--->
+		//处理分类
+		mpool <- true
+		go func(tmp, update, compare map[string]interface{}, bnil bool) {
+			defer func() {
+				<-mpool
+			}()
+			if !bnil && compare != nil {
+				subscopeclass, _ := compare["subscopeclass"].([]interface{})
+				if subscopeclass != nil {
+					//str := ","
+					m1 := map[string]bool{}
+					newclass := []string{}
+					for _, sc := range subscopeclass {
+						sclass, _ := sc.(string)
+						if !m1[sclass] {
+							m1[sclass] = true
+							//str += sclass + ","
+							newclass = append(newclass, sclass)
+						}
+					}
+					update["s_subscopeclass"] = strings.Join(newclass, ",")
+					update["subscopeclass"] = newclass
+				}
+				//处理中标企业
+				winner, _ := compare["winner"].(string)
+				m1 := map[string]bool{}
+				if winner != "" {
+					m1[winner] = true
+				}
+				package1 := compare["package"]
+				if package1 != nil {
+					packageM, _ := package1.(map[string]interface{})
+					for _, p := range packageM {
+						pm, _ := p.(map[string]interface{})
+						pw, _ := pm["winner"].(string)
+						if pw != "" {
+							m1[pw] = true
+						}
+					}
+				}
+				compare = nil
+				if len(m1) > 0 {
+					//str := ","
+					winnerarr := []string{}
+					for k, _ := range m1 {
+						//str += k + ","
+						winnerarr = append(winnerarr, k)
+					}
+					update["s_winner"] = strings.Join(winnerarr, ",")
+				}
+			}
+			//------------------对比结束
+
+			//处理key descript
+			//		if bkey == "" {
+			//			DealInfo(&tmp, &update)
+			//		}
+			//同时保存到elastic
+			for tk, tv := range update {
+				tmp[tk] = tv
+			}
+			//对projectscope字段的索引处理
+			ps, _ := tmp["projectscope"].(string)
+			if ps == "" {
+				tmp["projectscope"] = "" //= tmp["detail"]
+			}
+			if len(ps) > ESLEN {
+				tmp["projectscope"] = string(([]rune(ps))[:4000])
+			}
+			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
+				tmp["budget"] = nil
+			} else if sbd, ok := tmp["budget"].(string); ok {
+				tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+			}
+			if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
+				tmp["bidamount"] = nil
+			} else if sbd, ok := tmp["bidamount"].(string); ok {
+				tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+			}
+			//		for k1, _ := range tmp {
+			//			if strings.HasSuffix(k1, "_b") || k1 == "contenthtml" {
+			//				delete(tmp, k1)
+			//			}
+			//		}
+			//go IS.Add("bidding")
+			UpdatesLock.Lock()
+			if qutil.IntAll(update["extracttype"]) != -1 {
+				newTmp := map[string]interface{}{}
+				for _, v := range biddingIndexFields {
+					if tmp[v] != nil {
+						if "projectinfo" == v {
+							mp, _ := tmp[v].(map[string]interface{})
+							if mp != nil {
+								newmap := map[string]interface{}{}
+								for _, v1 := range projectinfoFields {
+									if mp[v1] != nil {
+										newmap[v1] = mp[v1]
+									}
+								}
+								newTmp[v] = newmap
+							}
+						} else {
+							if v == "detail" {
+								detail, _ := tmp[v].(string)
+								newTmp[v] = FilterDetail(detail)
+							} else {
+								newTmp[v] = tmp[v]
+							}
+						}
+					} else if v == "budget" || v == "bidamount" {
+						newTmp[v] = nil
+					}
+				}
+				arrEs = append(arrEs, newTmp)
+			}
+			if len(update) > 0 {
+				arr = append(arr, []map[string]interface{}{
+					map[string]interface{}{
+						"_id": tmp["_id"],
+					},
+					map[string]interface{}{
+						"$set": update,
+					},
+				})
+			}
+			if len(arr) >= BulkSize-1 {
+				mgo.UpdateBulkAll(db, c, arr...)
+				arr = [][]map[string]interface{}{}
+			}
+			if len(arrEs) >= BulkSize-1 {
+				tmps := arrEs
+				elastic.BulkSave(index, itype, &tmps, true)
+				if len(multiIndex) == 2 {
+					elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
+				}
+				arrEs = []map[string]interface{}{}
+			}
+			UpdatesLock.Unlock()
+		}(tmp, update, compare, bnil)
+		if n%100 == 0 {
+			log.Println("current:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	UpdatesLock.Lock()
+	if len(arr) > 0 {
+		mgo.UpdateBulkAll(db, c, arr...)
+	}
+	if len(arrEs) > 0 {
+		tmps := arrEs
+		elastic.BulkSave(index, itype, &tmps, true)
+		if len(multiIndex) == 2 {
+			elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
+		}
+	}
+	UpdatesLock.Unlock()
+	log.Println(mapInfo, "create bidding index...over", n)
+}

udpcreateindex/src/biddingdata.go

@@ -0,0 +1,295 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	qutil "qfw/util"
+	elastic "qfw/util/elastic"
+	"strings"
+	"sync"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+//对字段处理 bidamount  budget
+var indexfield = []string{
+	"_id",
+	"s_winner",
+	"winner",
+	"buyerclass",
+	"title",
+	"detail",
+	"area",
+	"site",
+	"bidopendate",
+	"bidopentime",
+	"buyer",
+	"city",
+	"comeintime",
+	"href",
+	"infoformat",
+	"projectcode",
+	"projectname",
+	"publishtime",
+	"s_sha",
+	"spidercode",
+	"subtype",
+	"toptype",
+	"agency",
+	"budget",
+	"bidamount",
+	"s_subscopeclass",
+	"projectscope",
+	"bidstatus",
+	"projectinfo",
+	"buyertel",
+	"buyerperson",
+	"projectid",
+	"buyerclass",
+	"district",
+	"topscopeclass",
+}
+
+//招标数据表和抽取表一一对应开始更新
+func biddingDataTask(data []byte, mapInfo map[string]interface{}) {
+	defer qutil.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  qutil.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": qutil.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	//bidding库
+	session := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(session)
+	//extract库
+	extractsession := extractmgo.GetMgoConn()
+	defer extractmgo.DestoryMongoConn(extractsession)
+	//连接信息
+	c, _ := mapInfo["coll"].(string)
+	if c == "" {
+		c, _ = bidding["collect"].(string)
+	}
+
+	extractc, _ := bidding["extractcollect"].(string)
+	db, _ := bidding["db"].(string)
+	extractdb, _ := bidding["extractdb"].(string)
+	index, _ := bidding["index"].(string)
+	itype, _ := bidding["type"].(string)
+	count, _ := session.DB(db).C(c).Find(&q).Count()
+	fields := strings.Split(bidding["fields"].(string), ",")
+	//线程池
+	UpdatesLock := sync.Mutex{}
+
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	//查询招标数据
+	query := session.DB(db).C(c).Find(q).Select(bson.M{
+		"projectinfo.attachment": 0,
+		"contenthtml":            0,
+	}).Sort("_id").Iter()
+	//查询抽取结果
+	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
+
+	n := 0
+
+	//更新数组
+
+	arrEs := []map[string]interface{}{}
+	//对比两张表数据，减少查询次数
+	var compare bson.M
+	bnil := false
+	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		update := map[string]interface{}{}
+		//对比方法----------------
+		for {
+			if compare == nil {
+				compare = make(bson.M)
+				if !extractquery.Next(compare) {
+					break
+				}
+			}
+			if compare != nil {
+				//对比
+				cid := qutil.BsonIdToSId(compare["_id"])
+				tid := qutil.BsonIdToSId(tmp["_id"])
+				if cid == tid {
+					bnil = false
+					//更新bidding表，生成索引
+					for _, k := range fields {
+						v1 := compare[k]
+						v2 := tmp[k]
+						if v2 == nil && v1 != nil {
+							update[k] = v1
+						} else if v2 != nil && v1 != nil {
+							//update[k+"_b"] = v2
+							update[k] = v1
+						} else if v2 != nil && v1 == nil {
+							update[k] = v2
+						}
+					}
+					if qutil.IntAll(compare["repeat"]) == 1 {
+						update["extracttype"] = -1
+					} else if qutil.IntAll(tmp["extracttype"]) == -1 {
+						update["extracttype"] = 1
+					}
+					break
+				} else {
+					if cid < tid {
+						bnil = false
+						compare = nil
+						continue
+					} else {
+						bnil = true
+						break
+					}
+				}
+			} else {
+				bnil = false
+				break
+			}
+		}
+		//下面可以多线程跑的--->
+		//处理分类
+		mpool <- true
+		go func(tmp, update, compare map[string]interface{}, bnil bool) {
+			defer func() {
+				<-mpool
+			}()
+			if !bnil && compare != nil {
+				subscopeclass, _ := compare["subscopeclass"].([]interface{})
+				if subscopeclass != nil {
+					//str := ","
+					m1 := map[string]bool{}
+					newclass := []string{}
+					for _, sc := range subscopeclass {
+						sclass, _ := sc.(string)
+						if !m1[sclass] {
+							m1[sclass] = true
+							//str += sclass + ","
+							newclass = append(newclass, sclass)
+						}
+					}
+					update["s_subscopeclass"] = strings.Join(newclass, ",")
+					update["subscopeclass"] = newclass
+				}
+				//处理中标企业
+				winner, _ := compare["winner"].(string)
+				m1 := map[string]bool{}
+				if winner != "" {
+					m1[winner] = true
+				}
+				package1 := compare["package"]
+				if package1 != nil {
+					packageM, _ := package1.(map[string]interface{})
+					for _, p := range packageM {
+						pm, _ := p.(map[string]interface{})
+						pw, _ := pm["winner"].(string)
+						if pw != "" {
+							m1[pw] = true
+						}
+					}
+				}
+				compare = nil
+				if len(m1) > 0 {
+					//str := ","
+					winnerarr := []string{}
+					for k, _ := range m1 {
+						//str += k + ","
+						winnerarr = append(winnerarr, k)
+					}
+					update["s_winner"] = strings.Join(winnerarr, ",")
+				}
+			}
+			//------------------对比结束
+
+			//处理key descript
+			//		if bkey == "" {
+			//			DealInfo(&tmp, &update)
+			//		}
+			//同时保存到elastic
+			for tk, tv := range update {
+				tmp[tk] = tv
+			}
+			//对projectscope字段的索引处理
+			ps, _ := tmp["projectscope"].(string)
+			if ps == "" {
+				tmp["projectscope"] = "" //= tmp["detail"]
+			}
+			if len(ps) > ESLEN {
+				tmp["projectscope"] = string(([]rune(ps))[:4000])
+			}
+			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
+				tmp["budget"] = nil
+			} else if sbd, ok := tmp["budget"].(string); ok {
+				tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+			}
+			if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
+				tmp["bidamount"] = nil
+			} else if sbd, ok := tmp["bidamount"].(string); ok {
+				tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+			}
+			//		for k1, _ := range tmp {
+			//			if strings.HasSuffix(k1, "_b") || k1 == "contenthtml" {
+			//				delete(tmp, k1)
+			//			}
+			//		}
+			//go IS.Add("bidding")
+			UpdatesLock.Lock()
+			if qutil.IntAll(update["extracttype"]) != -1 {
+				newTmp := map[string]interface{}{}
+				for _, v := range indexfield {
+					if tmp[v] != nil {
+						if "projectinfo" == v {
+							mp, _ := tmp[v].(map[string]interface{})
+							if mp != nil {
+								newmap := map[string]interface{}{}
+								for _, v1 := range projectinfoFields {
+									if mp[v1] != nil {
+										newmap[v1] = mp[v1]
+									}
+								}
+								newTmp[v] = newmap
+							}
+						} else {
+							if v == "detail" {
+								detail, _ := tmp[v].(string)
+								newTmp[v] = FilterDetail(detail)
+							} else {
+								newTmp[v] = tmp[v]
+							}
+						}
+					} else if v == "budget" || v == "bidamount" {
+						newTmp[v] = nil
+					}
+				}
+				arrEs = append(arrEs, newTmp)
+			}
+			if len(arrEs) >= BulkSize-1 {
+				tmps := arrEs
+				elastic.BulkSave(index, itype, &tmps, true)
+				if len(multiIndex) == 2 {
+					elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
+				}
+				arrEs = []map[string]interface{}{}
+			}
+			UpdatesLock.Unlock()
+		}(tmp, update, compare, bnil)
+		if n%1000 == 0 {
+			log.Println("current:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	UpdatesLock.Lock()
+	if len(arrEs) > 0 {
+		tmps := arrEs
+		elastic.BulkSave(index, itype, &tmps, true)
+		if len(multiIndex) == 2 {
+			elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
+		}
+	}
+	UpdatesLock.Unlock()
+	log.Println(mapInfo, "create bidding index...over", n)
+}

udpcreateindex/src/biddingindex.go

@@ -0,0 +1,395 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	mu "mfw/util"
+	"net"
+	qutil "qfw/util"
+	elastic "qfw/util/elastic"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+var BulkSize = 200
+
+//对字段处理 bidamount  budget
+//招标数据表和抽取表一一对应开始更新
+
+func biddingTask(data []byte, mapInfo map[string]interface{}) {
+	defer qutil.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	bkey, _ := mapInfo["bkey"].(string)
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  qutil.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": qutil.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	//连接信息
+	c, _ := bidding["collect"].(string)
+	extractc, _ := bidding["extractcollect"].(string)
+	db, _ := bidding["db"].(string)
+	extractdb, _ := bidding["extractdb"].(string)
+	index, _ := bidding["index"].(string)
+	itype, _ := bidding["type"].(string)
+
+	//extract库
+	extractsession := extractmgo.GetMgoConn(86400)
+	defer extractmgo.DestoryMongoConn(extractsession)
+	extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
+	eMap := map[string]map[string]interface{}{}
+	for tmp := make(map[string]interface{}); extractquery.Next(tmp); {
+		tid := qutil.BsonIdToSId(tmp["_id"])
+		eMap[tid] = tmp
+		tmp = make(map[string]interface{})
+	}
+
+	//bidding库
+	session := mgo.GetMgoConn(86400)
+	count, _ := session.DB(db).C(c).Find(&q).Count()
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	n1, n2 := 0, 0
+	if count < 200000 {
+		res := make([]map[string]interface{}, 1)
+		session.DB(db).C(c).Find(q).Select(bson.M{
+			"projectinfo.attachment": 0,
+			"contenthtml":            0,
+		}).All(&res)
+		mgo.DestoryMongoConn(session)
+		if len(res) != count {
+			log.Println("查询结果不一致", "count:", count, "res:", len(res))
+			time.Sleep(20 * time.Second)
+			toadd := &net.UDPAddr{
+				IP:   net.ParseIP("127.0.0.1"),
+				Port: qutil.IntAll(Sysconfig["udpport"]),
+			}
+			udpclient.WriteUdp(data, mu.OP_TYPE_DATA, toadd)
+		} else {
+			n1, n2 = doIndex(res, eMap, index, itype, db, c, bkey)
+			if (n1 + n2) != count {
+				log.Println("任务错误，结果不一致")
+			}
+		}
+	} else {
+		log.Println("数据量太大，放弃！", count)
+		mgo.DestoryMongoConn(session)
+	}
+	log.Println(mapInfo, "create bidding index...over", "all:", count, "n1:", n1, "n2:", n2)
+}
+
+func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, index, itype, db, c, bkey string) (int, int) {
+	n1, n2 := 0, 0
+	//线程池
+	UpdatesLock := sync.Mutex{}
+	fields := strings.Split(bidding["fields"].(string), ",")
+	//更新数组
+	arr := [][]map[string]interface{}{}
+	arrEs := []map[string]interface{}{}
+	//对比两张表数据，减少查询次数
+	var compare bson.M
+	log.Println("开始迭代..")
+	for n, tmp := range infos {
+		n1++
+		update := map[string]interface{}{}
+		//对比方法----------------
+		tid := qutil.BsonIdToSId(tmp["_id"])
+		if eMap[tid] != nil {
+			compare = eMap[tid]
+			delete(eMap, tid)
+			//更新bidding表，生成索引
+			for _, k := range fields {
+				v1 := compare[k]
+				v2 := tmp[k]
+				if v2 == nil && v1 != nil {
+					update[k] = v1
+				} else if v2 != nil && v1 != nil {
+					//update[k+"_b"] = v2
+					update[k] = v1
+				} else if v2 != nil && v1 == nil {
+					//update[k+"_b"] = v2
+				}
+			}
+			if qutil.IntAll(compare["repeat"]) == 1 {
+				update["extracttype"] = -1
+			} else {
+				update["extracttype"] = 1
+			}
+		} else {
+			compare = nil
+		}
+		//下面可以多线程跑的--->
+		//处理分类
+		if compare != nil {
+			subscopeclass, _ := compare["subscopeclass"].([]interface{})
+			if subscopeclass != nil {
+				//str := ","
+				m1 := map[string]bool{}
+				newclass := []string{}
+				for _, sc := range subscopeclass {
+					sclass, _ := sc.(string)
+					if !m1[sclass] {
+						m1[sclass] = true
+						//str += sclass + ","
+						newclass = append(newclass, sclass)
+					}
+				}
+				update["s_subscopeclass"] = strings.Join(newclass, ",")
+				update["subscopeclass"] = newclass
+			}
+			//处理中标企业
+			winner, _ := compare["winner"].(string)
+			m1 := map[string]bool{}
+			if winner != "" {
+				m1[winner] = true
+			}
+			package1 := compare["package"]
+			if package1 != nil {
+				packageM, _ := package1.(map[string]interface{})
+				for _, p := range packageM {
+					pm, _ := p.(map[string]interface{})
+					pw, _ := pm["winner"].(string)
+					if pw != "" {
+						m1[pw] = true
+					}
+				}
+			}
+			compare = nil
+			if len(m1) > 0 {
+				//str := ","
+				winnerarr := []string{}
+				for k, _ := range m1 {
+					//str += k + ","
+					winnerarr = append(winnerarr, k)
+				}
+				update["s_winner"] = strings.Join(winnerarr, ",")
+			}
+		}
+		//------------------对比结束
+
+		//处理key descript
+		if bkey == "" {
+			DealInfo(&tmp, &update)
+		}
+		//同时保存到elastic
+		for tk, tv := range update {
+			tmp[tk] = tv
+		}
+		//对projectscope字段的索引处理
+		ps, _ := tmp["projectscope"].(string)
+		if ps == "" {
+			tmp["projectscope"] = "" //= tmp["detail"]
+		}
+		if len(ps) > ESLEN {
+			tmp["projectscope"] = string(([]rune(ps))[:4000])
+		}
+		if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
+			tmp["budget"] = nil
+		} else if sbd, ok := tmp["budget"].(string); ok {
+			tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+		}
+		if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
+			tmp["bidamount"] = nil
+		} else if sbd, ok := tmp["bidamount"].(string); ok {
+			tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+		}
+		UpdatesLock.Lock()
+		//		for k1, _ := range tmp {
+		//			if strings.HasSuffix(k1, "_b") || k1 == "contenthtml" {
+		//				delete(tmp, k1)
+		//			}
+		//		}
+		go IS.Add("bidding")
+		if qutil.IntAll(update["extracttype"]) != -1 {
+			newTmp := map[string]interface{}{}
+			for _, v := range biddingIndexFields {
+				//			if tmp[v] != nil {
+				//				newTmp[v] = tmp[v]
+				//			}
+				if tmp[v] != nil {
+					if "projectinfo" == v {
+						mp, _ := tmp[v].(map[string]interface{})
+						if mp != nil {
+							newmap := map[string]interface{}{}
+							for _, v1 := range projectinfoFields {
+								if mp[v1] != nil {
+									newmap[v1] = mp[v1]
+								}
+							}
+							newTmp[v] = newmap
+						}
+					} else {
+						if v == "detail" {
+							detail, _ := tmp[v].(string)
+							newTmp[v] = FilterDetail(detail)
+						} else {
+							newTmp[v] = tmp[v]
+						}
+					}
+				} else if v == "budget" || v == "bidamount" {
+					newTmp[v] = nil
+				}
+			}
+			arrEs = append(arrEs, newTmp)
+		}
+		if len(update) > 0 {
+			arr = append(arr, []map[string]interface{}{
+				map[string]interface{}{
+					"_id": tmp["_id"],
+				},
+				map[string]interface{}{
+					"$set": update,
+				},
+			})
+		}
+		if len(arr) >= BulkSize-1 {
+			mgo.UpdateBulkAll(db, c, arr...)
+			arr = [][]map[string]interface{}{}
+		}
+		if len(arrEs) >= BulkSize-1 {
+			tmps := arrEs
+			elastic.BulkSave(index, itype, &tmps, true)
+			if len(multiIndex) == 2 {
+				elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
+			}
+			arrEs = []map[string]interface{}{}
+		}
+		UpdatesLock.Unlock()
+		if n%100 == 0 {
+			log.Println("current:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	UpdatesLock.Lock()
+	if len(arr) > 0 {
+		mgo.UpdateBulkAll(db, c, arr...)
+	}
+	if len(arrEs) > 0 {
+		tmps := arrEs
+		elastic.BulkSave(index, itype, &tmps, true)
+		if len(multiIndex) == 2 {
+			elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
+		}
+	}
+	UpdatesLock.Unlock()
+	return n1, n2
+}
+
+var client *mu.Client
+var reg = regexp.MustCompile("^[0-9a-zA-Z-.]+$")
+var reg_space = regexp.MustCompile("(?ism)(<style.*?>.*?</style>)|([.#]?\\w{1,20}\\{.*?\\})|(<.*?>)|(\\\\t)+|\\t|( +)|(　+)|(" + string(rune(160)) + "+)")
+var reg_row = regexp.MustCompile("(?i)<(tr|div|p)[^>]*?>|(\\n)+")
+var reg_dh = regexp.MustCompile("[,]+")
+var reg_newdb = regexp.MustCompile("([:,、：，。.;])[,]")
+var reg_no = regexp.MustCompile("^[0-9]*$")
+var MSG_SERVER = "123.56.236.148:7070"
+var DesLen = 120
+
+func inits() {
+	ser := qutil.ObjToString(Sysconfig["msg_server"])
+	if ser != "" {
+		MSG_SERVER = ser
+	}
+	cf := &mu.ClientConfig{
+		ClientName:      "剑鱼抽关键词",
+		EventHandler:    func(p *mu.Packet) {},
+		MsgServerAddr:   MSG_SERVER,
+		CanHandleEvents: []int{},
+		OnConnectSuccess: func() {
+			log.Println("c.")
+		},
+		ReadBufferSize:  10,
+		WriteBufferSize: 10,
+	}
+	client, _ = mu.NewClient(cf)
+
+}
+
+//var clientlock = &sync.Mutex{}
+var keypool = make(chan bool, 1)
+
+func DealInfo(obj, update *map[string]interface{}) {
+	defer qutil.Catch()
+	if (*obj)["keywords"] != nil && (*obj)["description"] != nil {
+		return
+	} else {
+		(*update)["keywords"] = ""
+		(*update)["description"] = ""
+	}
+	title := qutil.ObjToString((*obj)["title"])
+	var m [][]string
+	select {
+	case <-func() <-chan bool {
+		select {
+		case keypool <- true:
+			defer func() {
+				<-keypool
+			}()
+			ret, _ := client.Call("", mu.UUID(8), 4010, mu.SENDTO_TYPE_RAND_RECIVER, title, 1)
+			json.Unmarshal(ret, &m)
+		case <-time.After(5 * time.Millisecond):
+		}
+		ch := make(chan bool, 1)
+		ch <- true
+		return ch
+	}():
+	case <-time.After(20 * time.Millisecond):
+	}
+	arr := []string{}
+	keyword := []string{}
+	keywordnew := []string{}
+	for _, tmp := range m {
+		if reg.MatchString(tmp[0]) {
+			arr = append(arr, tmp[0])
+		} else {
+			if len(arr) > 0 {
+				str := strings.Join(arr, "")
+				keyword = append(keyword, str)
+				arr = []string{}
+			}
+			if len(tmp[0]) > 3 && (strings.HasPrefix(tmp[1], "n") || tmp[1] == "v" || tmp[1] == "vn" || strings.HasPrefix(tmp[1], "g")) {
+				keyword = append(keyword, tmp[0])
+			}
+		}
+	}
+	for _, v := range keyword {
+		v = reg_no.ReplaceAllString(v, "")
+		if len(v) > 0 {
+			keywordnew = append(keywordnew, v)
+		}
+	}
+	keywords := strings.Join(keywordnew, ",")
+	(*update)["keywords"] = keywords
+	content := ""
+	if (*obj)["detail_bak"] != nil {
+		content = qutil.ObjToString((*obj)["detail_bak"])
+	} else {
+		content = qutil.ObjToString((*obj)["detail"])
+	}
+	//内容替换
+	content = strings.Replace(content, " ", "", -1)
+	content = reg_space.ReplaceAllString(content, "")
+	content = reg_row.ReplaceAllString(content, ",")
+	content = reg_dh.ReplaceAllString(content, ",")
+	content = reg_newdb.ReplaceAllString(content, "$1")
+	if strings.HasPrefix(content, ",") {
+		content = content[1:]
+	}
+	//log.Println(content)
+	tc := []rune(content)
+	ltc := len(tc)
+	description := content
+	if ltc > DesLen {
+		description = string(tc[:DesLen])
+	}
+	(*update)["description"] = description
+	//保存到数据库
+	return
+}

udpcreateindex/src/biddingindexback.go

@@ -0,0 +1,175 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	qutil "qfw/util"
+	elastic "qfw/util/elastic"
+	//elastic "qfw/util/elastic_v5"
+	"regexp"
+	//	"strings"
+	"sync"
+	"time"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+var (
+	BulkSizeBack = 400
+	ESLEN        = 32766
+)
+
+func biddingBackTask(data []byte, mapInfo map[string]interface{}) {
+	defer qutil.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  qutil.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": qutil.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	//bidding库
+	session := mgo.GetMgoConn(86400)
+	defer mgo.DestoryMongoConn(session)
+	//连接信息
+	c, _ := mapInfo["coll"].(string)
+	if c == "" {
+		//		c, _ = bidding["collect"].(string)
+		c, _ = biddingback["collect"].(string)
+	}
+	db, _ := biddingback["db"].(string)
+	index, _ := biddingback["index"].(string)
+	itype, _ := biddingback["type"].(string)
+	count, _ := session.DB(db).C(c).Find(&q).Count()
+	//线程池
+	UpdatesLock := sync.Mutex{}
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	//查询招标数据
+	//	query := session.DB(db).C(c).Find(q).Select(bson.M{
+	//		//"projectinfo.attachment": 0,
+	//		"contenthtml": 0,
+	//	}).Sort("_id").Iter()
+	query := session.DB(db).C(c).Find(q).Select(bson.M{
+		"contenthtml": 0,
+		"s_sha":       0,
+	}).Sort("_id").Iter()
+	//查询抽取结果
+	n := 0
+	//更新数组
+	arrEs := []map[string]interface{}{}
+	//对比两张表数据，减少查询次数
+	thread := qutil.IntAll(mapInfo["thread"])
+	//不传为0只生成招标索引,1生成招标+预览,2只生成预览
+	_multiIndex := qutil.IntAll(mapInfo["multiIndex"])
+	if thread < 1 {
+		thread = 3
+	}
+	log.Println("es线程数:", thread)
+	espool := make(chan bool, thread)
+	now1 := time.Now().Unix()
+	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		if qutil.IntAll(tmp["extracttype"]) == -1 {
+			tmp = make(map[string]interface{})
+			continue
+		}
+		ct := qutil.Int64All(tmp["comeintime"])
+		pt := qutil.Int64All(tmp["publishtime"])
+		if pt > ct+86400 || pt > now1 { //时间问题,需要更新
+			if ct > now1 {
+				ct = now1
+			}
+			tmp["publishtime"] = ct
+		}
+
+		ps, _ := tmp["projectscope"].(string)
+		if ps == "" {
+			tmp["projectscope"] = "" //= tmp["detail"]
+		}
+		if len(ps) > ESLEN {
+			tmp["projectscope"] = string(([]rune(ps))[:4000])
+		}
+		if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
+			tmp["budget"] = nil
+		} else if sbd, ok := tmp["budget"].(string); ok {
+			tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+		}
+		if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
+			tmp["bidamount"] = nil
+		} else if sbd, ok := tmp["bidamount"].(string); ok {
+			tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+		}
+		UpdatesLock.Lock()
+		newTmp := map[string]interface{}{}
+		for _, v := range biddingIndexFields {
+			if tmp[v] != nil {
+				if "projectinfo" == v {
+					mp, _ := tmp[v].(map[string]interface{})
+					if mp != nil {
+						newmap := map[string]interface{}{}
+						for _, v1 := range projectinfoFields {
+							if mp[v1] != nil {
+								newmap[v1] = mp[v1]
+							}
+						}
+						newTmp[v] = newmap
+					}
+				} else {
+					if v == "detail" {
+						detail, _ := tmp[v].(string)
+						newTmp[v] = FilterDetail(detail)
+					} else {
+						newTmp[v] = tmp[v]
+					}
+				}
+			} else if v == "budget" || v == "bidamount" {
+				newTmp[v] = nil
+			}
+		}
+		arrEs = append(arrEs, newTmp)
+		if len(arrEs) >= BulkSizeBack {
+			tmps := arrEs
+			espool <- true
+			go func(tmps []map[string]interface{}) {
+				defer func() {
+					<-espool
+				}()
+				if _multiIndex == 0 {
+					elastic.BulkSave(index, itype, &tmps, true)
+				} else if _multiIndex == 1 && len(multiIndex) == 2 {
+					elastic.BulkSave(index, itype, &tmps, true)
+					elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
+				} else if _multiIndex == 2 && len(multiIndex) == 2 {
+					elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
+				}
+			}(tmps)
+			arrEs = []map[string]interface{}{}
+		}
+		UpdatesLock.Unlock()
+		if n%1000 == 0 {
+			log.Println("current:", n, qutil.BsonIdToSId(tmp["_id"]))
+		}
+		tmp = make(map[string]interface{})
+	}
+	UpdatesLock.Lock()
+	if len(arrEs) > 0 {
+		tmps := arrEs
+		if _multiIndex == 0 {
+			elastic.BulkSave(index, itype, &tmps, true)
+		} else if _multiIndex == 1 && len(multiIndex) == 2 {
+			elastic.BulkSave(index, itype, &tmps, true)
+			elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
+		} else if _multiIndex == 2 && len(multiIndex) == 2 {
+			elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
+		}
+	}
+	UpdatesLock.Unlock()
+	log.Println(mapInfo, "create biddingback index...over", n)
+}
+
+var filterReg = regexp.MustCompile("<[^>]+>")
+
+func FilterDetail(text string) string {
+	return filterReg.ReplaceAllString(text, "")
+}

udpcreateindex/src/buyerindex.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"log"
+	"qfw/util"
+	elastic "qfw/util/elastic"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+func buyerTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	session := extractmgo.GetMgoConn(1800)
+	defer extractmgo.DestoryMongoConn(session)
+	c, _ := buyer["collect"].(string)
+	db, _ := buyer["db"].(string)
+	index, _ := buyer["index"].(string)
+	itype, _ := buyer["type"].(string)
+	count, _ := session.DB(db).C(c).Find(&q).Count()
+	savepool := make(chan bool, 10)
+
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	query := session.DB(db).C(c).Find(q).Select(bson.M{"pici": 0}).Iter()
+
+	arr := make([]map[string]interface{}, savesizei)
+	var n int
+	i := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+		go IS.Add("buyer")
+		arr[i] = tmp
+		n++
+		if i == savesizei-1 {
+			savepool <- true
+			tmps := arr
+			go func(tmpn *[]map[string]interface{}) {
+				defer func() {
+					<-savepool
+				}()
+				elastic.BulkSave(index, itype, tmpn, true)
+			}(&tmps)
+			i = 0
+			arr = make([]map[string]interface{}, savesizei)
+		}
+		if n%savesizei == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	if i > 0 {
+		elastic.BulkSave(index, itype, &arr, true)
+	}
+	log.Println(mapInfo, "create buyer index...over", n)
+}

udpcreateindex/src/config.json

@@ -0,0 +1,91 @@
+{
+    "udpport": ":1483",
+    "msg_server": "123.56.236.148:7070",
+	"savedb": {
+        "addr": "172.17.145.163:27080",
+        "size": 6,
+        "db": "extract_v3"
+    },
+    "jkmail": {
+        "to":"renzheng@topnet.net.cn",
+		"api":"http://10.171.112.160:19281/_send/_mail"
+    },
+    "winner": {
+        "db": "qfw",
+        "collect": "winner",
+        "index": "winner",
+        "type": "winner"
+    },
+    "buyer": {
+        "db": "qfw",
+        "collect": "buyer",
+        "index": "buyer",
+        "type": "buyer"
+    },
+    "biddingback": {
+        "db": "qfw",
+        "collect": "bidding",
+        "index": "bidding",
+        "type": "bidding"
+    },
+    "bidding": {
+        "db": "qfw",
+        "collect": "bidding",
+        "index": "bidding",
+        "type": "bidding",
+        "extractdb": "extract_v3",
+        "extractcollect": "result_v3",
+        "indexfields": [
+            "_id",
+            "s_winner",
+            "winner",
+            "buyerclass",
+            "title",
+            "detail",
+            "area",
+            "site",
+            "bidopendate",
+            "bidopentime",
+            "buyer",
+            "city",
+            "comeintime",
+            "href",
+            "infoformat",
+            "projectcode",
+            "projectname",
+            "publishtime",
+            "s_sha",
+            "spidercode",
+            "subtype",
+            "toptype",
+            "agency",
+            "budget",
+            "bidamount",
+            "s_subscopeclass",
+            "projectscope",
+            "bidstatus",
+            "projectinfo",
+            "buyertel",
+            "buyerperson",
+            "projectid"
+        ],
+        "fields": "buyerclass,projectname,projectcode,bidamount,budget,agency,amount,winner,buyer,bidopendate,bidopentime,bidstatus,projectscope,buyertel,buyerperson,city,area,district,topscopeclass",
+        "projectinfo": "approvecode,approvecontent,approvestatus,approvetime,industry",
+        "multiIndex": ""
+    },
+    "project": {
+        "db": "qfw",
+        "collect": "projectset",
+        "index": "projectset_v1",
+        "type": "projectset"
+    },
+    "mongodb": {
+        "addr": "10.172.242.243:27080",
+        "pool": 10,
+        "db": "qfw"
+    },
+    "elastic": {
+        "addr": "http://39.96.199.144:9800",
+        "pool": 12
+    }
+}

udpcreateindex/src/datamonitor.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"sync"
+	"time"
+)
+
+/**
+增加数据监控服务，每15分钟保存一次抽取情况
+**/
+var IS = &InfoStatus{
+	time.Now().Unix(), 0, map[string]int{}, &sync.Mutex{}}
+
+func init() {
+	go IS.Save()
+}
+
+type InfoStatus struct {
+	Starttime int64
+	Endtime   int64
+	Val       map[string]int
+	Lock      *sync.Mutex
+}
+
+func (is *InfoStatus) Add(t string) {
+	is.Lock.Lock()
+	is.Val[t]++
+	is.Lock.Unlock()
+}
+
+func (is *InfoStatus) Save() {
+	is.Lock.Lock()
+	is.Endtime = time.Now().Unix()
+	save := map[string]interface{}{}
+	all := 0
+	for k, v := range is.Val {
+		all += v
+		save[k] = v
+	}
+	if all > 0 {
+		save["starttime"] = is.Starttime
+		save["endtime"] = is.Endtime
+		save["flag"] = "indexser"
+		is.Val = map[string]int{}
+		go extractmgo.Save("datamonitor", save)
+	}
+	is.Starttime = is.Endtime
+	is.Lock.Unlock()
+	time.AfterFunc(15*time.Minute, is.Save)
+}

udpcreateindex/src/default.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"log"
+	"qfw/util"
+	elastic "qfw/util/elastic"
+
+	"qfw/util/mongodb"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+func defaultFunc(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	tasktype, _ := mapInfo["stype"].(string)
+	if tasktype == "" {
+		return
+	}
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	var Mgo *mongodb.MongodbSim
+	c, _ := mapInfo["c"].(string)
+	db, _ := mapInfo["d"].(string)
+	index, _ := mapInfo["index"].(string)
+	itype, _ := mapInfo["type"].(string)
+	if itype == "" {
+		itype = index
+	}
+	if mapInfo["mgoaddr"] != nil {
+		Mgo = &mongodb.MongodbSim{
+			MongodbAddr: mapInfo["mgoaddr"].(string),
+			Size:        5,
+			DbName:      db,
+		}
+		Mgo.InitPool()
+	} else {
+		Mgo = mgo
+	}
+	session := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(session)
+	count, _ := session.DB(db).C(c).Find(&q).Count()
+	savepool := make(chan bool, 10)
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	query := session.DB(db).C(c).Find(q).Iter()
+
+	arr := make([]map[string]interface{}, savesizei)
+	var n int
+	i := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+		go IS.Add(tasktype)
+		arr[i] = tmp
+		n++
+		if i == savesizei-1 {
+			savepool <- true
+			tmps := arr
+			go func(tmpn *[]map[string]interface{}) {
+				defer func() {
+					<-savepool
+				}()
+				elastic.BulkSave(index, itype, tmpn, true)
+			}(&tmps)
+			i = 0
+			arr = make([]map[string]interface{}, savesizei)
+		}
+		if n%savesizei == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	if i > 0 {
+		elastic.BulkSave(index, itype, &arr, true)
+	}
+	log.Println(mapInfo, "create "+tasktype+" index...over", n)
+}

udpcreateindex/src/main.go

@@ -0,0 +1,179 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	mu "mfw/util"
+	"net"
+	"qfw/util"
+	elastic "qfw/util/elastic"
+	"qfw/util/mongodb"
+	"strings"
+	"time"
+)
+
+var (
+	Sysconfig                                    map[string]interface{} //配置文件
+	mgo                                          *mongodb.MongodbSim    //mongodb操作对象
+	extractmgo                                   *mongodb.MongodbSim    //mongodb操作对象
+	udpclient                                    mu.UdpClient           //udp对象
+	updport                                      string
+	winner, bidding, biddingback, project, buyer map[string]interface{}
+	savesizei                                    = 500
+	biddingIndexFields                           = []string{"_id", "buyerclass", "s_winner", "title", "detail", "detail_bak", "area", "areaval", "site", "type", "amount", "bidopendate", "bidopentime", "buyer", "channel", "city", "comeintime", "contenthtml", "descript", "description", "extracttype", "href", "infoformat", "keywords", "projectcode", "projectname", "publishtime", "s_sha", "spidercode", "subtype", "summary", "toptype", "urltop", "winner", "agency", "budget", "bidamount", "s_subscopeclass", "projectscope", "bidstatus"}
+	projectinfoFields                            []string
+	multiIndex                                   []string
+)
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	inits()
+	go checkMapJob()
+	updport, _ = Sysconfig["updport"].(string)
+	winner, _ = Sysconfig["winner"].(map[string]interface{})
+	buyer, _ = Sysconfig["buyer"].(map[string]interface{})
+	bidding, _ = Sysconfig["bidding"].(map[string]interface{})
+	biddingback, _ = Sysconfig["biddingback"].(map[string]interface{})
+	project, _ = Sysconfig["project"].(map[string]interface{})
+	mconf, _ := Sysconfig["mongodb"].(map[string]interface{})
+	mgo = &mongodb.MongodbSim{
+		MongodbAddr: mconf["addr"].(string),
+		Size:        util.IntAllDef(mconf["pool"], 5),
+		DbName:      mconf["db"].(string),
+	}
+	mgo.InitPool()
+
+	savedb, _ := Sysconfig["savedb"].(map[string]interface{})
+	if savedb == nil {
+		log.Println("未设置保存数据库，默认使用招标库")
+		extractmgo = mgo
+	} else {
+		addr, _ := savedb["addr"].(string)
+		size := util.IntAllDef(savedb["size"], 5)
+		db, _ := savedb["db"].(string)
+		extractmgo = &mongodb.MongodbSim{
+			MongodbAddr: addr,
+			Size:        size,
+			DbName:      db,
+		}
+		extractmgo.InitPool()
+	}
+
+	econf := Sysconfig["elastic"].(map[string]interface{})
+	elastic.InitElasticSize(econf["addr"].(string), util.IntAllDef(econf["pool"], 5))
+	if bidding["indexfields"] != nil {
+		biddingIndexFields = util.ObjArrToStringArr(bidding["indexfields"].([]interface{}))
+	}
+	if bidding["projectinfo"] != nil {
+		pf := util.ObjToString(bidding["projectinfo"])
+		if pf != "" {
+			projectinfoFields = strings.Split(pf, ",")
+		}
+	}
+	if bidding["multiIndex"] != nil {
+		mi := util.ObjToString(bidding["multiIndex"])
+		if mi != "" {
+			multiIndex = strings.Split(mi, ",")
+		}
+	}
+	log.Println(projectinfoFields)
+}
+
+func main() {
+	updport := Sysconfig["udpport"].(string)
+	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Println("Udp服务监听", updport)
+	time.Sleep(99999 * time.Hour)
+}
+
+var pool = make(chan bool, 20)
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA: //上个节点的数据
+		//从表中开始处理生成企业数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Println("err:", err, "mapInfo:", mapInfo, string(data))
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			key, _ := mapInfo["key"].(string)
+			if key == "" {
+				key = "udpok"
+			}
+			go udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
+			tasktype, _ := mapInfo["stype"].(string)
+			log.Println("tasktype:", tasktype)
+			switch tasktype {
+			case "winner":
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					winnerTask(data, mapInfo)
+				}()
+			case "bidding": //实时+udp调用，可选择是否生成关键词， 一次性最大20万
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					biddingTask(data, mapInfo)
+				}()
+			case "project":
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					projectTask(data, mapInfo)
+				}()
+			case "biddingback": //不合并，直接调用mongo库生成索引
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					biddingBackTask(data, mapInfo)
+				}()
+			case "biddingall": //重新合并，但不生成关键词
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					biddingAllTask(data, mapInfo)
+				}()
+			case "biddingdata": //重新合并，但不生成关键词
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					biddingDataTask(data, mapInfo)
+				}()
+			case "buyer":
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					buyerTask(data, mapInfo)
+				}()
+			default:
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					defaultFunc(data, mapInfo)
+				}()
+			}
+		}
+	case mu.OP_NOOP: //下个节点回应
+		log.Println("发送成功", string(data))
+	}
+}

udpcreateindex/src/projectindex.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"qfw/util"
+	elastic "qfw/util/elastic"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+func projectTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	session := extractmgo.GetMgoConn(3600)
+	defer extractmgo.DestoryMongoConn(session)
+	c, _ := project["collect"].(string)
+	db, _ := project["db"].(string)
+	index, _ := project["index"].(string)
+	itype, _ := project["type"].(string)
+	count, _ := session.DB(db).C(c).Find(&q).Count()
+	savepool := make(chan bool, 10)
+
+	log.Println(db, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
+	query := session.DB(db).C(c).Find(q).Iter()
+
+	arr := make([]map[string]interface{}, savesizei)
+	var n int
+	i := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+		delete(tmp, "package")
+		if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
+			tmp["budget"] = nil
+		}
+		if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
+			tmp["bidamount"] = nil
+		}
+		go IS.Add("project")
+		arr[i] = tmp
+		n++
+		if i == savesizei-1 {
+			savepool <- true
+			tmps := arr
+			go func(tmpn *[]map[string]interface{}) {
+				defer func() {
+					<-savepool
+				}()
+				elastic.BulkSave(index, itype, tmpn, true)
+			}(&tmps)
+			i = 0
+			arr = make([]map[string]interface{}, savesizei)
+		}
+		if n%savesizei == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	if i > 0 {
+		elastic.BulkSave(index, itype, &arr, true)
+	}
+	log.Println(mapInfo, "create project index...over", n)
+}

udpcreateindex/src/tonumber.go

@@ -0,0 +1,280 @@
+// tonumber
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var contentUnit *regexp.Regexp          //全文检索单位：万元
+var regOperator *regexp.Regexp          //运算符号
+var regNumFloat *regexp.Regexp          //提取整数或浮点数
+var moneyRegChar *regexp.Regexp         //提取中文数字
+var regStrUnit *regexp.Regexp           //提取单位
+var moneyChar = map[string]interface{}{ //"〇": "0", "零": "0",
+	"一": float64(1), "壹": float64(1), "二": float64(2), "贰": float64(2), "三": float64(3), "叁": float64(3), "四": float64(4), "肆": float64(4), "五": float64(5), "伍": float64(5),
+	"六": float64(6), "陆": float64(6), "七": float64(7), "柒": float64(7), "八": float64(8), "捌": float64(8), "九": float64(9), "玖": float64(9), "十": float64(10), "拾": float64(10),
+	"百": float64(100), "佰": float64(100), "千": float64(1000), "仟": float64(1000), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000),
+	"零": float64(0), "点": ".", "角": float64(0.1), "分": float64(0.01),
+}
+var moneyUnit = map[string]float64{
+	"元": float64(1), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000), //单位
+}
+var spaces = []string{"\u3000", "\u2003", "\u00a0"}
+var cutAllSpace = regexp.MustCompile(`\s*`)
+
+func init() {
+	regOperator, _ = regexp.Compile(`[*|+|)*)]`)
+	regNumFloat, _ = regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
+	regStrUnit, _ = regexp.Compile(`[元|万|亿]`)
+
+	regStrChar := `[〇|零|点|.|．|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
+	moneyRegChar, _ = regexp.Compile(regStrChar)
+
+	contentUnit, _ = regexp.Compile(`(万元|单位/万)`)
+}
+
+//转int
+func ObjToInt(data []interface{}) []interface{} {
+	tmp, err := strconv.Atoi(fmt.Sprint(data[0]))
+	if err != nil {
+		data[0] = 0
+		return data
+	} else {
+		data[0] = tmp
+		return data
+	}
+}
+
+//转float,精度小数点4位
+func ObjToFloat(data []interface{}) []interface{} {
+	tmp, err := strconv.ParseFloat(fmt.Sprint(data[0]), 64)
+	if err != nil {
+		return []interface{}{float64(0), data[1]}
+	} else {
+		tmp, err = strconv.ParseFloat(strconv.FormatFloat(tmp, 'f', 4, 64), 64)
+		if err != nil {
+			return []interface{}{float64(0), data[1]}
+		} else {
+			return []interface{}{tmp, data[1]}
+		}
+	}
+}
+
+//金额转换
+func ObjToMoney(data []interface{}) []interface{} {
+	isfindUnit := true
+	ret := capitalMoney(data)[0]
+	if ret.(float64) < float64(10000) || ret.(float64) > float64(50000000000) {
+		ret2, b := numMoney(data)
+		isfindUnit = b
+		if ret2[0].(float64) > ret.(float64) {
+			ret = ret2[0]
+		}
+	}
+	f, _ := strconv.ParseFloat(strconv.FormatFloat(ret.(float64), 'f', 4, 64), 64)
+	if f < 1 {
+		f = 0
+	}
+	//若果金额小于50，全文检索单位：万
+	if f < 50 && f > 0 && isfindUnit {
+		rep := contentUnit.FindAllStringIndex(fmt.Sprint(data[1]), -1)
+		if len(rep) > 0 {
+			f = f * 10000
+		}
+	}
+	data[0] = f
+	return data
+}
+
+//清理所有空白符
+func CutAllSpace(data []interface{}) []interface{} {
+	tmp := cutAllSpace.ReplaceAllString(fmt.Sprint(data[0]), "")
+	tmp = replaceSymbol(tmp, spaces)
+	data[0] = tmp
+	return data
+}
+
+//数字金额转换
+func numMoney(data []interface{}) ([]interface{}, bool) {
+	tmp := fmt.Sprint(data[0])
+	tmp = replaceSymbol(tmp, []string{",", "，", "（", "）", "(", ")", "：", "\n"})
+	tmp = replaceString(tmp, []string{"万元", "亿元", "．"}, []string{"万", "亿", "."})
+	tmp = fmt.Sprint(CutAllSpace([]interface{}{tmp, data[1]})[0])
+	rets := regNumFloat.FindAllString(tmp, -1)
+	fnums := []float64{}
+	unitstrs := []string{}
+	if len(rets) > 0 {
+		pindex := 0 //单位前置
+		for k, v := range rets {
+			f, err := strconv.ParseFloat(v, 64)
+			if err == nil {
+				fnums = append(fnums, f)
+				index := strings.Index(tmp, v)
+				//单位后置
+				start := index + len(v)
+				end := start + 3
+				//log.Println("vvv", tmp, v, pindex, index, start)
+				if k > 0 {
+					if start >= pindex+3 {
+						pstart := pindex + 3
+						if pstart >= index {
+							pstart = index
+						}
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:])
+						}
+					} else {
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				} else {
+					if len(tmp) > end {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:end])
+						}
+					} else {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				}
+				pindex = start
+			}
+		}
+	}
+	//log.Println("unitstrs", fnums, unitstrs)
+	unit := float64(0)
+	fnum := float64(0)
+	for k, v := range fnums {
+		fnum = v
+		units := regStrUnit.FindAllString(unitstrs[k], -1)
+		for _, v := range units {
+			if moneyUnit[v] != 0 {
+				unit = moneyUnit[v]
+				break
+			}
+		}
+		if unit != float64(0) { //取第一个
+			break
+		}
+	}
+	if unit == float64(0) {
+		data[0] = fnum
+	} else {
+		data[0] = fnum * unit
+	}
+	if unit == 10000 {
+		return data, false
+	} else {
+		return data, true
+	}
+}
+
+//大写数子金额转换
+func capitalMoney(data []interface{}) []interface{} {
+	nodes := []float64{}
+	node := float64(0)
+	tmp := float64(0)
+	decimals := 0.0
+	ishaspoint := false //是否含小数点
+	fnum := float64(0)
+	moneyRegChar.ReplaceAllStringFunc(fmt.Sprint(data[0]), func(key string) string {
+		if key == "元" || key == "圆" || key == "点" {
+			ishaspoint = true
+		}
+		if v, ok := moneyChar[key].(float64); ok {
+			if (ishaspoint && v > 10) || key == "整" || key == "正" { //排除后面有其他的单位
+				return ""
+			}
+			//log.Println(key, v, fnum)
+			if v < 10 && v >= 0 {
+				if ishaspoint { //小数部分
+					if v >= 1 {
+						fnum = v
+					} else if v < 1 && v > 0 {
+						decimals += fnum * v
+					}
+				} else {
+					if tmp != float64(0) {
+						node += tmp
+					}
+					tmp = float64(v)
+				}
+			} else if v == 10000 || v == 100000000 { //单位万、亿
+				if tmp != float64(0) {
+					node += tmp
+					tmp = float64(0)
+				}
+				nodes = append(nodes, node*float64(v))
+				node = float64(0)
+			} else {
+				if v == 10 && tmp == 0 {
+					tmp = 1
+				}
+				tmp = tmp * float64(v)
+				node += tmp
+				tmp = float64(0)
+			}
+		}
+		return ""
+	})
+	nodes = append(nodes, node, tmp)
+	ret := float64(0)
+	for _, v := range nodes {
+		ret = ret + v
+	}
+	return []interface{}{ret + decimals, data[1]}
+}
+
+//过滤符号
+func replaceSymbol(con string, rep []string) string {
+	for _, v := range rep {
+		con = strings.Replace(con, v, "", -1)
+	}
+	return con
+}
+
+//符号替换
+func replaceString(con string, ret, rep []string) string {
+	for k, v := range ret {
+		if len(rep) > k {
+			con = strings.Replace(con, v, rep[k], -1)
+		}
+	}
+	return con
+}
+
+//费率转小数
+func RateToFloat(con []interface{}) []interface{} {
+	tmp := fmt.Sprint(CutAllSpace(con)[0])
+	if strings.Contains(tmp, "%") || strings.Contains(tmp, "％") {
+		tmp = strings.Replace(tmp, "%", "", -1)
+		tmp = strings.Replace(tmp, "％", "", -1)
+		rep := ObjToFloat([]interface{}{tmp, con[1]})[0]
+		con[0] = rep.(float64) / 100
+		return con
+	} else {
+		return ObjToFloat([]interface{}{tmp, con[1]})
+	}
+}
+
+//大于一万亿的过滤掉
+func ClearMaxAmount(data []interface{}) []interface{} {
+	value, _ := data[0].(float64)
+	if value >= 1000000000000 {
+		data[0] = float64(0)
+	}
+	return data
+}

udpcreateindex/src/udptaskmap.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	mu "mfw/util"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+)
+
+var udptaskmap = &sync.Map{}
+var tomail string
+var api string
+
+type udpNode struct {
+	data      []byte
+	addr      *net.UDPAddr
+	timestamp int64
+	retry     int
+}
+
+func checkMapJob() {
+	//阿里云内网无法发送邮件
+	jkmail, _ := Sysconfig["jkmail"].(map[string]interface{})
+	if jkmail != nil {
+		tomail, _ = jkmail["to"].(string)
+		api, _ = jkmail["api"].(string)
+	}
+	log.Println("start checkMapJob", tomail, Sysconfig["jkmail"])
+	for {
+		udptaskmap.Range(func(k, v interface{}) bool {
+			now := time.Now().Unix()
+			node, _ := v.(*udpNode)
+			if now-node.timestamp > 120 {
+				node.retry++
+				if node.retry > 5 {
+					log.Println("udp重试失败", k)
+					udptaskmap.Delete(k)
+					res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, tomail, "extract-send-fail", k.(string)))
+					if err == nil {
+						defer res.Body.Close()
+						read, err := ioutil.ReadAll(res.Body)
+						log.Println("邮件发发送：", string(read), err)
+					}
+				} else {
+					log.Println("udp重发", k)
+					udpclient.WriteUdp(node.data, mu.OP_TYPE_DATA, node.addr)
+				}
+			} else if now-node.timestamp > 10 {
+				log.Println("udp任务超时中..", k)
+			}
+			return true
+		})
+		time.Sleep(60 * time.Second)
+	}
+}

udpcreateindex/src/winnerindex.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"log"
+	"qfw/util"
+	elastic "qfw/util/elastic"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+func winnerTask(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	session := extractmgo.GetMgoConn(1800)
+	defer extractmgo.DestoryMongoConn(session)
+	c, _ := winner["collect"].(string)
+	db, _ := winner["db"].(string)
+	index, _ := winner["index"].(string)
+	itype, _ := winner["type"].(string)
+	count, _ := session.DB(db).C(c).Find(&q).Count()
+	savepool := make(chan bool, 10)
+
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	query := session.DB(db).C(c).Find(q).Select(bson.M{"pici": 0}).Iter()
+
+	arr := make([]map[string]interface{}, savesizei)
+	var n int
+	i := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+		go IS.Add("winner")
+		arr[i] = tmp
+		n++
+		if i == savesizei-1 {
+			savepool <- true
+			tmps := arr
+			go func(tmpn *[]map[string]interface{}) {
+				defer func() {
+					<-savepool
+				}()
+				elastic.BulkSave(index, itype, tmpn, true)
+			}(&tmps)
+			i = 0
+			arr = make([]map[string]interface{}, savesizei)
+		}
+		if n%savesizei == 0 {
+			log.Println("当前:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	if i > 0 {
+		elastic.BulkSave(index, itype, &arr, true)
+	}
+	log.Println(mapInfo, "create winner index...over", n)
+}

udpextractbuyer/src/README.md

@@ -0,0 +1,2 @@
+根据pici字段进行创建全文检索
+要在表上建索引

udpextractbuyer/src/config.json

@@ -0,0 +1,21 @@
+{
+    "udpport": ":1481",
+    "collect": "buyer_20190220",
+    "mongodb": {
+        "addr": "172.17.145.163:27080",
+        "pool": 15,
+        "db": "extract_v3",
+        "collect": "result_20190220"
+    },
+    "redis": {
+        "addr": "buyer=10.172.228.50:2710",
+        "pool": 30
+    },
+    "nextNode": [
+        {
+            "addr": "127.0.0.1",
+            "port": 1483,
+            "memo": "创建采购单位索引"
+        }
+    ]
+}

udpextractbuyer/src/datamonitor.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"sync"
+	"time"
+)
+
+/**
+增加数据监控服务，每15分钟保存一次抽取情况
+**/
+var IS = &InfoStatus{
+	time.Now().Unix(), 0, map[string]int{}, &sync.Mutex{}}
+
+func init() {
+	go IS.Save()
+}
+
+type InfoStatus struct {
+	Starttime int64
+	Endtime   int64
+	Val       map[string]int
+	Lock      *sync.Mutex
+}
+
+func (is *InfoStatus) Add(t string) {
+	is.Lock.Lock()
+	is.Val[t]++
+	is.Lock.Unlock()
+}
+
+func (is *InfoStatus) Save() {
+	is.Lock.Lock()
+	is.Endtime = time.Now().Unix()
+	save := map[string]interface{}{}
+	all := 0
+	for k, v := range is.Val {
+		all += v
+		save[k] = v
+	}
+	if all > 0 {
+		save["receive"] = all
+		save["starttime"] = is.Starttime
+		save["endtime"] = is.Endtime
+		save["flag"] = "buyerser"
+		is.Val = map[string]int{}
+		go mgo.Save("datamonitor", save)
+	}
+	is.Starttime = is.Endtime
+	is.Lock.Unlock()
+	time.AfterFunc(15*time.Minute, is.Save)
+}

udpextractbuyer/src/main.go

@@ -0,0 +1,167 @@
+package main
+
+/**
+生成中标企业库
+redis去重后存入mongodb库，然后调用下一个节点生成索引
+**/
+
+import (
+	"encoding/json"
+	"log"
+	mu "mfw/util"
+	"net"
+	"qfw/util"
+	//"qfw/util/elastic"
+	"qfw/util/mongodb"
+	"qfw/util/redis"
+	"sync"
+	"time"
+)
+
+var (
+	Sysconfig map[string]interface{}   //配置文件
+	mconf     map[string]interface{}   //mongodb配置信息
+	mgo       *mongodb.MongodbSim      //mongodb操作对象
+	udpclient mu.UdpClient             //udp对象
+	nextNode  []map[string]interface{} //下节点数组
+	toaddr    = []*net.UDPAddr{}       //下节点对象
+	collect   string                   //保存的库名
+	redisLock = &sync.Mutex{}
+)
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	collect, _ = Sysconfig["collect"].(string)
+	nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
+	for _, m := range nextNode {
+		toaddr = append(toaddr, &net.UDPAddr{
+			IP:   net.ParseIP(m["addr"].(string)),
+			Port: util.IntAll(m["port"]),
+		})
+	}
+	mconf = Sysconfig["mongodb"].(map[string]interface{})
+	mgo = &mongodb.MongodbSim{
+		MongodbAddr: mconf["addr"].(string),
+		DbName:      mconf["db"].(string),
+		Size:        util.IntAllDef(mconf["pool"], 5),
+	}
+	mgo.InitPool()
+
+	rconf := Sysconfig["redis"].(map[string]interface{})
+	redis.InitRedisBySize(rconf["addr"].(string), util.IntAllDef(rconf["pool"], 5), 5, 240)
+}
+
+func main() {
+	updport := Sysconfig["udpport"].(string)
+	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Println("Udp服务监听", updport)
+	time.Sleep(99999 * time.Hour)
+}
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	log.Println(act, "---", mu.OP_TYPE_DATA)
+	switch act {
+	case mu.OP_TYPE_DATA: //上个节点的数据
+		//从表中开始处理生成企业数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Println("err:", err, "mapInfo:", mapInfo)
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			udpclient.WriteUdp([]byte("ok,run"), mu.OP_NOOP, ra)
+			time.Sleep(2 * time.Second)
+			go task(data, mapInfo)
+		}
+	case mu.OP_NOOP: //下个节点回应
+		log.Println("发送成功", string(data))
+	}
+}
+
+//开始判重程序
+func task(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	//区间id
+	sess := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
+			"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+		},
+	}
+	it := sess.DB(mgo.DbName).C(mconf["collect"].(string)).Find(&q).Iter()
+	nameArr := []map[string]interface{}{}
+	nameLock := &sync.Mutex{}
+	pool := make(chan bool, 8)
+	wg := &sync.WaitGroup{}
+	n, newN := 0, 0
+	pici := time.Now().Unix()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			names := []string{}
+			buyer, _ := tmp["buyer"].(string)
+			buyerclass, _ := tmp["buyerclass"].(string)
+			if len([]rune(buyer)) > 5 {
+				names = append(names, buyer)
+			}
+			if len(names) > 0 {
+				for _, buyer := range names {
+					redisLock.Lock()
+					b, _ := redis.Exists("buyer", "buyer_"+buyer)
+					if !b {
+						go IS.Add("normal")
+						newN++
+						redis.PutCKV("buyer", "buyer_"+buyer, 1)
+						nameLock.Lock()
+						nameArr = append(nameArr, map[string]interface{}{
+							"name":       buyer,
+							"sid":        util.BsonIdToSId(tmp["_id"]),
+							"pici":       pici,
+							"buyerclass": buyerclass,
+							"buyername":  buyer,
+						})
+						nameLock.Unlock()
+					} else {
+						go IS.Add("repeat")
+					}
+					redisLock.Unlock()
+				}
+			}
+			if n%1000 == 0 {
+				log.Println("current:", n)
+			}
+			nameLock.Lock()
+			if len(nameArr) >= 800 {
+				mgo.SaveBulk(collect, nameArr...)
+				nameArr = []map[string]interface{}{}
+			}
+			nameLock.Unlock()
+		}(tmp)
+
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	if len(nameArr) > 0 {
+		mgo.SaveBulk(collect, nameArr...)
+	}
+	log.Println("this task over.", n, "newN:", newN)
+	//任务完成，开始发送广播通知下面节点
+	if newN > 0 {
+		mapInfo["stype"] = "buyer"
+		mapInfo["query"] = map[string]interface{}{
+			"pici": pici,
+		}
+		data, _ = json.Marshal(mapInfo)
+		for _, to := range toaddr {
+			udpclient.WriteUdp(data, mu.OP_TYPE_DATA, to)
+		}
+	}
+}

udpextractwinner/src/README.md

@@ -0,0 +1,2 @@
+根据pici字段进行创建全文检索
+要在表上建索引

udpextractwinner/src/config.json

@@ -0,0 +1,25 @@
+{
+    "udpport": ":1480",
+    "collect": "winner_20190220",
+    "mongodb": {
+        "addr": "172.17.145.163:27080",
+        "pool": 15,
+        "db": "extract_v3",
+        "collect": "result_20190220"
+    },
+    "jkmail": {
+        "to": "zhangjinkun@topnet.net.cn",
+        "api": "http://10.171.112.160:19281/_send/_mail"
+    },
+    "redis": {
+        "addr": "winner=10.172.228.50:2710",
+        "pool": 30
+    },
+    "nextNode": [
+        {
+            "addr": "127.0.0.1",
+            "port": 1483,
+            "memo": "创建中标企业索引"
+        }
+    ]
+}

udpextractwinner/src/datamonitor.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"sync"
+	"time"
+)
+
+/**
+增加数据监控服务，每15分钟保存一次抽取情况
+**/
+var IS = &InfoStatus{
+	time.Now().Unix(), 0, map[string]int{}, &sync.Mutex{}}
+
+func init() {
+	go IS.Save()
+}
+
+type InfoStatus struct {
+	Starttime int64
+	Endtime   int64
+	Val       map[string]int
+	Lock      *sync.Mutex
+}
+
+func (is *InfoStatus) Add(t string) {
+	is.Lock.Lock()
+	is.Val[t]++
+	is.Lock.Unlock()
+}
+
+func (is *InfoStatus) Save() {
+	is.Lock.Lock()
+	is.Endtime = time.Now().Unix()
+	save := map[string]interface{}{}
+	all := 0
+	for k, v := range is.Val {
+		all += v
+		save[k] = v
+	}
+	if all > 0 {
+		save["receive"] = all
+		save["starttime"] = is.Starttime
+		save["endtime"] = is.Endtime
+		save["flag"] = "winnerser"
+		is.Val = map[string]int{}
+		go mgo.Save("datamonitor", save)
+	}
+	is.Starttime = is.Endtime
+	is.Lock.Unlock()
+	time.AfterFunc(15*time.Minute, is.Save)
+}

udpextractwinner/src/main.go

@@ -0,0 +1,211 @@
+package main
+
+/**
+生成中标企业库
+redis去重后存入mongodb库，然后调用下一个节点生成索引
+**/
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	mu "mfw/util"
+	"net"
+	"qfw/util"
+	//"qfw/util/elastic"
+	"qfw/util/mongodb"
+	"qfw/util/redis"
+	"sync"
+	"time"
+)
+
+var (
+	Sysconfig map[string]interface{}   //配置文件
+	mconf     map[string]interface{}   //mongodb配置信息
+	mgo       *mongodb.MongodbSim      //mongodb操作对象
+	udpclient mu.UdpClient             //udp对象
+	nextNode  []map[string]interface{} //下节点数组
+	toaddr    = []*net.UDPAddr{}       //下节点对象
+	collect   string                   //保存的库名
+	redisLock = &sync.Mutex{}
+)
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	collect, _ = Sysconfig["collect"].(string)
+	nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
+	for _, m := range nextNode {
+		toaddr = append(toaddr, &net.UDPAddr{
+			IP:   net.ParseIP(m["addr"].(string)),
+			Port: util.IntAll(m["port"]),
+		})
+	}
+	mconf = Sysconfig["mongodb"].(map[string]interface{})
+	mgo = &mongodb.MongodbSim{
+		MongodbAddr: mconf["addr"].(string),
+		DbName:      mconf["db"].(string),
+		Size:        util.IntAllDef(mconf["pool"], 5),
+	}
+	mgo.InitPool()
+
+	rconf := Sysconfig["redis"].(map[string]interface{})
+	redis.InitRedisBySize(rconf["addr"].(string), util.IntAllDef(rconf["pool"], 5), 5, 240)
+}
+
+func main() {
+	go checkMapJob()
+	updport := Sysconfig["udpport"].(string)
+	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Println("Udp服务监听", updport)
+	time.Sleep(99999 * time.Hour)
+}
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA: //上个节点的数据
+		//从表中开始处理生成企业数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Println("err:", err, "mapInfo:", mapInfo)
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			go task(data, mapInfo)
+			key, _ := mapInfo["key"].(string)
+			if key == "" {
+				key = "udpok"
+			}
+			udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
+		}
+	case mu.OP_NOOP: //下个节点回应
+		ok := string(data)
+		if ok != "" {
+			log.Println("ok:", ok)
+			udptaskmap.Delete(ok)
+		}
+	}
+}
+
+//开始判重程序
+func task(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	//区间id
+	sess := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
+			"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+		},
+	}
+	it := sess.DB(mgo.DbName).C(mconf["collect"].(string)).Find(&q).Iter()
+	nameArr := []map[string]interface{}{}
+	nameLock := &sync.Mutex{}
+	pool := make(chan bool, 8)
+	wg := &sync.WaitGroup{}
+	n, newN := 0, 0
+	pici := time.Now().Unix()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			//names := []string{}
+			names := []map[string]string{}
+			winner, _ := tmp["winner"].(string)
+			if len(winner) > 9 {
+				//names = append(names, winner)
+				names = append(names, map[string]string{"name": winner, "id": util.BsonIdToSId(tmp["_id"]), "type": "winner"})
+			}
+			winnerOrder, _ := tmp["winnerorder"].([]map[string]interface{})
+			if len(winnerOrder) > 0 {
+				for _, m := range winnerOrder {
+					wm := m["entname"].(string)
+					if len(wm) > 9 {
+						//names = append(names, wm)
+						names = append(names, map[string]string{"name": wm, "id": util.BsonIdToSId(tmp["_id"]), "type": "winnerorder"})
+					}
+				}
+			}
+			package1 := tmp["package"]
+			if package1 != nil {
+				packageM := package1.(map[string]interface{})
+				for _, p := range packageM {
+					pm := p.(map[string]interface{})
+					pw, _ := pm["winner"].(string)
+					if len(pw) > 9 {
+						//names = append(names, pw)
+						names = append(names, map[string]string{"name": pw, "id": util.BsonIdToSId(tmp["_id"]), "type": "package"})
+					}
+					pwo, _ := pm["winnerorder"].([]map[string]interface{})
+					if len(pwo) > 0 {
+						for _, m := range pwo {
+							wm := m["entname"].(string)
+							if len(wm) > 9 {
+								//names = append(names, wm)
+								names = append(names, map[string]string{"name": wm, "id": util.BsonIdToSId(tmp["_id"]), "type": "pwinnerorder"})
+							}
+						}
+					}
+				}
+			}
+			if len(names) > 0 {
+				for _, tmp := range names {
+					redisLock.Lock()
+					b, _ := redis.Exists("winner", tmp["name"])
+					if !b {
+						go IS.Add("normal")
+						newN++
+						redis.PutCKV("winner", tmp["name"], 1)
+						nameLock.Lock()
+						nameArr = append(nameArr, map[string]interface{}{
+							"name": tmp["name"],
+							"sid":  tmp["id"],
+							"type": tmp["type"],
+							"pici": pici,
+						})
+						nameLock.Unlock()
+					} else {
+						go IS.Add("repeat")
+					}
+					redisLock.Unlock()
+				}
+			}
+			if n%1000 == 0 {
+				log.Println("current:", n)
+			}
+			nameLock.Lock()
+			if len(nameArr) >= 800 {
+				mgo.SaveBulk(collect, nameArr...)
+				nameArr = []map[string]interface{}{}
+			}
+			nameLock.Unlock()
+		}(tmp)
+
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	if len(nameArr) > 0 {
+		mgo.SaveBulk(collect, nameArr...)
+	}
+	log.Println("this task over.", n, "newN:", newN)
+	//任务完成，开始发送广播通知下面节点
+	if newN > 0 && mapInfo["stop"] == nil {
+		mapInfo["stype"] = "winner"
+		for n, to := range toaddr {
+			key := fmt.Sprintf("%d-%s-%d", pici, "winner", n)
+			mapInfo["query"] = map[string]interface{}{
+				"pici": pici,
+			}
+			mapInfo["key"] = key
+			data, _ = json.Marshal(mapInfo)
+			node := &udpNode{data, to, time.Now().Unix(), 0}
+			udptaskmap.Store(key, node)
+			udpclient.WriteUdp(data, mu.OP_TYPE_DATA, to)
+		}
+	}
+}

udpextractwinner/src/udptaskmap.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	mu "mfw/util"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+)
+
+var udptaskmap = &sync.Map{}
+var tomail string
+var api string
+
+type udpNode struct {
+	data      []byte
+	addr      *net.UDPAddr
+	timestamp int64
+	retry     int
+}
+
+func checkMapJob() {
+	//阿里云内网无法发送邮件
+	jkmail, _ := Sysconfig["jkmail"].(map[string]interface{})
+	if jkmail != nil {
+		tomail, _ = jkmail["to"].(string)
+		api, _ = jkmail["api"].(string)
+	}
+	log.Println("start checkMapJob", tomail, Sysconfig["jkmail"])
+	for {
+		udptaskmap.Range(func(k, v interface{}) bool {
+			now := time.Now().Unix()
+			node, _ := v.(*udpNode)
+			if now-node.timestamp > 120 {
+				node.retry++
+				if node.retry > 5 {
+					log.Println("udp重试失败", k)
+					udptaskmap.Delete(k)
+					res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, tomail, "extract-send-fail", k.(string)))
+					if err == nil {
+						defer res.Body.Close()
+						read, err := ioutil.ReadAll(res.Body)
+						log.Println("邮件发发送：", string(read), err)
+					}
+				} else {
+					log.Println("udp重发", k)
+					udpclient.WriteUdp(node.data, mu.OP_TYPE_DATA, node.addr)
+				}
+			} else if now-node.timestamp > 10 {
+				log.Println("udp任务超时中..", k)
+			}
+			return true
+		})
+		time.Sleep(60 * time.Second)
+	}
+}

udpfilterdup/src/README.md

@@ -0,0 +1 @@
+基于内存的信息重复过滤

udpfilterdup/src/config.json

@@ -0,0 +1,30 @@
+{
+    "udpport": ":1485",
+    "dupdays": 2,
+    "mongodb": {
+        "addr": "192.168.3.207:27080",
+        "pool": 15,
+        "db": "biaozhu",
+        "extract": "extract_126",
+        "bidding": "bidding_126"
+    },
+    "jkmail": {
+        "to": "renzheng@topnet.net.cn",
+        "api": "http://10.171.112.160:19281/_send/_mail"
+    },
+    "nextNode": [
+        {
+            "addr": "127.0.0.11",
+            "port": 1482,
+            "stype": "project",
+            "memo": "合并项目"
+        },
+        {
+            "addr": "127.0.0.1",
+            "port": 1483,
+            "stype": "bidding",
+            "memo": "创建招标数据索引"
+        }
+    ],
+    "specialwords": "[(（]?[0-9一二三四五六七八九十零１２３４５６７８９再][)）]?[子分]?[次批标包]|重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研"
+}

udpfilterdup/src/datamap.go

@@ -0,0 +1,227 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"math"
+	qutil "qfw/util"
+	"qfw/util/mongodb"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Info struct {
+	id                 string
+	title              string
+	area               string
+	city               string
+	subtype            string
+	buyer              string
+	agency             string //代理机构
+	winner             string //中标单位
+	projectname        string
+	projectcode        string
+	publishtime        int64
+	comeintime         int64
+	ContainSpecialWord bool
+}
+
+var datelimit = float64(432000)
+
+type datamap struct {
+	lock   sync.Mutex //锁
+	days   int        //保留几天数据
+	data   map[string][]*Info
+	keymap []string
+	keys   map[string]bool
+}
+
+func NewDatamap(days int, lastid string) *datamap {
+	datelimit = qutil.Float64All(days * 86400)
+	dm := &datamap{sync.Mutex{}, days, map[string][]*Info{}, []string{}, map[string]bool{}}
+	if lastid == "" {
+		return dm
+	}
+	//初始化加载数据
+	sess := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(sess)
+	it := sess.DB(mgo.DbName).C(extract).Find(mongodb.ObjToMQ(`{"_id":{"$lte":"`+lastid+`"}}`, true)).Sort("-_id").Iter()
+	now1 := int64(0)
+	n, continuSum := 0, 0
+	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
+		//
+		if qutil.IntAll(tmp["repeat"]) == 1 || qutil.ObjToString(tmp["subtype"]) == "变更" {
+			continuSum++
+		} else {
+			cm := tmp["comeintime"]
+			//cm := tmp["publishtime"]
+			comeintime := qutil.Int64All(cm)
+			if comeintime == 0 {
+				id := qutil.BsonIdToSId(tmp["_id"])[0:8]
+				comeintime, _ = strconv.ParseInt(id, 16, 64)
+			}
+			if now1 == 0 {
+				now1 = comeintime
+			}
+			if qutil.Float64All(now1-comeintime) < datelimit {
+				info := NewInfo(tmp)
+				dkey := qutil.FormatDateWithObj(&cm, qutil.Date_yyyyMMdd)
+				k := fmt.Sprintf("%s_%s_%s", dkey, info.subtype, info.area)
+				data := dm.data[k]
+				if data == nil {
+					data = []*Info{}
+					//log.Println(k)
+				}
+				data = append(data, info)
+				dm.data[k] = data
+				dm.keys[dkey] = true
+			} else {
+				break
+			}
+		}
+		if n%5000 == 0 {
+			log.Println("current n:", n, continuSum)
+		}
+		tmp = make(map[string]interface{})
+	}
+	log.Println("load data:", n)
+	return dm
+}
+func NewInfo(tmp map[string]interface{}) *Info {
+	subtype := qutil.ObjToString(tmp["subtype"])
+	area := qutil.ObjToString(tmp["area"])
+	if area == "A" {
+		area = "全国"
+	}
+	info := &Info{}
+	info.id = qutil.BsonIdToSId(tmp["_id"])
+	info.title = qutil.ObjToString(tmp["title"])
+	info.area = area
+	info.subtype = subtype
+	info.buyer = qutil.ObjToString(tmp["buyer"])
+	info.projectname = qutil.ObjToString(tmp["projectname"])
+	info.ContainSpecialWord = FilterRegexp.MatchString(info.projectname) || FilterRegexp.MatchString(info.title)
+	info.projectcode = qutil.ObjToString(tmp["projectcode"])
+	info.city = qutil.ObjToString(tmp["city"])
+	info.agency = qutil.ObjToString(tmp["agency"])
+	//info.winner = qutil.ObjToString(tmp["winner"])
+	info.publishtime = qutil.Int64All(tmp["publishtime"])
+	return info
+}
+
+func (d *datamap) check(info *Info) (b bool, id string) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	keys := []string{}
+	for k, _ := range d.keys {
+		keys = append(keys, fmt.Sprintf("%s_%s_%s", k, info.subtype, info.area))
+		if info.area != "全国" { //这个后续可以不要
+			keys = append(keys, fmt.Sprintf("%s_%s_%s", k, info.subtype, "全国"))
+		}
+	}
+L:
+	for _, k := range keys {
+		data := d.data[k]
+		if len(data) > 0 { //对比
+			for _, v := range data {
+				if v.id == info.id {
+					return false, v.id
+				}
+				if math.Abs(qutil.Float64All(v.publishtime-info.publishtime)) > datelimit {
+					continue
+				}
+				if v.agency != "" && info.agency != "" && v.agency != info.agency {
+					continue
+				}
+				n := 0
+				if v.buyer != "" && v.buyer == info.buyer {
+					n++
+				}
+				if v.projectname != "" && v.projectname == info.projectname {
+					n++
+				}
+				if !info.ContainSpecialWord && n > 1 {
+					b = true
+					id = v.id
+					break L
+				} else if v.projectcode != "" && v.projectcode == info.projectcode {
+					n++
+				}
+				if !info.ContainSpecialWord && n > 1 || n > 2 {
+					b = true
+					id = v.id
+					break L
+				}
+				//标题长度大于10且相等即为重复
+				//				if len([]rune(info.title)) > 10 && v.title == info.title {
+				//					b = true
+				//					id = v.id
+				//					break L
+				//				}
+				//标题长度大于10且包含关系+buyer/projectname/projectcode/city（全国/A的只判断包含关系即可）相等即为重复
+				if len([]rune(info.title)) > 10 && len([]rune(v.title)) > 10 && (strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title)) {
+					if info.area == "全国" || n > 0 || info.city == v.city {
+						b = true
+						id = v.id
+						break L
+					}
+				}
+			}
+		}
+	}
+	if !b {
+		ct, _ := strconv.ParseInt(info.id[:8], 16, 64)
+		dkey := qutil.FormatDateByInt64(&ct, qutil.Date_yyyyMMdd)
+		k := fmt.Sprintf("%s_%s_%s", dkey, info.subtype, info.area)
+		data := d.data[k]
+		if data == nil {
+			data = []*Info{info}
+			d.data[k] = data
+			if !d.keys[dkey] {
+				d.keys[dkey] = true
+				d.update(ct)
+			}
+		} else {
+			data = append(data, info)
+			d.data[k] = data
+		}
+	}
+	return
+}
+
+func (d *datamap) update(t int64) {
+	//每天0点清除历史数据
+	d.keymap = d.GetLatelyFiveDay(t)
+	m := map[string]bool{}
+	for _, v := range d.keymap {
+		m[v] = true
+	}
+	all, all1 := 0, 0
+	for k, v := range d.data {
+		all += len(v)
+		if !m[k[:8]] {
+			delete(d.data, k)
+		}
+	}
+	for k, _ := range d.keys {
+		if !m[k] {
+			delete(d.keys, k)
+		}
+	}
+	for _, v := range d.data {
+		all1 += len(v)
+	}
+	log.Println("更新前后数据:", all, all1)
+}
+
+func (d *datamap) GetLatelyFiveDay(t int64) []string {
+	array := make([]string, d.days)
+	now := time.Unix(t, 0)
+	for i := 0; i < d.days; i++ {
+		array[i] = now.Format(qutil.Date_yyyyMMdd)
+		now = now.AddDate(0, 0, -1)
+	}
+	return array
+}

udpfilterdup/src/datamap.go.bak

@@ -0,0 +1,205 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"math"
+	qutil "qfw/util"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Info struct {
+	id                 string
+	title              string
+	area               string
+	city               string
+	subtype            string
+	buyer              string
+	agency             string //代理机构
+	winner             string //中标单位
+	projectname        string
+	projectcode        string
+	publishtime        int64
+	ContainSpecialWord bool
+}
+
+var datelimit = float64(432000)
+
+type datamap struct {
+	lock   sync.Mutex //锁
+	days   int        //保留几天数据
+	data   map[string][]*Info
+	keymap []string
+}
+
+func NewDatamap(days int) *datamap {
+	datelimit = qutil.Float64All(days * 86400)
+	dm := &datamap{sync.Mutex{}, days, map[string][]*Info{}, []string{}}
+	dm.keymap = dm.GetLatelyFiveDay()
+	//初始化加载数据
+	sess := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(sess)
+	it := sess.DB(mgo.DbName).C(extract).Find(nil).Sort("-_id").Iter()
+	now1 := time.Now().Unix()
+	n, continuSum := 0, 0
+	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
+		//
+		if qutil.IntAll(tmp["repeat"]) == 1 || qutil.ObjToString(tmp["subtype"]) == "变更" {
+			continuSum++
+		} else {
+			cm := tmp["comeintime"]
+			comeintime := qutil.Int64All(cm)
+			if qutil.Float64All(now1-comeintime) < datelimit {
+				info := NewInfo(tmp)
+				k := fmt.Sprintf("%s_%s_%s", qutil.FormatDateWithObj(&cm, qutil.Date_yyyyMMdd), info.subtype, info.area)
+				data := dm.data[k]
+				if data == nil {
+					data = []*Info{}
+					//log.Println(k)
+				}
+				data = append(data, info)
+				dm.data[k] = data
+			} else {
+				break
+			}
+		}
+		if n%5000 == 0 {
+			log.Println("current n:", n, continuSum)
+		}
+		tmp = make(map[string]interface{})
+	}
+	log.Println("load data:", n)
+	//启动定时任务
+	now := time.Now()
+	t2 := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, time.Local)
+	go time.AfterFunc(time.Duration(int64(t2.Unix()-now.Unix()))*time.Second, func() {
+		//go time.AfterFunc(time.Duration(10)*time.Second, func() {
+		dm.update()
+	})
+	return dm
+}
+func NewInfo(tmp map[string]interface{}) *Info {
+	subtype := qutil.ObjToString(tmp["subtype"])
+	area := qutil.ObjToString(tmp["area"])
+	if area == "A" {
+		area = "全国"
+	}
+	info := &Info{}
+	info.id = qutil.BsonIdToSId(tmp["_id"])
+	info.title = qutil.ObjToString(tmp["title"])
+	info.area = area
+	info.subtype = subtype
+	info.buyer = qutil.ObjToString(tmp["buyer"])
+	info.projectname = qutil.ObjToString(tmp["projectname"])
+	info.ContainSpecialWord = FilterRegexp.MatchString(info.projectname) || FilterRegexp.MatchString(info.title)
+	info.projectcode = qutil.ObjToString(tmp["projectcode"])
+	info.city = qutil.ObjToString(tmp["city"])
+	info.agency = qutil.ObjToString(tmp["agency"])
+	//info.winner = qutil.ObjToString(tmp["winner"])
+	info.publishtime = qutil.Int64All(tmp["publishtime"])
+	return info
+}
+
+func (d *datamap) check(info *Info) (b bool, id string) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	keys := []string{}
+	for _, k := range d.keymap {
+		keys = append(keys, fmt.Sprintf("%s_%s_%s", k, info.subtype, info.area))
+		if info.area != "全国" { //这个后续可以不要
+			keys = append(keys, fmt.Sprintf("%s_%s_%s", k, info.subtype, "全国"))
+		}
+	}
+L:
+	for _, k := range keys {
+		data := d.data[k]
+		if len(data) > 1 { //对比
+			for _, v := range data {
+				if math.Abs(qutil.Float64All(v.publishtime-info.publishtime)) > datelimit {
+					continue
+				}
+				if v.agency != "" && info.agency != "" && v.agency != info.agency {
+					continue
+				}
+				n := 0
+				if v.buyer != "" && v.buyer == info.buyer {
+					n++
+				}
+				if v.projectname != "" && v.projectname == info.projectname {
+					n++
+				}
+				if !info.ContainSpecialWord && n > 1 {
+					b = true
+					id = v.id
+					break L
+				} else if v.projectcode != "" && v.projectcode == info.projectcode {
+					n++
+				}
+				if !info.ContainSpecialWord && n > 1 || n > 2 {
+					b = true
+					id = v.id
+					break L
+				}
+				//标题长度大于10且相等即为重复
+				//				if len([]rune(info.title)) > 10 && v.title == info.title {
+				//					b = true
+				//					id = v.id
+				//					break L
+				//				}
+				//标题长度大于10且包含关系+buyer/projectname/projectcode/city（全国/A的只判断包含关系即可）相等即为重复
+				if len([]rune(info.title)) > 10 && len([]rune(v.title)) > 10 && (strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title)) {
+					if info.area == "全国" || n > 0 || info.city == v.city {
+						b = true
+						id = v.id
+						break L
+					}
+				}
+			}
+		}
+	}
+	if !b {
+		k := fmt.Sprintf("%s_%s_%s", time.Now().Format(qutil.Date_yyyyMMdd), info.subtype, info.area)
+		data := d.data[k]
+		if data == nil {
+			data = []*Info{info}
+		} else {
+			data = append(data, info)
+		}
+		d.data[k] = data
+	}
+	return
+}
+
+func (d *datamap) update() {
+	//每天0点清除历史数据
+	d.lock.Lock()
+	now, now1 := time.Now(), time.Now()
+	t2 := time.Date(now1.Year(), now1.Month(), now1.Day()+1, 0, 0, 0, 0, time.Local)
+	date := now.AddDate(0, 0, -d.days).Format(qutil.Date_yyyyMMdd)
+	all, all1 := 0, 0
+	for k, v := range d.data {
+		all += len(v)
+		if strings.HasPrefix(k, date) {
+			delete(d.data, k)
+		}
+	}
+	for _, v := range d.data {
+		all1 += len(v)
+	}
+	log.Println("更新前后数据:", all, all1)
+	d.keymap = d.GetLatelyFiveDay()
+	d.lock.Unlock()
+	time.AfterFunc(time.Duration(int64(t2.Unix()-now1.Unix()))*time.Second, d.update)
+}
+
+func (d *datamap) GetLatelyFiveDay() []string {
+	array := make([]string, d.days)
+	now := time.Now()
+	for i := 0; i < d.days; i++ {
+		array[i] = now.Format(qutil.Date_yyyyMMdd)
+		now = now.AddDate(0, 0, -1)
+	}
+	return array
+}

udpfilterdup/src/datamonitor.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"sync"
+	"time"
+)
+
+/**
+增加数据监控服务，每15分钟保存一次重复情况
+**/
+var IS = &InfoStatus{
+	time.Now().Unix(), 0, map[string]int{}, &sync.Mutex{}}
+
+func init() {
+	//go IS.Save()
+}
+
+type InfoStatus struct {
+	Starttime int64
+	Endtime   int64
+	Val       map[string]int
+	Lock      *sync.Mutex
+}
+
+func (is *InfoStatus) Add(t string) {
+	is.Lock.Lock()
+	is.Val[t]++
+	is.Lock.Unlock()
+}
+
+func (is *InfoStatus) Save() {
+	is.Lock.Lock()
+	is.Endtime = time.Now().Unix()
+	save := map[string]interface{}{}
+	all := 0
+	for k, v := range is.Val {
+		all += v
+		save[k] = v
+	}
+	if all > 0 {
+		save["receive"] = all
+		save["starttime"] = is.Starttime
+		save["endtime"] = is.Endtime
+		save["flag"] = "dup"
+		is.Val = map[string]int{}
+		go mgo.Save("datamonitor", save)
+	}
+	is.Starttime = is.Endtime
+	is.Lock.Unlock()
+	time.AfterFunc(15*time.Minute, is.Save)
+}

udpfilterdup/src/main.go

@@ -0,0 +1,193 @@
+package main
+
+/**
+招标信息判重
+**/
+
+import (
+	"encoding/json"
+	"flag"
+	"log"
+	mu "mfw/util"
+	"net"
+	"qfw/util"
+	"qfw/util/mongodb"
+	"regexp"
+	"sync"
+	"time"
+)
+
+var (
+	Sysconfig    map[string]interface{} //配置文件
+	mconf        map[string]interface{} //mongodb配置信息
+	mgo          *mongodb.MongodbSim    //mongodb操作对象
+	extract      string
+	bidding      string
+	udpclient    mu.UdpClient             //udp对象
+	nextNode     []map[string]interface{} //下节点数组
+	dupdays      = 3                      //初始化判重范围
+	DM           *datamap                 //判重数据
+	FilterRegexp = regexp.MustCompile("^_$")
+	lastid       = ""
+)
+
+func init() {
+	flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
+	flag.Parse()
+	util.ReadConfig(&Sysconfig)
+	nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
+	mconf = Sysconfig["mongodb"].(map[string]interface{})
+	mgo = &mongodb.MongodbSim{
+		MongodbAddr: mconf["addr"].(string),
+		DbName:      mconf["db"].(string),
+		Size:        util.IntAllDef(mconf["pool"], 10),
+	}
+	extract = mconf["extract"].(string)
+	//bidding = mconf["bidding"].(string)
+	mgo.InitPool()
+	dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
+	//加载数据
+	DM = NewDatamap(dupdays, lastid)
+	sw := util.ObjToString(Sysconfig["specialwords"])
+	if sw != "" {
+		FilterRegexp = regexp.MustCompile(sw)
+	}
+}
+
+func main() {
+	go checkMapJob()
+	updport := Sysconfig["udpport"].(string)
+	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Println("Udp服务监听", updport)
+	time.Sleep(99999 * time.Hour)
+}
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA: //上个节点的数据
+		//从表中开始处理
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Println("err:", err, "mapInfo:", mapInfo)
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			go task(data, mapInfo)
+			key, _ := mapInfo["key"].(string)
+			if key == "" {
+				key = "udpok"
+			}
+			udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
+		}
+	case mu.OP_NOOP: //下个节点回应
+		ok := string(data)
+		if ok != "" {
+			log.Println("ok:", ok)
+			udptaskmap.Delete(ok)
+		}
+	}
+}
+
+//开始判重程序
+func task(data []byte, mapInfo map[string]interface{}) {
+	defer util.Catch()
+	//区间id
+	sess := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
+			"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+		},
+	}
+	it := sess.DB(mgo.DbName).C(extract).Find(&q).Iter()
+	updateExtract := [][]map[string]interface{}{}
+	pool := make(chan bool, 16)
+	wg := &sync.WaitGroup{}
+	mapLock := &sync.Mutex{}
+	n, repeateN := 0, 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
+		if util.ObjToString(tmp["subtype"]) == "变更" {
+			//go IS.Add("update")
+			continue
+		}
+		if n%10000 == 0 {
+			log.Println("current:", n, tmp["_id"])
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			info := NewInfo(tmp)
+			b, id := DM.check(info)
+			if b { //有重复,生成更新语句，更新抽取和更新招标
+				//IS.Add("repeat")
+				repeateN++
+				mapLock.Lock()
+				updateExtract = append(updateExtract, []map[string]interface{}{
+					map[string]interface{}{
+						"_id": tmp["_id"],
+					},
+					map[string]interface{}{
+						"$set": map[string]interface{}{
+							"repeat":   1,
+							"repeatid": id,
+						},
+					},
+				})
+				//				updateBidding = append(updateBidding, []map[string]interface{}{
+				//					map[string]interface{}{
+				//						"_id": tmp["_id"],
+				//					},
+				//					map[string]interface{}{
+				//						"$set": map[string]interface{}{
+				//							"extracttype": -1,
+				//							"repeatid":    id,
+				//						},
+				//					},
+				//				})
+				if len(updateExtract) > 500 {
+					mgo.UpdateBulk(extract, updateExtract...)
+					//mgo.UpdateBulk(bidding, updateBidding...)
+					//updateExtract, updateBidding = [][]map[string]interface{}{}, [][]map[string]interface{}{}
+					updateExtract = [][]map[string]interface{}{}
+				}
+				mapLock.Unlock()
+			} else {
+				//IS.Add("new")
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	if len(updateExtract) > 0 {
+		mgo.UpdateBulk(extract, updateExtract...)
+		//mgo.UpdateBulk(bidding, updateBidding...)
+	}
+	log.Println("this task over.", n, "repeateN:", repeateN, mapInfo["stop"])
+	//任务完成，开始发送广播通知下面节点
+	if n > repeateN && mapInfo["stop"] == nil {
+		for _, to := range nextNode {
+			sid, _ := mapInfo["gtid"].(string)
+			eid, _ := mapInfo["lteid"].(string)
+			key := sid + "-" + eid + "-" + util.ObjToString(to["stype"])
+			by, _ := json.Marshal(map[string]interface{}{
+				"gtid":  sid,
+				"lteid": eid,
+				"stype": util.ObjToString(to["stype"]),
+				"key":   key,
+			})
+			addr := &net.UDPAddr{
+				IP:   net.ParseIP(to["addr"].(string)),
+				Port: util.IntAll(to["port"]),
+			}
+			node := &udpNode{by, addr, time.Now().Unix(), 0}
+			udptaskmap.Store(key, node)
+			udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
+		}
+	}
+}

udpfilterdup/src/udptaskmap.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	mu "mfw/util"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+)
+
+var udptaskmap = &sync.Map{}
+var tomail string
+var api string
+
+type udpNode struct {
+	data      []byte
+	addr      *net.UDPAddr
+	timestamp int64
+	retry     int
+}
+
+func checkMapJob() {
+	//阿里云内网无法发送邮件
+	jkmail, _ := Sysconfig["jkmail"].(map[string]interface{})
+	if jkmail != nil {
+		tomail, _ = jkmail["to"].(string)
+		api, _ = jkmail["api"].(string)
+	}
+	log.Println("start checkMapJob", tomail, Sysconfig["jkmail"])
+	for {
+		udptaskmap.Range(func(k, v interface{}) bool {
+			now := time.Now().Unix()
+			node, _ := v.(*udpNode)
+			if now-node.timestamp > 120 {
+				node.retry++
+				if node.retry > 5 {
+					log.Println("udp重试失败", k)
+					udptaskmap.Delete(k)
+					res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, tomail, "extract-send-fail", k.(string)))
+					if err == nil {
+						defer res.Body.Close()
+						read, err := ioutil.ReadAll(res.Body)
+						log.Println("邮件发发送：", string(read), err)
+					}
+				} else {
+					log.Println("udp重发", k)
+					udpclient.WriteUdp(node.data, mu.OP_TYPE_DATA, node.addr)
+				}
+			} else if now-node.timestamp > 10 {
+				log.Println("udp任务超时中..", k)
+			}
+			return true
+		})
+		time.Sleep(60 * time.Second)
+	}
+}

udpprojectset/src/city.json

@@ -0,0 +1,3239 @@
+{
+    "city": {
+        "金银川镇": "新疆",
+        "郑东新区": "河南",
+        "信阳客整所": "湖北",
+        "北屯": "新疆",
+        "鼓楼区": "江苏",
+        "辽河": "辽宁",
+        "南大港": "河北",
+        "北京": "北京",
+        "密云": "北京",
+        "延庆": "北京",
+        "天津": "天津",
+        "宁河": "天津",
+        "静海": "天津",
+        "蓟县": "天津",
+        "河北": "河北",
+        "石家庄": "河北",
+        "井陉": "河北",
+        "正定": "河北",
+        "栾城": "河北",
+        "行唐": "河北",
+        "灵寿": "河北",
+        "高邑": "河北",
+        "深泽": "河北",
+        "赞皇": "河北",
+        "无极县": "河北",
+        "平山县": "河北",
+        "元氏": "河北",
+        "赵县": "河北",
+        "辛集市": "河北",
+        "藁城": "河北",
+        "晋州": "河北",
+        "新乐": "河北",
+        "鹿泉": "河北",
+        "唐山": "河北",
+        "滦县": "河北",
+        "滦南": "河北",
+        "乐亭": "河北",
+        "迁西": "河北",
+        "玉田": "河北",
+        "唐海": "河北",
+        "遵化": "河北",
+        "迁安市": "河北",
+        "秦皇岛": "河北",
+        "青龙县": "河北",
+        "青龙满族自治县": "河北",
+        "昌黎": "河北",
+        "抚宁": "河北",
+        "卢龙": "河北",
+        "邯郸": "河北",
+        "临漳": "河北",
+        "成安": "河北",
+        "大名县": "河北",
+        "涉县": "河北",
+        "磁县": "河北",
+        "肥乡": "河北",
+        "永年": "河北",
+        "邱县": "河北",
+        "鸡泽": "河北",
+        "广平县": "河北",
+        "馆陶": "河北",
+        "魏县": "河北",
+        "曲周": "河北",
+        "武安": "河北",
+        "邢台": "河北",
+        "临城": "河北",
+        "内丘": "河北",
+        "柏乡县": "河北",
+        "隆尧": "河北",
+        "任县": "河北",
+        "南和": "河北",
+        "宁晋": "河北",
+        "巨鹿": "河北",
+        "新河县": "河北",
+        "广宗": "河北",
+        "平乡": "河北",
+        "威县": "河北",
+        "清河": "河北",
+        "临西": "河北",
+        "南宫": "河北",
+        "沙河市": "河北",
+        "保定": "河北",
+        "满城": "河北",
+        "清苑": "河北",
+        "涞水": "河北",
+        "阜平": "河北",
+        "徐水": "河北",
+        "定兴": "河北",
+        "唐县": "河北",
+        "高阳": "河北",
+        "容城": "河北",
+        "涞源": "河北",
+        "望都": "河北",
+        "安新": "河北",
+        "易县": "河北",
+        "曲阳": "河北",
+        "蠡县": "河北",
+        "顺平": "河北",
+        "博野": "河北",
+        "雄县": "河北",
+        "涿州": "河北",
+        "定州": "河北",
+        "安国": "河北",
+        "高碑店": "河北",
+        "张家口": "河北",
+        "宣化": "河北",
+        "张北": "河北",
+        "康保": "河北",
+        "沽源": "河北",
+        "尚义": "河北",
+        "蔚县": "河北",
+        "阳原": "河北",
+        "怀安": "河北",
+        "万全县": "河北",
+        "怀来": "河北",
+        "涿鹿": "河北",
+        "赤城": "河北",
+        "崇礼": "河北",
+        "承德": "河北",
+        "兴隆": "河北",
+        "平泉县": "河北",
+        "滦平": "河北",
+        "隆化": "河北",
+        "丰宁": "河北",
+        "宽城满族自治县": "河北",
+        "宽城县": "河北",
+        "围场": "河北",
+        "沧州": "河北",
+        "沧县": "河北",
+        "青县": "河北",
+        "东光县": "河北",
+        "海兴": "河北",
+        "盐山": "河北",
+        "肃宁": "河北",
+        "南皮": "河北",
+        "吴桥县": "河北",
+        "献县": "河北",
+        "孟村回族自治县": "河北",
+        "孟村县": "河北",
+        "泊头": "河北",
+        "任丘": "河北",
+        "黄骅": "河北",
+        "河间市": "河北",
+        "廊坊": "河北",
+        "固安": "河北",
+        "永清": "河北",
+        "香河": "河北",
+        "大城县": "河北",
+        "文安": "河北",
+        "大厂回族自治县": "河北",
+        "大厂县": "河北",
+        "霸州": "河北",
+        "三河": "河北",
+        "衡水": "河北",
+        "枣强": "河北",
+        "武邑": "河北",
+        "武强": "河北",
+        "饶阳": "河北",
+        "安平": "河北",
+        "故城县": "河北",
+        "景县": "河北",
+        "阜城县": "河北",
+        "冀州": "河北",
+        "深州": "河北",
+        "山西": "山西",
+        "太原": "山西",
+        "清徐": "山西",
+        "阳曲": "山西",
+        "娄烦": "山西",
+        "古交": "山西",
+        "大同市": "山西",
+        "阳高": "山西",
+        "天镇": "山西",
+        "广灵": "山西",
+        "灵丘": "山西",
+        "浑源": "山西",
+        "左云": "山西",
+        "阳泉": "山西",
+        "平定": "山西",
+        "盂县": "山西",
+        "长治": "山西",
+        "襄垣": "山西",
+        "屯留": "山西",
+        "平顺": "山西",
+        "黎城": "山西",
+        "壶关": "山西",
+        "长子": "山西",
+        "武乡": "山西",
+        "沁县": "山西",
+        "沁源": "山西",
+        "潞城": "山西",
+        "晋城": "山西",
+        "沁水": "山西",
+        "阳城": "山西",
+        "陵川": "山西",
+        "泽州": "山西",
+        "高平": "山西",
+        "朔州": "山西",
+        "山阴": "山西",
+        "应县": "山西",
+        "右玉": "山西",
+        "怀仁": "山西",
+        "晋中": "山西",
+        "榆社": "山西",
+        "左权县": "山西",
+        "和顺": "山西",
+        "昔阳": "山西",
+        "寿阳": "山西",
+        "太谷": "山西",
+        "祁县": "山西",
+        "平遥": "山西",
+        "灵石": "山西",
+        "介休": "山西",
+        "运城": "山西",
+        "临猗": "山西",
+        "万荣": "山西",
+        "闻喜": "山西",
+        "稷山": "山西",
+        "新绛": "山西",
+        "绛县": "山西",
+        "垣曲": "山西",
+        "夏县": "山西",
+        "平陆": "山西",
+        "芮城": "山西",
+        "永济": "山西",
+        "河津市": "山西",
+        "忻州": "山西",
+        "定襄": "山西",
+        "五台县": "山西",
+        "代县": "山西",
+        "繁峙": "山西",
+        "宁武": "山西",
+        "静乐": "山西",
+        "神池": "山西",
+        "五寨": "山西",
+        "岢岚": "山西",
+        "河曲": "山西",
+        "保德": "山西",
+        "偏关": "山西",
+        "原平": "山西",
+        "临汾": "山西",
+        "曲沃": "山西",
+        "翼城": "山西",
+        "襄汾": "山西",
+        "洪洞": "山西",
+        "古县": "山西",
+        "安泽": "山西",
+        "浮山": "山西",
+        "吉县": "山西",
+        "乡宁": "山西",
+        "大宁": "山西",
+        "隰县": "山西",
+        "永和": "山西",
+        "蒲县": "山西",
+        "汾西": "山西",
+        "侯马": "山西",
+        "霍州": "山西",
+        "吕梁": "山西",
+        "文水": "山西",
+        "交城": "山西",
+        "兴县": "山西",
+        "临县": "山西",
+        "柳林": "山西",
+        "石楼": "山西",
+        "岚县": "山西",
+        "方山县": "山西",
+        "中阳县": "山西",
+        "交口县": "山西",
+        "孝义市": "山西",
+        "汾阳": "山西",
+        "内蒙古": "内蒙古",
+        "呼和浩特": "内蒙古",
+        "土默特左旗": "内蒙古",
+        "托克托": "内蒙古",
+        "和林格尔": "内蒙古",
+        "清水河": "内蒙古",
+        "武川": "内蒙古",
+        "包头": "内蒙古",
+        "土默特右旗": "内蒙古",
+        "固阳": "内蒙古",
+        "达尔罕茂明安联合旗": "内蒙古",
+        "乌海": "内蒙古",
+        "赤峰": "内蒙古",
+        "阿鲁科尔沁旗": "内蒙古",
+        "巴林左旗": "内蒙古",
+        "巴林右旗": "内蒙古",
+        "林西": "内蒙古",
+        "克什克腾旗": "内蒙古",
+        "翁牛特旗": "内蒙古",
+        "喀喇沁旗": "内蒙古",
+        "宁城": "内蒙古",
+        "敖汉旗": "内蒙古",
+        "通辽": "内蒙古",
+        "科尔沁左翼中旗": "内蒙古",
+        "科尔沁左翼后旗": "内蒙古",
+        "开鲁": "内蒙古",
+        "库伦旗": "内蒙古",
+        "奈曼旗": "内蒙古",
+        "扎鲁特旗": "内蒙古",
+        "霍林郭勒": "内蒙古",
+        "鄂尔多斯": "内蒙古",
+        "达拉特旗": "内蒙古",
+        "准格尔旗": "内蒙古",
+        "鄂托克前旗": "内蒙古",
+        "鄂托克旗": "内蒙古",
+        "杭锦旗": "内蒙古",
+        "乌审旗": "内蒙古",
+        "伊金霍洛旗": "内蒙古",
+        "呼伦贝尔": "内蒙古",
+        "阿荣旗": "内蒙古",
+        "莫力达瓦达斡尔族自治旗": "内蒙古",
+        "鄂伦春自治旗": "内蒙古",
+        "鄂温克族自治旗": "内蒙古",
+        "陈巴尔虎旗": "内蒙古",
+        "新巴尔虎左旗": "内蒙古",
+        "新巴尔虎右旗": "内蒙古",
+        "满洲里": "内蒙古",
+        "牙克石": "内蒙古",
+        "扎兰屯": "内蒙古",
+        "额尔古纳": "内蒙古",
+        "根河": "内蒙古",
+        "巴彦淖尔": "内蒙古",
+        "五原": "内蒙古",
+        "磴口": "内蒙古",
+        "乌拉特前旗": "内蒙古",
+        "乌拉特中旗": "内蒙古",
+        "乌拉特后旗": "内蒙古",
+        "杭锦后旗": "内蒙古",
+        "乌兰察布": "内蒙古",
+        "卓资": "内蒙古",
+        "化德": "内蒙古",
+        "商都县": "内蒙古",
+        "兴和县": "内蒙古",
+        "凉城": "内蒙古",
+        "察哈尔右翼前旗": "内蒙古",
+        "察哈尔右翼中旗": "内蒙古",
+        "察哈尔右翼后旗": "内蒙古",
+        "四子王旗": "内蒙古",
+        "丰镇": "内蒙古",
+        "兴安盟": "内蒙古",
+        "乌兰浩特": "内蒙古",
+        "阿尔山": "内蒙古",
+        "科尔沁右翼前旗": "内蒙古",
+        "科尔沁右翼中旗": "内蒙古",
+        "扎赉特旗": "内蒙古",
+        "突泉": "内蒙古",
+        "锡林郭勒盟": "内蒙古",
+        "二连浩特": "内蒙古",
+        "锡林浩特": "内蒙古",
+        "阿巴嘎旗": "内蒙古",
+        "苏尼特左旗": "内蒙古",
+        "苏尼特右旗": "内蒙古",
+        "东乌珠穆沁旗": "内蒙古",
+        "西乌珠穆沁旗": "内蒙古",
+        "太仆寺旗": "内蒙古",
+        "镶黄旗": "内蒙古",
+        "正镶白旗": "内蒙古",
+        "正蓝旗": "内蒙古",
+        "多伦": "内蒙古",
+        "阿拉善盟": "内蒙古",
+        "阿拉善左旗": "内蒙古",
+        "阿拉善右旗": "内蒙古",
+        "额济纳旗": "内蒙古",
+        "辽宁": "辽宁",
+        "沈阳": "辽宁",
+        "辽中": "辽宁",
+        "康平": "辽宁",
+        "法库": "辽宁",
+        "新民市": "辽宁",
+        "大连": "辽宁",
+        "长海": "辽宁",
+        "瓦房店": "辽宁",
+        "普兰店": "辽宁",
+        "庄河": "辽宁",
+        "鞍山": "辽宁",
+        "台安县": "辽宁",
+        "岫岩": "辽宁",
+        "海城市": "辽宁",
+        "抚顺": "辽宁",
+        "新宾": "辽宁",
+        "清原": "辽宁",
+        "本溪": "辽宁",
+        "桓仁": "辽宁",
+        "丹东": "辽宁",
+        "宽甸": "辽宁",
+        "东港市": "辽宁",
+        "凤城": "辽宁",
+        "锦州": "辽宁",
+        "黑山县": "辽宁",
+        "义县": "辽宁",
+        "凌海": "辽宁",
+        "北镇": "辽宁",
+        "营口": "辽宁",
+        "盖州": "辽宁",
+        "大石桥市": "辽宁",
+        "阜新": "辽宁",
+        "彰武": "辽宁",
+        "辽阳": "辽宁",
+        "灯塔市": "辽宁",
+        "盘锦": "辽宁",
+        "大洼": "辽宁",
+        "盘山": "辽宁",
+        "铁岭": "辽宁",
+        "西丰": "辽宁",
+        "昌图": "辽宁",
+        "调兵山": "辽宁",
+        "开原": "辽宁",
+        "朝阳市": "辽宁",
+        "建平": "辽宁",
+        "喀喇沁": "辽宁",
+        "北票": "辽宁",
+        "凌源": "辽宁",
+        "葫芦岛": "辽宁",
+        "绥中": "辽宁",
+        "建昌": "辽宁",
+        "兴城": "辽宁",
+        "吉林": "吉林",
+        "长春": "吉林",
+        "农安": "吉林",
+        "九台": "吉林",
+        "榆树": "吉林",
+        "德惠": "吉林",
+        "吉林": "吉林",
+        "永吉": "吉林",
+        "蛟河": "吉林",
+        "桦甸": "吉林",
+        "舒兰": "吉林",
+        "磐石": "吉林",
+        "四平": "吉林",
+        "梨树县": "吉林",
+        "伊通": "吉林",
+        "公主岭": "吉林",
+        "双辽": "吉林",
+        "辽源": "吉林",
+        "东丰": "吉林",
+        "东辽": "吉林",
+        "通化": "吉林",
+        "辉南": "吉林",
+        "柳河": "吉林",
+        "梅河口": "吉林",
+        "集安": "吉林",
+        "白山市": "吉林",
+        "抚松": "吉林",
+        "靖宇": "吉林",
+        "长白": "吉林",
+        "临江": "吉林",
+        "松原": "吉林",
+        "前郭尔罗斯": "吉林",
+        "长岭": "吉林",
+        "乾安": "吉林",
+        "扶余": "吉林",
+        "白城": "吉林",
+        "镇赉": "吉林",
+        "通榆": "吉林",
+        "洮南": "吉林",
+        "大安市": "吉林",
+        "延边州": "吉林",
+        "延吉": "吉林",
+        "图们": "吉林",
+        "敦化": "吉林",
+        "珲春": "吉林",
+        "龙井市": "吉林",
+        "和龙": "吉林",
+        "汪清": "吉林",
+        "安图县": "吉林",
+        "黑龙江": "黑龙江",
+        "哈尔滨": "黑龙江",
+        "依兰县": "黑龙江",
+        "方正县": "黑龙江",
+        "宾县": "黑龙江",
+        "巴彦": "黑龙江",
+        "木兰县": "黑龙江",
+        "通河县": "黑龙江",
+        "延寿": "黑龙江",
+        "双城区": "黑龙江",
+        "尚志": "黑龙江",
+        "五常市": "黑龙江",
+        "齐齐哈尔": "黑龙江",
+        "龙江县": "黑龙江",
+        "依安": "黑龙江",
+        "泰来": "黑龙江",
+        "甘南": "黑龙江",
+        "富裕县": "黑龙江",
+        "克山县": "黑龙江",
+        "克东县": "黑龙江",
+        "拜泉": "黑龙江",
+        "讷河": "黑龙江",
+        "鸡西市": "黑龙江",
+        "鸡东县": "黑龙江",
+        "虎林": "黑龙江",
+        "密山": "黑龙江",
+        "鹤岗": "黑龙江",
+        "萝北": "黑龙江",
+        "绥滨": "黑龙江",
+        "双鸭山": "黑龙江",
+        "集贤": "黑龙江",
+        "友谊县": "黑龙江",
+        "宝清": "黑龙江",
+        "饶河": "黑龙江",
+        "大庆": "黑龙江",
+        "肇州": "黑龙江",
+        "肇源": "黑龙江",
+        "林甸": "黑龙江",
+        "杜尔伯特": "黑龙江",
+        "伊春市": "黑龙江",
+        "嘉荫": "黑龙江",
+        "铁力": "黑龙江",
+        "佳木斯": "黑龙江",
+        "桦南": "黑龙江",
+        "桦川": "黑龙江",
+        "汤原": "黑龙江",
+        "抚远": "黑龙江",
+        "同江市": "黑龙江",
+        "富锦": "黑龙江",
+        "七台河": "黑龙江",
+        "勃利": "黑龙江",
+        "牡丹江": "黑龙江",
+        "东宁": "黑龙江",
+        "林口": "黑龙江",
+        "绥芬河": "黑龙江",
+        "海林": "黑龙江",
+        "宁安": "黑龙江",
+        "穆棱": "黑龙江",
+        "黑河": "黑龙江",
+        "嫩江": "黑龙江",
+        "逊克": "黑龙江",
+        "孙吴": "黑龙江",
+        "北安": "黑龙江",
+        "五大连池": "黑龙江",
+        "绥化": "黑龙江",
+        "望奎": "黑龙江",
+        "兰西": "黑龙江",
+        "青冈": "黑龙江",
+        "庆安": "黑龙江",
+        "明水": "黑龙江",
+        "绥棱": "黑龙江",
+        "安达": "黑龙江",
+        "肇东": "黑龙江",
+        "海伦": "黑龙江",
+        "呼玛": "黑龙江",
+        "塔河": "黑龙江",
+        "漠河": "黑龙江",
+        "上海": "上海",
+        "崇明": "上海",
+        "江苏": "江苏",
+        "南京": "江苏",
+        "溧水": "江苏",
+        "高淳": "江苏",
+        "无锡": "江苏",
+        "江阴": "江苏",
+        "宜兴": "江苏",
+        "徐州": "江苏",
+        "丰县": "江苏",
+        "沛县": "江苏",
+        "铜山": "江苏",
+        "睢宁": "江苏",
+        "新沂": "江苏",
+        "邳州": "江苏",
+        "常州": "江苏",
+        "溧阳": "江苏",
+        "金坛": "江苏",
+        "苏州": "江苏",
+        "常熟": "江苏",
+        "张家港": "江苏",
+        "昆山": "江苏",
+        "吴江": "江苏",
+        "太仓": "江苏",
+        "南通": "江苏",
+        "海安": "江苏",
+        "如东": "江苏",
+        "启东": "江苏",
+        "如皋": "江苏",
+        "通州": "江苏",
+        "海门": "江苏",
+        "连云港": "江苏",
+        "赣榆": "江苏",
+        "东海": "江苏",
+        "灌云": "江苏",
+        "灌南": "江苏",
+        "淮安": "江苏",
+        "涟水": "江苏",
+        "洪泽区": "江苏",
+        "洪泽县": "江苏",
+        "盱眙": "江苏",
+        "金湖": "江苏",
+        "盐城": "江苏",
+        "响水": "江苏",
+        "滨海县": "江苏",
+        "阜宁": "江苏",
+        "射阳": "江苏",
+        "建湖": "江苏",
+        "东台": "江苏",
+        "大丰": "江苏",
+        "扬州": "江苏",
+        "宝应": "江苏",
+        "仪征": "江苏",
+        "高邮": "江苏",
+        "江都": "江苏",
+        "镇江": "江苏",
+        "丹阳": "江苏",
+        "扬中": "江苏",
+        "句容": "江苏",
+        "泰州": "江苏",
+        "兴化": "江苏",
+        "靖江": "江苏",
+        "泰兴市": "江苏",
+        "泰兴镇": "四川",
+        "姜堰": "江苏",
+        "宿迁": "江苏",
+        "沭阳": "江苏",
+        "泗阳": "江苏",
+        "泗洪": "江苏",
+        "浙江": "浙江",
+        "杭州": "浙江",
+        "桐庐": "浙江",
+        "淳安": "浙江",
+        "建德": "浙江",
+        "富阳区": "浙江",
+        "临安": "浙江",
+        "宁波": "浙江",
+        "象山": "浙江",
+        "宁海": "浙江",
+        "余姚": "浙江",
+        "慈溪": "浙江",
+        "奉化": "浙江",
+        "温州": "浙江",
+        "洞头": "浙江",
+        "永嘉": "浙江",
+        "平阳": "浙江",
+        "苍南": "浙江",
+        "文成县": "浙江",
+        "泰顺": "浙江",
+        "瑞安": "浙江",
+        "乐清": "浙江",
+        "嘉兴": "浙江",
+        "嘉善": "浙江",
+        "海盐": "浙江",
+        "海宁": "浙江",
+        "平湖": "浙江",
+        "桐乡": "浙江",
+        "湖州": "浙江",
+        "德清": "浙江",
+        "长兴": "浙江",
+        "安吉": "浙江",
+        "绍兴": "浙江",
+        "新昌": "浙江",
+        "诸暨": "浙江",
+        "上虞": "浙江",
+        "嵊州": "浙江",
+        "金华": "浙江",
+        "武义": "浙江",
+        "浦江": "浙江",
+        "磐安": "浙江",
+        "兰溪": "浙江",
+        "义乌": "浙江",
+        "东阳": "浙江",
+        "永康": "浙江",
+        "衢州": "浙江",
+        "常山": "浙江",
+        "开化": "浙江",
+        "龙游": "浙江",
+        "江山市": "浙江",
+        "舟山": "浙江",
+        "岱山": "浙江",
+        "嵊泗": "浙江",
+        "台州": "浙江",
+        "玉环": "浙江",
+        "三门县": "浙江",
+        "天台县": "浙江",
+        "仙居": "浙江",
+        "温岭": "浙江",
+        "临海": "浙江",
+        "丽水": "浙江",
+        "青田": "浙江",
+        "缙云": "浙江",
+        "遂昌": "浙江",
+        "松阳": "浙江",
+        "云和": "浙江",
+        "庆元": "浙江",
+        "景宁": "浙江",
+        "龙泉市": "浙江",
+        "安徽": "安徽",
+        "合肥": "安徽",
+        "长丰": "安徽",
+        "肥东": "安徽",
+        "肥西": "安徽",
+        "芜湖": "安徽",
+        "繁昌": "安徽",
+        "南陵": "安徽",
+        "蚌埠": "安徽",
+        "怀远": "安徽",
+        "五河": "安徽",
+        "固镇": "安徽",
+        "淮南": "安徽",
+        "凤台": "安徽",
+        "马鞍山": "安徽",
+        "当涂": "安徽",
+        "含山": "安徽",
+        "和县": "安徽",
+        "淮北": "安徽",
+        "濉溪": "安徽",
+        "铜陵": "安徽",
+        "安庆": "安徽",
+        "怀宁": "安徽",
+        "枞阳": "安徽",
+        "潜山": "安徽",
+        "太湖县": "安徽",
+        "宿松": "安徽",
+        "望江": "安徽",
+        "岳西": "安徽",
+        "桐城": "安徽",
+        "黄山": "安徽",
+        "歙县": "安徽",
+        "休宁": "安徽",
+        "黟县": "安徽",
+        "祁门": "安徽",
+        "滁州": "安徽",
+        "来安": "安徽",
+        "全椒": "安徽",
+        "定远": "安徽",
+        "凤阳": "安徽",
+        "天长市": "安徽",
+        "明光市": "安徽",
+        "阜阳": "安徽",
+        "临泉": "安徽",
+        "太和县": "安徽",
+        "阜南": "安徽",
+        "颍上": "安徽",
+        "界首": "安徽",
+        "宿州": "安徽",
+        "砀山": "安徽",
+        "萧县": "安徽",
+        "灵璧": "安徽",
+        "泗县": "安徽",
+        "巢湖": "安徽",
+        "庐江": "安徽",
+        "无为县": "安徽",
+        "六安": "安徽",
+        "寿县": "安徽",
+        "霍邱": "安徽",
+        "舒城": "安徽",
+        "金寨": "安徽",
+        "霍山": "安徽",
+        "亳州": "安徽",
+        "涡阳": "安徽",
+        "蒙城": "安徽",
+        "利辛": "安徽",
+        "池州": "安徽",
+        "东至县": "安徽",
+        "石台": "安徽",
+        "青阳": "安徽",
+        "宣城": "安徽",
+        "郎溪": "安徽",
+        "广德": "安徽",
+        "泾县": "安徽",
+        "绩溪": "安徽",
+        "旌德": "安徽",
+        "宁国市": "安徽",
+        "福建": "福建",
+        "福州": "福建",
+        "闽侯": "福建",
+        "连江": "福建",
+        "罗源": "福建",
+        "闽清": "福建",
+        "永泰县": "福建",
+        "平潭": "福建",
+        "福清": "福建",
+        "长乐": "福建",
+        "厦门": "福建",
+        "莆田": "福建",
+        "仙游": "福建",
+        "三明": "福建",
+        "明溪": "福建",
+        "清流": "福建",
+        "宁化": "福建",
+        "大田": "福建",
+        "尤溪": "福建",
+        "沙县": "福建",
+        "将乐": "福建",
+        "泰宁": "福建",
+        "建宁": "福建",
+        "永安": "福建",
+        "泉州": "福建",
+        "惠安县": "福建",
+        "安溪": "福建",
+        "永春": "福建",
+        "德化": "福建",
+        "金门": "福建",
+        "石狮": "福建",
+        "晋江": "福建",
+        "南安": "福建",
+        "漳州": "福建",
+        "云霄": "福建",
+        "漳浦": "福建",
+        "诏安": "福建",
+        "长泰": "福建",
+        "东山县": "福建",
+        "南靖": "福建",
+        "平和县": "福建",
+        "华安": "福建",
+        "龙海": "福建",
+        "南平": "福建",
+        "顺昌": "福建",
+        "浦城": "福建",
+        "光泽": "福建",
+        "松溪": "福建",
+        "政和县": "福建",
+        "邵武": "福建",
+        "武夷山": "福建",
+        "建瓯": "福建",
+        "建阳": "福建",
+        "龙岩": "福建",
+        "长汀": "福建",
+        "永定": "福建",
+        "上杭": "福建",
+        "武平": "福建",
+        "连城": "福建",
+        "漳平": "福建",
+        "宁德": "福建",
+        "霞浦": "福建",
+        "古田": "福建",
+        "屏南": "福建",
+        "寿宁": "福建",
+        "周宁": "福建",
+        "柘荣": "福建",
+        "福安": "福建",
+        "福鼎": "福建",
+        "江西": "江西",
+        "南昌": "江西",
+        "新建区": "江西",
+        "安义": "江西",
+        "进贤": "江西",
+        "景德镇": "江西",
+        "浮梁": "江西",
+        "乐平市": "江西",
+        "萍乡": "江西",
+        "莲花": "江西",
+        "上栗": "江西",
+        "芦溪": "江西",
+        "九江市": "江西",
+        "武宁": "江西",
+        "修水县": "江西",
+        "永修": "江西",
+        "德安": "江西",
+        "星子县": "江西",
+        "都昌": "江西",
+        "湖口县": "江西",
+        "彭泽": "江西",
+        "瑞昌": "江西",
+        "新余": "江西",
+        "分宜": "江西",
+        "鹰潭": "江西",
+        "余江": "江西",
+        "贵溪": "江西",
+        "赣州": "江西",
+        "赣县": "江西",
+        "信丰": "江西",
+        "大余": "江西",
+        "上犹": "江西",
+        "崇义": "江西",
+        "安远": "江西",
+        "龙南县": "江西",
+        "定南": "江西",
+        "全南": "江西",
+        "宁都": "江西",
+        "于都": "江西",
+        "兴国县": "江西",
+        "会昌": "江西",
+        "寻乌": "江西",
+        "石城": "江西",
+        "瑞金": "江西",
+        "南康": "江西",
+        "吉安": "江西",
+        "吉水": "江西",
+        "峡江": "江西",
+        "新干": "江西",
+        "永丰": "江西",
+        "泰和县": "江西",
+        "遂川": "江西",
+        "万安县": "江西",
+        "安福县": "江西",
+        "永新县": "江西",
+        "井冈山": "江西",
+        "宜春": "江西",
+        "奉新": "江西",
+        "万载县": "江西",
+        "上高": "江西",
+        "宜丰": "江西",
+        "靖安": "江西",
+        "铜鼓": "江西",
+        "丰城": "江西",
+        "樟树": "江西",
+        "高安": "江西",
+        "抚州": "江西",
+        "南城": "江西",
+        "黎川": "江西",
+        "南丰": "江西",
+        "崇仁": "江西",
+        "乐安": "江西",
+        "宜黄": "江西",
+        "金溪": "江西",
+        "资溪": "江西",
+        "东乡县": "江西",
+        "广昌": "江西",
+        "上饶": "江西",
+        "广丰": "江西",
+        "玉山": "江西",
+        "铅山": "江西",
+        "横峰": "江西",
+        "弋阳": "江西",
+        "余干": "江西",
+        "鄱阳": "江西",
+        "万年县": "江西",
+        "婺源": "江西",
+        "德兴": "江西",
+        "山东": "山东",
+        "济南": "山东",
+        "平阴": "山东",
+        "济阳": "山东",
+        "商河县": "山东",
+        "章丘": "山东",
+        "青岛": "山东",
+        "胶州": "山东",
+        "即墨": "山东",
+        "平度市": "山东",
+        "胶南": "山东",
+        "莱西": "山东",
+        "淄博": "山东",
+        "桓台": "山东",
+        "高青": "山东",
+        "沂源": "山东",
+        "枣庄": "山东",
+        "滕州": "山东",
+        "东营市": "山东",
+        "东营区": "山东",
+        "垦利": "山东",
+        "利津": "山东",
+        "广饶": "山东",
+        "烟台": "山东",
+        "长岛县": "山东",
+        "龙口市": "山东",
+        "莱阳": "山东",
+        "莱州": "山东",
+        "蓬莱": "山东",
+        "招远": "山东",
+        "栖霞市": "山东",
+        "海阳": "山东",
+        "潍坊": "山东",
+        "临朐": "山东",
+        "昌乐": "山东",
+        "青州": "山东",
+        "诸城": "山东",
+        "寿光": "山东",
+        "安丘": "山东",
+        "高密": "山东",
+        "昌邑市": "山东",
+        "济宁": "山东",
+        "微山": "山东",
+        "鱼台": "山东",
+        "金乡": "山东",
+        "嘉祥": "山东",
+        "汶上": "山东",
+        "泗水": "山东",
+        "梁山": "山东",
+        "曲阜": "山东",
+        "兖州": "山东",
+        "邹城": "山东",
+        "泰安": "山东",
+        "宁阳": "山东",
+        "东平": "山东",
+        "新泰": "山东",
+        "肥城": "山东",
+        "威海": "山东",
+        "文登": "山东",
+        "荣成": "山东",
+        "乳山": "山东",
+        "日照": "山东",
+        "五莲": "山东",
+        "莒县": "山东",
+        "莱芜": "山东",
+        "临沂": "山东",
+        "沂南": "山东",
+        "郯城": "山东",
+        "沂水": "山东",
+        "苍山": "山东",
+        "费县": "山东",
+        "平邑": "山东",
+        "莒南": "山东",
+        "蒙阴": "山东",
+        "临沭": "山东",
+        "德州": "山东",
+        "陵县": "山东",
+        "宁津": "山东",
+        "庆云": "山东",
+        "临邑": "山东",
+        "齐河": "山东",
+        "平原": "山东",
+        "夏津": "山东",
+        "武城": "山东",
+        "乐陵": "山东",
+        "禹城": "山东",
+        "聊城": "山东",
+        "阳谷": "山东",
+        "莘县": "山东",
+        "茌平": "山东",
+        "东阿": "山东",
+        "冠县": "山东",
+        "高唐": "山东",
+        "临清": "山东",
+        "滨州": "山东",
+        "惠民": "山东",
+        "阳信": "山东",
+        "无棣": "山东",
+        "沾化": "山东",
+        "博兴": "山东",
+        "邹平": "山东",
+        "菏泽": "山东",
+        "曹县": "山东",
+        "单县": "山东",
+        "成武": "山东",
+        "巨野": "山东",
+        "郓城": "山东",
+        "鄄城": "山东",
+        "定陶": "山东",
+        "东明县": "山东",
+        "河南": "河南",
+        "郑州": "河南",
+        "管城": "河南",
+        "中牟": "河南",
+        "巩义": "河南",
+        "荥阳": "河南",
+        "新密": "河南",
+        "新郑": "河南",
+        "登封": "河南",
+        "开封": "河南",
+        "杞县": "河南",
+        "通许": "河南",
+        "尉氏": "河南",
+        "兰考": "河南",
+        "洛阳": "河南",
+        "孟津": "河南",
+        "新安": "河南",
+        "栾川": "河南",
+        "嵩县": "河南",
+        "汝阳": "河南",
+        "宜阳": "河南",
+        "洛宁": "河南",
+        "伊川": "河南",
+        "偃师": "河南",
+        "平顶山": "河南",
+        "宝丰": "河南",
+        "叶县": "河南",
+        "鲁山": "河南",
+        "郏县": "河南",
+        "舞钢": "河南",
+        "汝州": "河南",
+        "安阳": "河南",
+        "汤阴": "河南",
+        "滑县": "河南",
+        "内黄": "河南",
+        "林州": "河南",
+        "鹤壁": "河南",
+        "浚县": "河南",
+        "淇县": "河南",
+        "新乡": "河南",
+        "获嘉": "河南",
+        "原阳": "河南",
+        "延津": "河南",
+        "封丘": "河南",
+        "长垣": "河南",
+        "卫辉": "河南",
+        "辉市": "河南",
+        "焦作": "河南",
+        "修武": "河南",
+        "博爱": "河南",
+        "武陟": "河南",
+        "温县": "河南",
+        "沁阳": "河南",
+        "孟州": "河南",
+        "濮阳": "河南",
+        "清丰": "河南",
+        "南乐": "河南",
+        "范县": "河南",
+        "台前": "河南",
+        "许昌": "河南",
+        "鄢陵": "河南",
+        "襄城县": "河南",
+        "禹州": "河南",
+        "长葛": "河南",
+        "漯河": "河南",
+        "舞阳": "河南",
+        "临颍": "河南",
+        "三门峡": "河南",
+        "渑池": "河南",
+        "陕县": "河南",
+        "卢氏": "河南",
+        "义马": "河南",
+        "灵宝": "河南",
+        "南阳": "河南",
+        "南召": "河南",
+        "方城": "河南",
+        "西峡": "河南",
+        "镇平县": "河南",
+        "内乡": "河南",
+        "淅川": "河南",
+        "社旗": "河南",
+        "唐河": "河南",
+        "新野": "河南",
+        "桐柏": "河南",
+        "邓州": "河南",
+        "商丘": "河南",
+        "民权": "河南",
+        "睢县": "河南",
+        "宁陵": "河南",
+        "柘城": "河南",
+        "虞城": "河南",
+        "夏邑": "河南",
+        "永城": "河南",
+        "信阳": "河南",
+        "罗山": "河南",
+        "光山": "河南",
+        "新县": "河南",
+        "商城": "河南",
+        "固始": "河南",
+        "潢川": "河南",
+        "淮滨": "河南",
+        "息县": "河南",
+        "周口": "河南",
+        "扶沟": "河南",
+        "西华": "河南",
+        "商水": "河南",
+        "沈丘": "河南",
+        "郸城": "河南",
+        "淮阳": "河南",
+        "太康": "河南",
+        "鹿邑": "河南",
+        "项城": "河南",
+        "驻马店": "河南",
+        "西平": "河南",
+        "上蔡": "河南",
+        "平舆": "河南",
+        "正阳": "河南",
+        "确山": "河南",
+        "泌阳": "河南",
+        "汝南": "河南",
+        "遂平": "河南",
+        "新蔡": "河南",
+        "济源": "河南",
+        "湖北": "湖北",
+        "武汉": "湖北",
+        "黄石市": "湖北",
+        "阳新县": "湖北",
+        "大冶": "湖北",
+        "十堰": "湖北",
+        "郧县": "湖北",
+        "郧西": "湖北",
+        "竹山": "湖北",
+        "竹溪": "湖北",
+        "房县": "湖北",
+        "丹江口": "湖北",
+        "宜昌": "湖北",
+        "远安": "湖北",
+        "兴山": "湖北",
+        "秭归": "湖北",
+        "长阳": "湖北",
+        "五峰": "湖北",
+        "宜都": "湖北",
+        "当阳": "湖北",
+        "枝江": "湖北",
+        "襄阳": "湖北",
+        "南漳": "湖北",
+        "谷城": "湖北",
+        "保康": "湖北",
+        "老河口": "湖北",
+        "枣阳": "湖北",
+        "宜城": "湖北",
+        "鄂州": "湖北",
+        "荆门": "湖北",
+        "京山县": "湖北",
+        "沙洋": "湖北",
+        "钟祥": "湖北",
+        "屈家岭": "湖北",
+        "孝感": "湖北",
+        "孝昌": "湖北",
+        "大悟": "湖北",
+        "云梦": "湖北",
+        "应城": "湖北",
+        "安陆": "湖北",
+        "汉川": "湖北",
+        "荆州": "湖北",
+        "公安县": "湖北",
+        "监利": "湖北",
+        "江陵": "湖北",
+        "石首": "湖北",
+        "洪湖": "湖北",
+        "松滋": "湖北",
+        "黄冈市": "湖北",
+        "团风县": "湖北",
+        "红安": "湖北",
+        "罗田": "湖北",
+        "英山": "湖北",
+        "浠水": "湖北",
+        "蕲春": "湖北",
+        "黄梅": "湖北",
+        "麻城": "湖北",
+        "武穴": "湖北",
+        "咸宁": "湖北",
+        "嘉鱼": "湖北",
+        "通城": "湖北",
+        "崇阳": "湖北",
+        "通山": "湖北",
+        "赤壁": "湖北",
+        "随州": "湖北",
+        "随县": "湖北",
+        "广水": "湖北",
+        "恩施州": "湖北",
+        "恩施": "湖北",
+        "利川": "湖北",
+        "建始县": "湖北",
+        "巴东": "湖北",
+        "宣恩": "湖北",
+        "咸丰": "湖北",
+        "来凤": "湖北",
+        "鹤峰": "湖北",
+        "仙桃": "湖北",
+        "潜江": "湖北",
+        "天门市": "湖北",
+        "神农架": "湖北",
+        "江汉油田": "湖北",
+        "湖南": "湖南",
+        "长沙": "湖南",
+        "望城": "湖南",
+        "宁乡": "湖南",
+        "浏阳": "湖南",
+        "株洲": "湖南",
+        "攸县": "湖南",
+        "茶陵": "湖南",
+        "炎陵": "湖南",
+        "醴陵": "湖南",
+        "湘潭": "湖南",
+        "湘乡": "湖南",
+        "韶山": "湖南",
+        "衡阳": "湖南",
+        "衡南": "湖南",
+        "衡山": "湖南",
+        "衡东": "湖南",
+        "祁东": "湖南",
+        "耒阳": "湖南",
+        "常宁": "湖南",
+        "邵阳": "湖南",
+        "邵东": "湖南",
+        "新邵": "湖南",
+        "隆回": "湖南",
+        "洞口县": "湖南",
+        "绥宁": "湖南",
+        "新宁县": "湖南",
+        "城步": "湖南",
+        "武冈": "湖南",
+        "岳阳": "湖南",
+        "华容县": "湖南",
+        "湘阴": "湖南",
+        "平江": "湖南",
+        "汨罗": "湖南",
+        "临湘": "湖南",
+        "常德": "湖南",
+        "安乡县": "湖南",
+        "汉寿": "湖南",
+        "澧县": "湖南",
+        "临澧": "湖南",
+        "桃源县": "湖南",
+        "石门": "湖南",
+        "津市市": "湖南",
+        "张家界": "湖南",
+        "慈利": "湖南",
+        "桑植": "湖南",
+        "益阳": "湖南",
+        "南县": "湖南",
+        "桃江": "湖南",
+        "安化": "湖南",
+        "沅江": "湖南",
+        "郴州": "湖南",
+        "桂阳": "湖南",
+        "宜章": "湖南",
+        "永兴": "湖南",
+        "嘉禾": "湖南",
+        "临武": "湖南",
+        "汝城": "湖南",
+        "桂东": "湖南",
+        "安仁": "湖南",
+        "资兴": "湖南",
+        "永州": "湖南",
+        "祁阳": "湖南",
+        "东安县": "湖南",
+        "双牌": "湖南",
+        "道县": "湖南",
+        "江永": "湖南",
+        "宁远": "湖南",
+        "蓝山": "湖南",
+        "新田": "湖南",
+        "江华": "湖南",
+        "怀化": "湖南",
+        "中方县": "湖南",
+        "沅陵": "湖南",
+        "辰溪": "湖南",
+        "溆浦": "湖南",
+        "会同": "湖南",
+        "麻阳": "湖南",
+        "新晃": "湖南",
+        "芷江": "湖南",
+        "靖州": "湖南",
+        "通道县": "湖南",
+        "通道侗族自治县": "湖南",
+        "洪江": "湖南",
+        "娄底": "湖南",
+        "双峰": "湖南",
+        "新化": "湖南",
+        "冷水江": "湖南",
+        "涟源": "湖南",
+        "湘西": "湖南",
+        "吉首": "湖南",
+        "泸溪": "湖南",
+        "凤凰县": "湖南",
+        "花垣": "湖南",
+        "保靖": "湖南",
+        "古丈": "湖南",
+        "永顺": "湖南",
+        "龙山县": "湖南",
+        "广东": "广东",
+        "广州": "广东",
+        "增城": "广东",
+        "从化": "广东",
+        "韶关": "广东",
+        "始兴": "广东",
+        "仁化": "广东",
+        "翁源": "广东",
+        "乳源": "广东",
+        "新丰": "广东",
+        "乐昌": "广东",
+        "南雄": "广东",
+        "深圳": "广东",
+        "珠海": "广东",
+        "汕头": "广东",
+        "南澳": "广东",
+        "佛山": "广东",
+        "江门市": "广东",
+        "台山市": "广东",
+        "开平市": "广东",
+        "鹤山市": "广东",
+        "恩平": "广东",
+        "湛江": "广东",
+        "遂溪": "广东",
+        "徐闻": "广东",
+        "廉江": "广东",
+        "雷州": "广东",
+        "吴川": "广东",
+        "茂名": "广东",
+        "电白": "广东",
+        "高州": "广东",
+        "化州": "广东",
+        "信宜": "广东",
+        "肇庆": "广东",
+        "广宁": "广东",
+        "怀集": "广东",
+        "封开": "广东",
+        "德庆": "广东",
+        "高要区": "广东",
+        "四会市": "广东",
+        "惠州": "广东",
+        "博罗": "广东",
+        "惠东": "广东",
+        "龙门县": "广东",
+        "梅州": "广东",
+        "梅县": "广东",
+        "大埔": "广东",
+        "丰顺": "广东",
+        "五华县": "广东",
+        "平远": "广东",
+        "蕉岭": "广东",
+        "兴宁市": "广东",
+        "汕尾": "广东",
+        "海丰": "广东",
+        "陆河": "广东",
+        "陆丰": "广东",
+        "河源": "广东",
+        "紫金县": "广东",
+        "龙川": "广东",
+        "连平": "广东",
+        "和平县": "广东",
+        "东源": "广东",
+        "阳江市": "广东",
+        "阳西": "广东",
+        "阳东": "广东",
+        "阳春": "广东",
+        "清远": "广东",
+        "佛冈": "广东",
+        "阳山": "广东",
+        "连山县": "广东",
+        "连山壮族瑶族自治县": "广东",
+        "连南": "广东",
+        "清新区": "广东",
+        "英德": "广东",
+        "连州": "广东",
+        "东莞": "广东",
+        "中山市": "广东",
+        "潮州": "广东",
+        "潮安": "广东",
+        "饶平": "广东",
+        "揭阳": "广东",
+        "揭东": "广东",
+        "揭西": "广东",
+        "惠来": "广东",
+        "普宁": "广东",
+        "云浮": "广东",
+        "新兴": "广东",
+        "郁南": "广东",
+        "云安": "广东",
+        "罗定": "广东",
+        "广西": "广西",
+        "南宁": "广西",
+        "武鸣": "广西",
+        "隆安": "广西",
+        "马山": "广西",
+        "上林": "广西",
+        "宾阳": "广西",
+        "横县": "广西",
+        "柳州": "广西",
+        "柳江": "广西",
+        "柳城": "广西",
+        "鹿寨": "广西",
+        "融安": "广西",
+        "融水": "广西",
+        "三江侗族自治县": "广西",
+        "三江县": "广西",
+        "桂林": "广西",
+        "阳朔": "广西",
+        "临桂": "广西",
+        "灵川": "广西",
+        "全州": "广西",
+        "兴安": "广西",
+        "永福": "广西",
+        "灌阳": "广西",
+        "龙胜": "广西",
+        "资源县": "广西",
+        "平乐": "广西",
+        "荔蒲": "广西",
+        "恭城": "广西",
+        "梧州": "广西",
+        "苍梧": "广西",
+        "藤县": "广西",
+        "蒙山": "广西",
+        "岑溪": "广西",
+        "北海市": "广西",
+        "北海经济开发区": "山东",
+        "合浦": "广西",
+        "防城港": "广西",
+        "上思": "广西",
+        "东兴市": "广西",
+        "钦州": "广西",
+        "灵山县": "广西",
+        "浦北": "广西",
+        "贵港": "广西",
+        "平南": "广西",
+        "桂平": "广西",
+        "玉林": "广西",
+        "容县": "广西",
+        "陆川": "广西",
+        "博白": "广西",
+        "兴业县": "广西",
+        "北流": "广西",
+        "百色": "广西",
+        "田阳": "广西",
+        "田东": "广西",
+        "平果县": "广西",
+        "德保": "广西",
+        "靖西": "广西",
+        "那坡": "广西",
+        "凌云县": "广西",
+        "乐业县": "广西",
+        "田林县": "广西",
+        "西林": "广西",
+        "隆林": "广西",
+        "贺州": "广西",
+        "昭平": "广西",
+        "钟山县": "广西",
+        "富川": "广西",
+        "河池": "广西",
+        "南丹": "广西",
+        "天峨": "广西",
+        "凤山": "广西",
+        "东兰": "广西",
+        "罗城": "广西",
+        "环江": "广西",
+        "巴马": "广西",
+        "都安": "广西",
+        "大化": "广西",
+        "宜州": "广西",
+        "来宾": "广西",
+        "忻城": "广西",
+        "象州": "广西",
+        "武宣": "广西",
+        "金秀": "广西",
+        "合山": "广西",
+        "崇左": "广西",
+        "扶绥": "广西",
+        "宁明": "广西",
+        "龙州": "广西",
+        "大新县": "广西",
+        "天等": "广西",
+        "凭祥": "广西",
+        "海南": "海南",
+        "三沙": "海南",
+        "海口": "海南",
+        "三亚": "海南",
+        "五指山": "海南",
+        "琼海": "海南",
+        "儋州": "海南",
+        "文昌": "海南",
+        "万宁": "海南",
+        "东方市": "海南",
+        "定安": "海南",
+        "屯昌": "海南",
+        "澄迈": "海南",
+        "临高": "海南",
+        "白沙县": "海南",
+        "白沙黎族自治县": "海南",
+        "昌江自治县": "海南",
+        "乐东": "海南",
+        "陵水": "海南",
+        "保亭": "海南",
+        "琼中": "海南",
+        "重庆": "重庆",
+        "綦江": "重庆",
+        "潼南": "重庆",
+        "铜梁": "重庆",
+        "大足": "重庆",
+        "荣昌": "重庆",
+        "璧山": "重庆",
+        "梁平": "重庆",
+        "城口县": "重庆",
+        "丰都": "重庆",
+        "垫江": "重庆",
+        "武隆": "重庆",
+        "忠县": "重庆",
+        "开县": "重庆",
+        "云阳": "重庆",
+        "奉节": "重庆",
+        "巫山": "重庆",
+        "巫溪": "重庆",
+        "石柱土家族自治": "重庆",
+        "秀山土家族苗族自治": "重庆",
+        "酉阳土家族苗族自治": "重庆",
+        "彭水苗族土家族自治": "重庆",
+        "四川": "四川",
+        "成都": "四川",
+        "金堂": "四川",
+        "双流": "四川",
+        "郫县": "四川",
+        "大邑": "四川",
+        "蒲江": "四川",
+        "新津": "四川",
+        "都江堰": "四川",
+        "彭州": "四川",
+        "邛崃": "四川",
+        "崇州": "四川",
+        "自贡": "四川",
+        "荣县": "四川",
+        "富顺": "四川",
+        "攀枝花": "四川",
+        "米易": "四川",
+        "盐边": "四川",
+        "泸州": "四川",
+        "泸县": "四川",
+        "合江": "四川",
+        "叙永": "四川",
+        "古蔺": "四川",
+        "南部县": "四川",
+        "德阳": "四川",
+        "中江": "四川",
+        "罗江": "四川",
+        "广汉": "四川",
+        "什邡": "四川",
+        "绵竹": "四川",
+        "绵阳": "四川",
+        "三台县": "四川",
+        "盐亭": "四川",
+        "安县": "四川",
+        "梓潼": "四川",
+        "北川羌族自治": "四川",
+        "平武": "四川",
+        "江油": "四川",
+        "广元": "四川",
+        "旺苍": "四川",
+        "青川": "四川",
+        "剑阁": "四川",
+        "苍溪": "四川",
+        "遂宁": "四川",
+        "蓬溪": "四川",
+        "射洪": "四川",
+        "大英": "四川",
+        "内江": "四川",
+        "威远": "四川",
+        "资中": "四川",
+        "隆昌": "四川",
+        "乐山": "四川",
+        "犍为": "四川",
+        "井研": "四川",
+        "夹江": "四川",
+        "沐川": "四川",
+        "峨边彝族自治": "四川",
+        "马边彝族自治": "四川",
+        "峨眉山": "四川",
+        "南充": "四川",
+        "营山": "四川",
+        "蓬安": "四川",
+        "仪陇": "四川",
+        "西充": "四川",
+        "阆中": "四川",
+        "眉山": "四川",
+        "仁寿": "四川",
+        "彭山": "四川",
+        "洪雅": "四川",
+        "丹棱": "四川",
+        "青神": "四川",
+        "宜宾": "四川",
+        "南溪": "四川",
+        "江安": "四川",
+        "长宁县": "四川",
+        "高县": "四川",
+        "珙县": "四川",
+        "筠连": "四川",
+        "兴文": "四川",
+        "屏山": "四川",
+        "广安": "四川",
+        "岳池": "四川",
+        "武胜": "四川",
+        "邻水": "四川",
+        "华蓥": "四川",
+        "达州": "四川",
+        "达县": "四川",
+        "宣汉": "四川",
+        "开江县": "四川",
+        "大竹县": "四川",
+        "渠县": "四川",
+        "万源市": "四川",
+        "雅安市": "四川",
+        "名山县": "四川",
+        "荥经": "四川",
+        "汉源": "四川",
+        "石棉县": "四川",
+        "天全": "四川",
+        "芦山": "四川",
+        "宝兴": "四川",
+        "巴中": "四川",
+        "通江": "四川",
+        "南江县": "四川",
+        "平昌": "四川",
+        "资阳": "四川",
+        "安岳": "四川",
+        "乐至县": "四川",
+        "简阳": "四川",
+        "阿坝州": "四川",
+        "汶川": "四川",
+        "理县": "四川",
+        "茂县": "四川",
+        "松潘": "四川",
+        "九寨沟": "四川",
+        "金川县": "四川",
+        "小金县": "四川",
+        "黑水县": "四川",
+        "马尔康": "四川",
+        "壤塘": "四川",
+        "阿坝": "四川",
+        "若尔盖": "四川",
+        "红原": "四川",
+        "甘孜": "四川",
+        "康定": "四川",
+        "泸定": "四川",
+        "丹巴": "四川",
+        "九龙县": "四川",
+        "雅江": "四川",
+        "道孚": "四川",
+        "炉霍": "四川",
+        "新龙": "四川",
+        "德格": "四川",
+        "白玉县": "四川",
+        "石渠": "四川",
+        "色达": "四川",
+        "理塘": "四川",
+        "巴塘": "四川",
+        "乡城县": "四川",
+        "稻城": "四川",
+        "得荣": "四川",
+        "凉山": "四川",
+        "西昌": "四川",
+        "木里": "四川",
+        "盐源": "四川",
+        "德昌": "四川",
+        "会理": "四川",
+        "会东": "四川",
+        "宁南": "四川",
+        "普格": "四川",
+        "布拖": "四川",
+        "金阳": "四川",
+        "昭觉": "四川",
+        "喜德": "四川",
+        "冕宁": "四川",
+        "越西": "四川",
+        "甘洛": "四川",
+        "美姑": "四川",
+        "雷波": "四川",
+        "贵州": "贵州",
+        "贵阳": "贵州",
+        "开阳": "贵州",
+        "息烽": "贵州",
+        "修文": "贵州",
+        "清镇": "贵州",
+        "六盘水": "贵州",
+        "水城": "贵州",
+        "盘县": "贵州",
+        "遵义": "贵州",
+        "桐梓": "贵州",
+        "绥阳": "贵州",
+        "正安": "贵州",
+        "道真": "贵州",
+        "务川": "贵州",
+        "凤冈": "贵州",
+        "湄潭": "贵州",
+        "余庆": "贵州",
+        "习水": "贵州",
+        "赤水": "贵州",
+        "仁怀": "贵州",
+        "安顺": "贵州",
+        "平坝": "贵州",
+        "普定": "贵州",
+        "镇宁": "贵州",
+        "关岭": "贵州",
+        "紫云县": "贵州",
+        "紫云苗族布依族自治县": "贵州",
+        "毕节": "贵州",
+        "大方县": "贵州",
+        "黔西": "贵州",
+        "金沙县": "贵州",
+        "织金": "贵州",
+        "纳雍": "贵州",
+        "威宁": "贵州",
+        "赫章": "贵州",
+        "铜仁": "贵州",
+        "江口县": "贵州",
+        "玉屏": "贵州",
+        "石阡": "贵州",
+        "思南": "贵州",
+        "印江": "贵州",
+        "德江": "贵州",
+        "沿河": "贵州",
+        "松桃": "贵州",
+        "黔西南": "贵州",
+        "兴义": "贵州",
+        "兴仁": "贵州",
+        "普安": "贵州",
+        "晴隆": "贵州",
+        "贞丰": "贵州",
+        "望谟": "贵州",
+        "册亨": "贵州",
+        "安龙县": "贵州",
+        "黔东南": "贵州",
+        "凯里市": "贵州",
+        "黄平县": "贵州",
+        "施秉": "贵州",
+        "三穗": "贵州",
+        "镇远县": "贵州",
+        "岑巩": "贵州",
+        "天柱": "贵州",
+        "锦屏": "贵州",
+        "剑河": "贵州",
+        "台江县": "贵州",
+        "台江区": "福建",
+        "黎平": "贵州",
+        "榕江": "贵州",
+        "从江": "贵州",
+        "雷山": "贵州",
+        "麻江": "贵州",
+        "丹寨": "贵州",
+        "黔南布": "贵州",
+        "都匀": "贵州",
+        "福泉": "贵州",
+        "荔波": "贵州",
+        "贵定": "贵州",
+        "瓮安": "贵州",
+        "独山县": "贵州",
+        "平塘": "贵州",
+        "罗甸": "贵州",
+        "长顺": "贵州",
+        "龙里": "贵州",
+        "惠水": "贵州",
+        "三都": "贵州",
+        "云南": "云南",
+        "昆明": "云南",
+        "呈贡": "云南",
+        "晋宁": "云南",
+        "富民县": "云南",
+        "宜良": "云南",
+        "石林": "云南",
+        "嵩明": "云南",
+        "禄劝": "云南",
+        "寻甸": "云南",
+        "安宁": "云南",
+        "曲靖": "云南",
+        "马龙": "云南",
+        "陆良": "云南",
+        "师宗": "云南",
+        "罗平县": "云南",
+        "富源": "云南",
+        "会泽": "云南",
+        "沾益": "云南",
+        "宣威": "云南",
+        "玉溪": "云南",
+        "江川": "云南",
+        "澄江": "云南",
+        "通海": "云南",
+        "华宁": "云南",
+        "易门": "云南",
+        "峨山": "云南",
+        "新平": "云南",
+        "元江": "云南",
+        "保山": "云南",
+        "施甸": "云南",
+        "腾冲": "云南",
+        "龙陵": "云南",
+        "昌宁": "云南",
+        "昭通": "云南",
+        "鲁甸": "云南",
+        "巧家": "云南",
+        "盐津": "云南",
+        "大关县": "云南",
+        "永善": "云南",
+        "绥江": "云南",
+        "镇雄": "云南",
+        "彝良": "云南",
+        "威信": "云南",
+        "水富": "云南",
+        "丽江": "云南",
+        "玉龙县": "云南",
+        "玉龙纳西族自治县": "云南",
+        "永胜": "云南",
+        "华坪": "云南",
+        "宁蒗": "云南",
+        "普洱": "云南",
+        "宁洱": "云南",
+        "墨江": "云南",
+        "景东县": "云南",
+        "景东彝族自治县": "云南",
+        "景谷": "云南",
+        "镇沅": "云南",
+        "江城自治县": "云南",
+        "江城县": "云南",
+        "孟连": "云南",
+        "澜沧": "云南",
+        "西盟": "云南",
+        "临沧": "云南",
+        "凤庆": "云南",
+        "云县": "云南",
+        "永德": "云南",
+        "镇康": "云南",
+        "双江": "云南",
+        "耿马": "云南",
+        "沧源": "云南",
+        "楚雄": "云南",
+        "双柏": "云南",
+        "牟定": "云南",
+        "南华": "云南",
+        "姚安": "云南",
+        "大姚": "云南",
+        "永仁": "云南",
+        "元谋": "云南",
+        "武定": "云南",
+        "禄丰": "云南",
+        "红河": "云南",
+        "个旧": "云南",
+        "开远": "云南",
+        "蒙自": "云南",
+        "屏边": "云南",
+        "建水": "云南",
+        "石屏": "云南",
+        "弥勒": "云南",
+        "泸西": "云南",
+        "元阳": "云南",
+        "金平自治县": "云南",
+        "金平县": "云南",
+        "绿春": "云南",
+        "河口自治县": "云南",
+        "文山": "云南",
+        "砚山": "云南",
+        "西畴": "云南",
+        "麻栗坡": "云南",
+        "马关县": "云南",
+        "丘北县": "云南",
+        "广南": "云南",
+        "富宁": "云南",
+        "西双版纳": "云南",
+        "景洪": "云南",
+        "勐海": "云南",
+        "勐腊": "云南",
+        "大理白族自治州": "云南",
+        "大理州": "云南",
+        "大理市": "云南",
+        "漾濞": "云南",
+        "祥云": "云南",
+        "宾川": "云南",
+        "弥渡": "云南",
+        "南涧": "云南",
+        "巍山": "云南",
+        "永平": "云南",
+        "云龙": "云南",
+        "洱源": "云南",
+        "剑川": "云南",
+        "鹤庆": "云南",
+        "德宏": "云南",
+        "瑞丽": "云南",
+        "潞西": "云南",
+        "梁河": "云南",
+        "盈江": "云南",
+        "陇川": "云南",
+        "怒江": "云南",
+        "泸水": "云南",
+        "福贡": "云南",
+        "贡山": "云南",
+        "兰坪": "云南",
+        "迪庆": "云南",
+        "香格里拉": "云南",
+        "德钦": "云南",
+        "维西": "云南",
+        "西藏": "西藏",
+        "拉萨": "西藏",
+        "林周": "西藏",
+        "当雄": "西藏",
+        "尼木": "西藏",
+        "曲水": "西藏",
+        "堆龙德庆": "西藏",
+        "达孜": "西藏",
+        "墨竹工卡": "西藏",
+        "昌都": "西藏",
+        "江达": "西藏",
+        "贡觉": "西藏",
+        "类乌齐": "西藏",
+        "丁青": "西藏",
+        "察雅": "西藏",
+        "八宿": "西藏",
+        "左贡": "西藏",
+        "芒康": "西藏",
+        "洛隆": "西藏",
+        "边坝": "西藏",
+        "乃东": "西藏",
+        "扎囊": "西藏",
+        "贡嘎": "西藏",
+        "桑日": "西藏",
+        "琼结": "西藏",
+        "曲松": "西藏",
+        "措美": "西藏",
+        "洛扎": "西藏",
+        "加查": "西藏",
+        "隆子": "西藏",
+        "错那": "西藏",
+        "浪卡子": "西藏",
+        "日喀则": "西藏",
+        "南木林": "西藏",
+        "江孜": "西藏",
+        "定日": "西藏",
+        "萨迦": "西藏",
+        "拉孜": "西藏",
+        "昂仁": "西藏",
+        "谢通门": "西藏",
+        "白朗": "西藏",
+        "仁布": "西藏",
+        "康马": "西藏",
+        "定结": "西藏",
+        "仲巴": "西藏",
+        "亚东": "西藏",
+        "吉隆": "西藏",
+        "聂拉木": "西藏",
+        "萨嘎": "西藏",
+        "岗巴": "西藏",
+        "那曲": "西藏",
+        "嘉黎": "西藏",
+        "比如县": "西藏",
+        "聂荣县": "西藏",
+        "安多县": "西藏",
+        "申扎": "西藏",
+        "索县": "西藏",
+        "班戈": "西藏",
+        "巴青": "西藏",
+        "尼玛": "西藏",
+        "普兰": "西藏",
+        "札达": "西藏",
+        "噶尔": "西藏",
+        "日土": "西藏",
+        "革吉": "西藏",
+        "改则": "西藏",
+        "措勤": "西藏",
+        "林芝市": "西藏",
+        "林芝县": "西藏",
+        "工布江达": "西藏",
+        "米林": "西藏",
+        "墨脱": "西藏",
+        "波密": "西藏",
+        "察隅": "西藏",
+        "朗县": "西藏",
+        "陕西": "陕西",
+        "西安": "陕西",
+        "蓝田": "陕西",
+        "周至": "陕西",
+        "户县": "陕西",
+        "高陵": "陕西",
+        "铜川": "陕西",
+        "宜君": "陕西",
+        "宝鸡": "陕西",
+        "凤翔县": "陕西",
+        "岐山": "陕西",
+        "扶风": "陕西",
+        "眉县": "陕西",
+        "陇县": "陕西",
+        "千阳": "陕西",
+        "麟游": "陕西",
+        "凤县": "陕西",
+        "太白县": "陕西",
+        "咸阳": "陕西",
+        "三原县": "陕西",
+        "泾阳": "陕西",
+        "乾县": "陕西",
+        "礼泉": "陕西",
+        "永寿": "陕西",
+        "彬县": "陕西",
+        "长武": "陕西",
+        "旬邑": "陕西",
+        "淳化": "陕西",
+        "武功县": "陕西",
+        "兴平": "陕西",
+        "渭南": "陕西",
+        "潼关": "陕西",
+        "大荔": "陕西",
+        "合阳": "陕西",
+        "澄城": "陕西",
+        "蒲城": "陕西",
+        "白水": "陕西",
+        "富平": "陕西",
+        "韩城": "陕西",
+        "华阴": "陕西",
+        "延安": "陕西",
+        "延长县": "陕西",
+        "延川": "陕西",
+        "子长": "陕西",
+        "安塞": "陕西",
+        "志丹": "陕西",
+        "吴起": "陕西",
+        "甘泉": "陕西",
+        "富县": "陕西",
+        "洛川": "陕西",
+        "宜川": "陕西",
+        "黄龙": "陕西",
+        "黄陵": "陕西",
+        "汉中": "陕西",
+        "南郑县": "陕西",
+        "城固": "陕西",
+        "洋县": "陕西",
+        "西乡": "陕西",
+        "勉县": "陕西",
+        "宁强": "陕西",
+        "略阳": "陕西",
+        "镇巴": "陕西",
+        "留坝": "陕西",
+        "佛坪": "陕西",
+        "榆林": "陕西",
+        "神木": "陕西",
+        "府谷": "陕西",
+        "横山": "陕西",
+        "靖边": "陕西",
+        "定边": "陕西",
+        "绥德": "陕西",
+        "米脂": "陕西",
+        "佳县": "陕西",
+        "吴堡": "陕西",
+        "清涧": "陕西",
+        "子洲": "陕西",
+        "安康": "陕西",
+        "汉阴": "陕西",
+        "石泉": "陕西",
+        "宁陕": "陕西",
+        "紫阳": "陕西",
+        "岚皋": "陕西",
+        "平利": "陕西",
+        "镇坪": "陕西",
+        "旬阳": "陕西",
+        "白河县": "陕西",
+        "商洛": "陕西",
+        "洛南": "陕西",
+        "丹凤": "陕西",
+        "商南": "陕西",
+        "山阳": "陕西",
+        "镇安县": "陕西",
+        "柞水": "陕西",
+        "甘肃": "甘肃",
+        "兰州": "甘肃",
+        "东乡自治县": "甘肃",
+        "永登": "甘肃",
+        "皋兰": "甘肃",
+        "榆中": "甘肃",
+        "嘉峪关": "甘肃",
+        "金昌": "甘肃",
+        "永昌": "甘肃",
+        "白银市": "甘肃",
+        "靖远": "甘肃",
+        "会宁": "甘肃",
+        "景泰县": "甘肃",
+        "天水": "甘肃",
+        "清水县": "甘肃",
+        "秦安": "甘肃",
+        "甘谷": "甘肃",
+        "武山": "甘肃",
+        "张家川": "甘肃",
+        "武威": "甘肃",
+        "民勤": "甘肃",
+        "古浪": "甘肃",
+        "天祝": "甘肃",
+        "张掖": "甘肃",
+        "肃南": "甘肃",
+        "民乐县": "甘肃",
+        "临泽": "甘肃",
+        "高台": "甘肃",
+        "山丹": "甘肃",
+        "平凉": "甘肃",
+        "泾川": "甘肃",
+        "灵台": "甘肃",
+        "崇信": "甘肃",
+        "华亭": "甘肃",
+        "庄浪": "甘肃",
+        "静宁": "甘肃",
+        "酒泉": "甘肃",
+        "金塔": "甘肃",
+        "瓜州": "甘肃",
+        "肃北": "甘肃",
+        "阿克塞": "甘肃",
+        "玉门": "甘肃",
+        "敦煌": "甘肃",
+        "庆阳": "甘肃",
+        "庆城": "甘肃",
+        "环县": "甘肃",
+        "华池": "甘肃",
+        "合水县": "甘肃",
+        "正宁": "甘肃",
+        "宁县": "甘肃",
+        "镇原县": "甘肃",
+        "定西": "甘肃",
+        "通渭": "甘肃",
+        "陇西": "甘肃",
+        "渭源": "甘肃",
+        "临洮": "甘肃",
+        "漳县": "甘肃",
+        "岷县": "甘肃",
+        "陇南": "甘肃",
+        "成县": "甘肃",
+        "文县": "甘肃",
+        "宕昌": "甘肃",
+        "康县": "甘肃",
+        "西和": "甘肃",
+        "礼县": "甘肃",
+        "徽县": "甘肃",
+        "两当县": "甘肃",
+        "临夏": "甘肃",
+        "康乐县": "甘肃",
+        "永靖": "甘肃",
+        "广河": "甘肃",
+        "和政": "甘肃",
+        "积石山": "甘肃",
+        "合作市": "甘肃",
+        "临潭": "甘肃",
+        "卓尼": "甘肃",
+        "舟曲": "甘肃",
+        "迭部": "甘肃",
+        "玛曲": "甘肃",
+        "碌曲": "甘肃",
+        "夏河": "甘肃",
+        "青海": "青海",
+        "共和县": "青海",
+        "西宁": "青海",
+        "大通自治县": "青海",
+        "大通县": "青海",
+        "湟中": "青海",
+        "湟源": "青海",
+        "平安县": "青海",
+        "民和": "青海",
+        "乐都": "青海",
+        "河南蒙古族自治县": "青海",
+        "互助土族自治县": "青海",
+        "互助县": "青海",
+        "化隆": "青海",
+        "循化": "青海",
+        "海北州": "青海",
+        "海北藏族自治州": "青海",
+        "门源": "青海",
+        "祁连": "青海",
+        "海晏": "青海",
+        "刚察": "青海",
+        "黄南": "青海",
+        "同仁": "青海",
+        "尖扎": "青海",
+        "泽库": "青海",
+        "同德": "青海",
+        "贵德": "青海",
+        "兴海": "青海",
+        "贵南": "青海",
+        "果洛": "青海",
+        "玛沁": "青海",
+        "班玛": "青海",
+        "甘德": "青海",
+        "达日": "青海",
+        "久治": "青海",
+        "玛多": "青海",
+        "玉树": "青海",
+        "杂多": "青海",
+        "称多": "青海",
+        "治多": "青海",
+        "囊谦": "青海",
+        "曲麻莱": "青海",
+        "海西州": "青海",
+        "海西蒙古族藏族自治州": "青海",
+        "格尔木": "青海",
+        "德令哈": "青海",
+        "乌兰": "青海",
+        "都兰": "青海",
+        "天峻": "青海",
+        "宁夏": "宁夏",
+        "银川": "宁夏",
+        "永宁": "宁夏",
+        "贺兰": "宁夏",
+        "灵武": "宁夏",
+        "石嘴山": "宁夏",
+        "平罗": "宁夏",
+        "吴忠": "宁夏",
+        "盐池": "宁夏",
+        "同心县": "宁夏",
+        "青铜峡": "宁夏",
+        "固原": "宁夏",
+        "西吉": "宁夏",
+        "隆德": "宁夏",
+        "泾源": "宁夏",
+        "彭阳": "宁夏",
+        "中卫": "宁夏",
+        "中宁": "宁夏",
+        "海原": "宁夏",
+        "新疆": "新疆",
+        "乌鲁木齐": "新疆",
+        "克拉玛依": "新疆",
+        "吐鲁番": "新疆",
+        "鄯善": "新疆",
+        "托克逊": "新疆",
+        "哈密": "新疆",
+        "巴里坤": "新疆",
+        "伊吾": "新疆",
+        "昌吉": "新疆",
+        "阜康": "新疆",
+        "呼图壁": "新疆",
+        "玛纳斯": "新疆",
+        "奇台": "新疆",
+        "吉木萨尔": "新疆",
+        "木垒哈萨克": "新疆",
+        "博尔塔拉": "新疆",
+        "博乐": "新疆",
+        "精河": "新疆",
+        "温泉县": "新疆",
+        "巴音郭楞": "新疆",
+        "库尔勒": "新疆",
+        "轮台": "新疆",
+        "尉犁": "新疆",
+        "若羌": "新疆",
+        "且末": "新疆",
+        "焉耆": "新疆",
+        "和静": "新疆",
+        "和硕": "新疆",
+        "博湖": "新疆",
+        "阿克苏": "新疆",
+        "温宿": "新疆",
+        "库车": "新疆",
+        "沙雅": "新疆",
+        "新和县": "新疆",
+        "拜城": "新疆",
+        "乌什": "新疆",
+        "阿瓦提": "新疆",
+        "柯坪": "新疆",
+        "克孜勒": "新疆",
+        "阿图什": "新疆",
+        "阿克陶": "新疆",
+        "阿合奇": "新疆",
+        "乌恰": "新疆",
+        "喀什": "新疆",
+        "疏附": "新疆",
+        "疏勒": "新疆",
+        "英吉沙": "新疆",
+        "泽普": "新疆",
+        "莎车": "新疆",
+        "叶城": "新疆",
+        "麦盖提": "新疆",
+        "岳普湖": "新疆",
+        "伽师": "新疆",
+        "巴楚": "新疆",
+        "塔什": "新疆",
+        "和田": "新疆",
+        "墨玉": "新疆",
+        "皮山": "新疆",
+        "洛浦": "新疆",
+        "策勒": "新疆",
+        "于田县": "新疆",
+        "民丰": "新疆",
+        "伊犁": "新疆",
+        "伊宁": "新疆",
+        "奎屯": "新疆",
+        "察布查尔锡伯": "新疆",
+        "霍城": "新疆",
+        "巩留": "新疆",
+        "新源": "新疆",
+        "昭苏": "新疆",
+        "特克斯": "新疆",
+        "尼勒克": "新疆",
+        "塔城": "新疆",
+        "乌苏": "新疆",
+        "额敏": "新疆",
+        "沙湾": "新疆",
+        "托里": "新疆",
+        "裕民": "新疆",
+        "和布克赛尔": "新疆",
+        "阿勒泰": "新疆",
+        "布尔津": "新疆",
+        "富蕴": "新疆",
+        "福海": "新疆",
+        "哈巴河": "新疆",
+        "青河": "新疆",
+        "吉木乃": "新疆",
+        "石河子": "新疆",
+        "阿拉尔": "新疆",
+        "图木舒克": "新疆",
+        "五家渠": "新疆",
+        "台湾": "台湾",
+        "台北": "台湾",
+        "高雄": "台湾",
+        "基隆": "台湾",
+        "台中市": "台湾",
+        "台南": "台湾",
+        "新竹": "台湾",
+        "嘉义": "台湾",
+        "宜兰": "台湾",
+        "桃园市": "台湾",
+        "苗栗": "台湾",
+        "彰化": "台湾",
+        "南投": "台湾",
+        "云林": "台湾",
+        "屏东": "台湾",
+        "台东": "台湾",
+        "花莲": "台湾",
+        "澎湖": "台湾",
+        "香港": "香港",
+        "澳门": "澳门",
+        "东城区": "北京",
+        "西城区": "北京",
+        "崇文": "北京",
+        "宣武": "北京",
+        "丰台": "北京",
+        "石景山": "北京",
+        "海淀": "北京",
+        "门头沟": "北京",
+        "房山": "北京",
+        "顺义": "北京",
+        "昌平": "北京",
+        "大兴区": "北京",
+        "怀柔": "北京",
+        "平谷": "北京",
+        "河西区": "天津",
+        "滨海新区": "天津",
+        "南开区": "天津",
+        "红桥区": "天津",
+        "塘沽": "天津",
+        "汉沽": "天津",
+        "大港": "天津",
+        "东丽": "天津",
+        "西青": "天津",
+        "津南": "天津",
+        "北辰": "天津",
+        "武清": "天津",
+        "宝坻": "天津",
+        "井陉矿": "河北",
+        "裕华": "河北",
+        "路南区": "河北",
+        "路北区": "河北",
+        "古冶": "河北",
+        "丰南": "河北",
+        "丰润": "河北",
+        "海港区": "河北",
+        "山海关": "河北",
+        "北戴河": "河北",
+        "邯山": "河北",
+        "丛台": "河北",
+        "复兴区": "河北",
+        "峰峰矿": "河北",
+        "北市区": "河北",
+        "南市区": "河北",
+        "下花园": "河北",
+        "双滦": "河北",
+        "鹰手营子矿": "河北",
+        "运河区": "河北",
+        "安次": "河北",
+        "广阳": "河北",
+        "桃城": "河北",
+        "小店区": "山西",
+        "迎泽": "山西",
+        "杏花岭": "山西",
+        "尖草坪": "山西",
+        "万柏林": "山西",
+        "晋源": "山西",
+        "南郊区": "山西",
+        "新荣": "山西",
+        "朔城": "山西",
+        "平鲁": "山西",
+        "榆次": "山西",
+        "盐湖": "山西",
+        "忻府": "山西",
+        "尧都": "山西",
+        "离石": "山西",
+        "玉泉": "内蒙古",
+        "赛罕": "内蒙古",
+        "东河区": "内蒙古",
+        "昆都仑": "内蒙古",
+        "石拐": "内蒙古",
+        "白云矿": "内蒙古",
+        "九原": "内蒙古",
+        "海勃湾": "内蒙古",
+        "乌达": "内蒙古",
+        "红山区": "内蒙古",
+        "元宝山": "内蒙古",
+        "松山": "内蒙古",
+        "科尔沁": "内蒙古",
+        "东胜": "内蒙古",
+        "海拉尔": "内蒙古",
+        "临河": "内蒙古",
+        "集宁": "内蒙古",
+        "沈河": "辽宁",
+        "大东区": "辽宁",
+        "皇姑区": "辽宁",
+        "苏家屯": "辽宁",
+        "东陵区": "辽宁",
+        "沈北新": "辽宁",
+        "于洪": "辽宁",
+        "西岗区": "辽宁",
+        "沙河口": "辽宁",
+        "甘井子": "辽宁",
+        "旅顺口": "辽宁",
+        "金州": "辽宁",
+        "立山": "辽宁",
+        "千山": "辽宁",
+        "新抚": "辽宁",
+        "东洲": "辽宁",
+        "望花": "辽宁",
+        "顺城": "辽宁",
+        "溪湖": "辽宁",
+        "明山区": "辽宁",
+        "南芬": "辽宁",
+        "元宝区": "辽宁",
+        "振兴区": "辽宁",
+        "振安": "辽宁",
+        "古塔区": "辽宁",
+        "凌河": "辽宁",
+        "站前区": "辽宁",
+        "西市区": "辽宁",
+        "鲅鱼圈": "辽宁",
+        "老边": "辽宁",
+        "新邱": "辽宁",
+        "太平区": "辽宁",
+        "清河门": "辽宁",
+        "细河": "辽宁",
+        "白塔": "辽宁",
+        "文圣区": "辽宁",
+        "宏伟区": "辽宁",
+        "弓长岭": "辽宁",
+        "太子河": "辽宁",
+        "双台子": "辽宁",
+        "兴隆台": "辽宁",
+        "银州": "辽宁",
+        "双塔区": "辽宁",
+        "龙城区": "辽宁",
+        "龙港区": "辽宁",
+        "南票区": "辽宁",
+        "杨家杖子开发": "辽宁",
+        "南关区": "吉林",
+        "二道区": "吉林",
+        "绿园区": "吉林",
+        "双阳区": "吉林",
+        "龙潭区": "吉林",
+        "船营": "吉林",
+        "丰满区": "吉林",
+        "东昌区": "吉林",
+        "东昌府区": "山东",
+        "二道江": "吉林",
+        "八道江": "吉林",
+        "江源区": "吉林",
+        "宁江": "吉林",
+        "洮北": "吉林",
+        "道里": "黑龙江",
+        "南岗区": "黑龙江",
+        "道外区": "黑龙江",
+        "平房区": "黑龙江",
+        "松北": "黑龙江",
+        "香坊": "黑龙江",
+        "呼兰": "黑龙江",
+        "阿城": "黑龙江",
+        "龙沙": "黑龙江",
+        "建华区": "黑龙江",
+        "铁锋": "黑龙江",
+        "昂昂溪": "黑龙江",
+        "富拉尔基": "黑龙江",
+        "碾子山": "黑龙江",
+        "梅里斯达斡尔族": "黑龙江",
+        "鸡冠区": "黑龙江",
+        "恒山": "黑龙江",
+        "滴道": "黑龙江",
+        "城子河": "黑龙江",
+        "麻山": "黑龙江",
+        "工农区": "黑龙江",
+        "尖山": "黑龙江",
+        "岭东": "黑龙江",
+        "四方台区": "黑龙江",
+        "萨尔图": "黑龙江",
+        "龙凤区": "黑龙江",
+        "让胡路": "黑龙江",
+        "红岗区": "黑龙江",
+        "南岔": "黑龙江",
+        "友好区": "黑龙江",
+        "翠峦": "黑龙江",
+        "新青区": "黑龙江",
+        "美溪": "黑龙江",
+        "金山屯": "黑龙江",
+        "五营区": "黑龙江",
+        "乌马河": "黑龙江",
+        "汤旺河": "黑龙江",
+        "带岭": "黑龙江",
+        "乌伊岭": "黑龙江",
+        "上甘岭": "黑龙江",
+        "东风区": "黑龙江",
+        "桃山": "黑龙江",
+        "茄子河": "黑龙江",
+        "阳明区": "黑龙江",
+        "爱民区": "黑龙江",
+        "爱辉区": "黑龙江",
+        "北林区": "黑龙江",
+        "大兴安岭": "黑龙江",
+        "加格达奇": "黑龙江",
+        "松岭区": "黑龙江",
+        "新林区": "黑龙江",
+        "呼中": "黑龙江",
+        "黄浦": "上海",
+        "徐汇": "上海",
+        "静安": "上海",
+        "闸北": "上海",
+        "虹口": "上海",
+        "杨浦": "上海",
+        "闵行": "上海",
+        "嘉定": "上海",
+        "浦东新": "上海",
+        "金山区 ": "上海",
+        "松江": "上海",
+        "青浦": "上海",
+        "奉贤": "上海",
+        "玄武": "江苏",
+        "白下": "江苏",
+        "秦淮": "江苏",
+        "建邺": "江苏",
+        "下关区": "江苏",
+        "浦口区": "江苏",
+        "雨花台": "江苏",
+        "江宁": "江苏",
+        "六合": "江苏",
+        "崇安": "江苏",
+        "南长": "江苏",
+        "北塘": "江苏",
+        "锡山": "江苏",
+        "惠山": "江苏",
+        "滨湖": "江苏",
+        "九里": "江苏",
+        "贾汪": "江苏",
+        "泉山": "江苏",
+        "天宁": "江苏",
+        "钟楼": "江苏",
+        "戚墅堰": "江苏",
+        "新北": "江苏",
+        "武进": "江苏",
+        "沧浪": "江苏",
+        "金阊": "江苏",
+        "虎丘": "江苏",
+        "吴中": "江苏",
+        "相城": "江苏",
+        "崇川": "江苏",
+        "港闸": "江苏",
+        "新浦": "江苏",
+        "楚州": "江苏",
+        "淮阴区": "江苏",
+        "清浦": "江苏",
+        "亭湖": "江苏",
+        "盐都": "江苏",
+        "广陵": "江苏",
+        "邗江": "江苏",
+        "维扬": "江苏",
+        "京口": "江苏",
+        "润州": "江苏",
+        "丹徒": "江苏",
+        "海陵": "江苏",
+        "高港": "江苏",
+        "宿城": "江苏",
+        "宿豫": "江苏",
+        "上城区": "浙江",
+        "下城区": "浙江",
+        "江干区": "浙江",
+        "拱墅": "浙江",
+        "滨江区": "浙江",
+        "萧山": "浙江",
+        "余杭": "浙江",
+        "海曙": "浙江",
+        "江东区": "浙江",
+        "北仑": "浙江",
+        "镇海": "浙江",
+        "鄞州": "浙江",
+        "鹿城": "浙江",
+        "龙湾区": "浙江",
+        "瓯海": "浙江",
+        "南湖区": "浙江",
+        "秀洲": "浙江",
+        "吴兴": "浙江",
+        "南浔": "浙江",
+        "越城": "浙江",
+        "婺城": "浙江",
+        "金东": "浙江",
+        "柯城": "浙江",
+        "衢江": "浙江",
+        "定海": "浙江",
+        "椒江": "浙江",
+        "黄岩": "浙江",
+        "路桥区": "浙江",
+        "莲都": "浙江",
+        "瑶海": "安徽",
+        "庐阳": "安徽",
+        "蜀山": "安徽",
+        "新站": "安徽",
+        "包河": "安徽",
+        "镜湖区": "安徽",
+        "镜湖新区": "浙江",
+        "弋江": "安徽",
+        "鸠江": "安徽",
+        "三山区": "安徽",
+        "蚌山": "安徽",
+        "禹会": "安徽",
+        "淮上": "安徽",
+        "田家庵": "安徽",
+        "谢家集": "安徽",
+        "八公山": "安徽",
+        "潘集": "安徽",
+        "金家庄": "安徽",
+        "花山区": "安徽",
+        "雨山": "安徽",
+        "博望新": "安徽",
+        "杜集区": "安徽",
+        "相山": "安徽",
+        "烈山": "安徽",
+        "铜官山": "安徽",
+        "狮子山": "安徽",
+        "迎江": "安徽",
+        "大观区": "安徽",
+        "宜秀": "安徽",
+        "屯溪": "安徽",
+        "徽州": "安徽",
+        "琅琊区": "安徽",
+        "南谯": "安徽",
+        "颍州": "安徽",
+        "颍东": "安徽",
+        "颍泉": "安徽",
+        "埇桥": "安徽",
+        "居巢区": "安徽",
+        "金安": "安徽",
+        "裕安": "安徽",
+        "谯城": "安徽",
+        "贵池": "安徽",
+        "宣州": "安徽",
+        "仓山": "福建",
+        "马尾": "福建",
+        "晋安": "福建",
+        "思明区": "福建",
+        "海沧": "福建",
+        "湖里": "福建",
+        "集美区": "福建",
+        "同安区": "福建",
+        "同安县": "福建",
+        "翔安区": "福建",
+        "城厢": "福建",
+        "涵江": "福建",
+        "荔城": "福建",
+        "秀屿区": "福建",
+        "梅列": "福建",
+        "三元区": "福建",
+        "鲤城": "福建",
+        "丰泽区": "福建",
+        "洛江": "福建",
+        "泉港": "福建",
+        "芗城": "福建",
+        "龙文": "福建",
+        "延平": "福建",
+        "新罗": "福建",
+        "蕉城": "福建",
+        "东湖区": "江西",
+        "青云谱": "江西",
+        "湾里区": "江西",
+        "青山湖": "江西",
+        "珠山": "江西",
+        "安源": "江西",
+        "湘东": "江西",
+        "庐山": "江西",
+        "浔阳": "江西",
+        "渝水": "江西",
+        "月湖区": "江西",
+        "章贡": "江西",
+        "吉州": "江西",
+        "青原区": "江西",
+        "袁州": "江西",
+        "临川": "江西",
+        "信州": "江西",
+        "历下": "山东",
+        "槐荫": "山东",
+        "天桥区": "山东",
+        "历城": "山东",
+        "长清": "山东",
+        "市南区": "山东",
+        "市北区": "山东",
+        "四方区": "山东",
+        "黄岛": "山东",
+        "崂山": "山东",
+        "李沧": "山东",
+        "城阳": "山东",
+        "淄川": "山东",
+        "张店": "山东",
+        "博山": "山东",
+        "临淄": "山东",
+        "周村": "山东",
+        "薛城": "山东",
+        "峄城": "山东",
+        "台儿庄": "山东",
+        "山亭": "山东",
+        "芝罘": "山东",
+        "福山": "山东",
+        "牟平": "山东",
+        "莱山": "山东",
+        "潍城": "山东",
+        "寒亭": "山东",
+        "坊子": "山东",
+        "奎文": "山东",
+        "任城": "山东",
+        "泰山区": "山东",
+        "岱岳": "山东",
+        "环翠": "山东",
+        "岚山": "山东",
+        "莱城": "山东",
+        "钢城区": "山东",
+        "兰山": "山东",
+        "罗庄区": "山东",
+        "德城区": "山东",
+        "东昌府": "山东",
+        "滨城区": "山东",
+        "牡丹区": "山东",
+        "中原区": "河南",
+        "二七区": "河南",
+        "金水区": "河南",
+        "上街": "河南",
+        "惠济区": "河南",
+        "龙亭区": "河南",
+        "顺河回族": "河南",
+        "禹王台": "河南",
+        "金明": "河南",
+        "西工区": "河南",
+        "瀍河回族": "河南",
+        "涧西": "河南",
+        "吉利区": "河南",
+        "洛龙": "河南",
+        "卫东区": "河南",
+        "石龙": "河南",
+        "湛河": "河南",
+        "文峰": "河南",
+        "北关区": "河南",
+        "殷都": "河南",
+        "龙安": "河南",
+        "山城区": "河南",
+        "淇滨": "河南",
+        "红旗区": "河南",
+        "卫滨": "河南",
+        "凤泉": "河南",
+        "牧野区": "河南",
+        "中站区": "河南",
+        "马村区": "河南",
+        "华龙区": "河南",
+        "魏都": "河南",
+        "源汇": "河南",
+        "郾城": "河南",
+        "召陵": "河南",
+        "湖滨": "河南",
+        "宛城": "河南",
+        "卧龙区": "河南",
+        "梁园": "河南",
+        "睢阳": "河南",
+        "浉河": "河南",
+        "平桥区": "河南",
+        "川汇": "河南",
+        "驿城": "河南",
+        "江岸区": "湖北",
+        "江汉区": "湖北",
+        "硚口": "湖北",
+        "汉阳": "湖北",
+        "武昌": "湖北",
+        "洪山": "湖北",
+        "东西湖": "湖北",
+        "汉南": "湖北",
+        "蔡甸": "湖北",
+        "江夏": "湖北",
+        "黄陂": "湖北",
+        "新洲区": "湖北",
+        "东湖开发": "湖北",
+        "武汉经济开发": "湖北",
+        "黄石港": "湖北",
+        "西塞山": "湖北",
+        "下陆": "湖北",
+        "铁山区": "湖北",
+        "黄石经济开发": "湖北",
+        "茅箭": "湖北",
+        "张湾": "湖北",
+        "十堰经济开发": "湖北",
+        "武当山旅游经济特": "湖北",
+        "西陵": "湖北",
+        "伍家岗": "湖北",
+        "点军": "湖北",
+        "猇亭": "湖北",
+        "夷陵": "湖北",
+        "樊城": "湖北",
+        "隆中风景": "湖北",
+        "高新开发": "湖北",
+        "鱼梁洲开发": "湖北",
+        "梁子湖": "湖北",
+        "鄂城": "湖北",
+        "葛店开发": "湖北",
+        "鄂州经济开发": "湖北",
+        "东宝": "湖北",
+        "掇刀": "湖北",
+        "荆门经济开发": "湖北",
+        "孝南": "湖北",
+        "沙市": "湖北",
+        "荆州经济开发": "湖北",
+        "黄州": "湖北",
+        "龙感湖管理": "湖北",
+        "咸安": "湖北",
+        "曾都": "湖北",
+        "芙蓉区": "湖南",
+        "天心区": "湖南",
+        "岳麓": "湖南",
+        "开福": "湖南",
+        "雨花区": "湖南",
+        "荷塘区": "湖南",
+        "芦淞": "湖南",
+        "石峰": "湖南",
+        "天元区": "湖南",
+        "雨湖": "湖南",
+        "岳塘": "湖南",
+        "珠晖": "湖南",
+        "雁峰": "湖南",
+        "石鼓": "湖南",
+        "蒸湘": "湖南",
+        "南岳": "湖南",
+        "双清": "湖南",
+        "大祥": "湖南",
+        "北塔区": "湖南",
+        "岳阳楼": "湖南",
+        "云溪": "湖南",
+        "君山": "湖南",
+        "武陵": "湖南",
+        "鼎城": "湖南",
+        "武陵源": "湖南",
+        "赫山": "湖南",
+        "北湖": "湖南",
+        "苏仙": "湖南",
+        "零陵": "湖南",
+        "冷水滩": "湖南",
+        "鹤城": "湖南",
+        "娄星": "湖南",
+        "荔湾": "广东",
+        "越秀": "广东",
+        "海珠": "广东",
+        "天河区": "广东",
+        "黄埔": "广东",
+        "番禺": "广东",
+        "花都区": "广东",
+        "南沙区": "广东",
+        "萝岗": "广东",
+        "武江": "广东",
+        "浈江": "广东",
+        "曲江": "广东",
+        "罗湖": "广东",
+        "福田": "广东",
+        "宝安": "广东",
+        "龙岗": "广东",
+        "盐田": "广东",
+        "香洲": "广东",
+        "斗门区": "广东",
+        "斗门镇": "广东",
+        "金湾区": "广东",
+        "龙湖区": "广东",
+        "濠江": "广东",
+        "潮阳": "广东",
+        "潮南": "广东",
+        "澄海": "广东",
+        "禅城": "广东",
+        "南海区": "广东",
+        "顺德": "广东",
+        "三水": "广东",
+        "高明区": "广东",
+        "蓬江": "广东",
+        "江海区": "广东",
+        "新会": "广东",
+        "赤坎": "广东",
+        "霞山": "广东",
+        "坡头区": "广东",
+        "麻章": "广东",
+        "茂南": "广东",
+        "茂港": "广东",
+        "端州": "广东",
+        "鼎湖": "广东",
+        "惠城": "广东",
+        "惠阳": "广东",
+        "梅江": "广东",
+        "源城": "广东",
+        "清城": "广东",
+        "枫溪": "广东",
+        "湘桥": "广东",
+        "榕城": "广东",
+        "云城": "广东",
+        "青秀": "广西",
+        "江南区": "广西",
+        "西乡塘": "广西",
+        "良庆": "广西",
+        "邕宁": "广西",
+        "鱼峰": "广西",
+        "柳南": "广西",
+        "柳北": "广西",
+        "秀峰区": "广西",
+        "叠彩": "广西",
+        "七星区": "广西",
+        "雁山区": "广西",
+        "万秀区": "广西",
+        "蝶山": "广西",
+        "长洲": "广西",
+        "银海": "广西",
+        "铁山港": "广西",
+        "港口区": "广西",
+        "防城": "广西",
+        "钦南": "广西",
+        "钦北": "广西",
+        "港北": "广西",
+        "港南": "广西",
+        "覃塘": "广西",
+        "玉州": "广西",
+        "右江区": "广西",
+        "八步": "广西",
+        "金城江": "广西",
+        "兴宾": "广西",
+        "江洲": "广西",
+        "秀英区": "海南",
+        "龙华区": "海南",
+        "琼山": "海南",
+        "美兰": "海南",
+        "洋浦经济开发": "海南",
+        "万州": "重庆",
+        "涪陵": "重庆",
+        "渝中": "重庆",
+        "大渡口区": "重庆",
+        "沙坪坝": "重庆",
+        "九龙坡": "重庆",
+        "南岸区": "重庆",
+        "北碚": "重庆",
+        "万盛区": "重庆",
+        "渝北": "重庆",
+        "巴南": "重庆",
+        "黔江": "重庆",
+        "长寿区": "重庆",
+        "江津区": "重庆",
+        "合川": "重庆",
+        "南川": "重庆",
+        "永川": "重庆",
+        "北部新": "重庆",
+        "锦江": "四川",
+        "青羊": "四川",
+        "金牛": "四川",
+        "武侯": "四川",
+        "成华": "四川",
+        "龙泉驿": "四川",
+        "青白江": "四川",
+        "新都区": "四川",
+        "温江区": "四川",
+        "自流井": "四川",
+        "贡井": "四川",
+        "沿滩区": "四川",
+        "仁和区": "四川",
+        "江阳区": "四川",
+        "纳溪": "四川",
+        "龙马潭": "四川",
+        "旌阳": "四川",
+        "涪城": "四川",
+        "游仙": "四川",
+        "元坝": "四川",
+        "朝天区": "四川",
+        "船山": "四川",
+        "安居区": "四川",
+        "五通桥": "四川",
+        "金口河": "四川",
+        "顺庆": "四川",
+        "高坪": "四川",
+        "嘉陵": "四川",
+        "东坡区": "四川",
+        "翠屏": "四川",
+        "通川": "四川",
+        "雨城": "四川",
+        "巴州": "四川",
+        "雁江": "四川",
+        "南明": "贵州",
+        "云岩": "贵州",
+        "花溪": "贵州",
+        "乌当": "贵州",
+        "小河区": "贵州",
+        "金阳新": "贵州",
+        "六枝特": "贵州",
+        "红花岗": "贵州",
+        "汇川": "贵州",
+        "新浦新": "贵州",
+        "西秀": "贵州",
+        "黄果树开发": "贵州",
+        "七星关": "贵州",
+        "碧江": "贵州",
+        "万山特": "贵州",
+        "凯里经济开发": "贵州",
+        "盘龙": "云南",
+        "官渡区": "云南",
+        "西山区": "云南",
+        "东川区": "云南",
+        "麒麟区": "云南",
+        "红塔区": "云南",
+        "隆阳": "云南",
+        "昭阳": "云南",
+        "古城区": "云南",
+        "思茅": "云南",
+        "临翔": "云南",
+        "山南市": "西藏",
+        "阿里地区": "西藏",
+        "碑林": "陕西",
+        "莲湖": "陕西",
+        "灞桥": "陕西",
+        "未央": "陕西",
+        "雁塔": "陕西",
+        "阎良": "陕西",
+        "临潼": "陕西",
+        "沣渭新": "陕西",
+        "王益": "陕西",
+        "印台": "陕西",
+        "耀州": "陕西",
+        "渭滨": "陕西",
+        "金台": "陕西",
+        "陈仓": "陕西",
+        "秦都": "陕西",
+        "渭城": "陕西",
+        "临渭": "陕西",
+        "宝塔区": "陕西",
+        "汉台": "陕西",
+        "榆阳": "陕西",
+        "汉滨": "陕西",
+        "商州": "陕西",
+        "杨凌示范": "陕西",
+        "七里河区": "甘肃",
+        "西固": "甘肃",
+        "红古": "甘肃",
+        "平川": "甘肃",
+        "秦州": "甘肃",
+        "麦积": "甘肃",
+        "凉州": "甘肃",
+        "甘州": "甘肃",
+        "崆峒": "甘肃",
+        "肃州": "甘肃",
+        "西峰": "甘肃",
+        "安定": "甘肃",
+        "武都": "甘肃",
+        "海东": "青海",
+        "兴庆": "宁夏",
+        "西夏区": "宁夏",
+        "金凤": "宁夏",
+        "大武口": "宁夏",
+        "惠农": "宁夏",
+        "利通": "宁夏",
+        "红寺堡": "宁夏",
+        "原州": "宁夏",
+        "沙坡头": "宁夏",
+        "天山区": "新疆",
+        "沙依巴克": "新疆",
+        "水磨沟": "新疆",
+        "头屯河": "新疆",
+        "达坂城": "新疆",
+        "米东": "新疆",
+        "独山子": "新疆",
+        "白碱滩": "新疆",
+        "乌尔禾": "新疆",
+        "中西区": "香港",
+        "九龙城": "香港",
+        "观塘": "香港",
+        "深水埗": "香港",
+        "湾仔区": "香港",
+        "黄大仙": "香港",
+        "油尖旺": "香港",
+        "离岛": "香港",
+        "葵青": "香港",
+        "西贡": "香港",
+        "沙田": "香港",
+        "屯门": "香港",
+        "荃湾": "香港",
+        "元朗": "香港",
+        "花地玛堂": "澳门",
+        "圣安多尼堂": "澳门",
+        "大堂": "澳门",
+        "望德堂": "澳门",
+        "风顺堂": "澳门",
+        "嘉模堂": "澳门",
+        "圣方济各堂": "澳门",
+        "大学": "大学",
+        "学院": "学院",
+        "医院": "医院",
+        "中学": "中学",
+        "小学": "小学",
+        "幼儿园": "幼儿园",
+        "中心": "中心",
+        "公司": "公司",
+        "集团": "集团",
+        "办事处": "办事处",
+        "研究院": "研究院",
+        "研究所": "研究所",
+        "移动": "移动",
+        "联通": "联通",
+        "电信": "电信",
+        "工商": "工商",
+        "公安": "公安",
+        "检察院": "检察院",
+        "检查院": "检查院",
+		"法院":"法院",
+		"司法":"司法",
+		"采购中心":"采购中心",
+		"办公室":"办公室",
+        "地税": "地税",
+        "国税": "国税",
+        "包钢": "包钢",
+        "镇": "镇",
+        "乡": "乡",
+        "市": "市",
+		"局":"局",
+		"厅":"厅",
+		"滇":"滇",
+		"辽":"辽",
+		"沪":"沪",
+		"浙":"浙",
+		"皖":"皖",
+		"闽":"闽",
+		"赣":"赣",
+		"鲁":"鲁",
+		"豫":"豫",
+		"鄂":"鄂",
+		"湘":"湘",
+		"粤":"粤",
+		"桂":"桂",
+		"琼":"琼",
+		"渝":"渝",
+		"川":"川",
+		"藏":"藏",
+		"陕":"陕",
+		"甘":"甘",
+		"黑":"黑",
+		"蒙":"蒙",
+		"吉":"吉",
+		"澳":"澳",
+		"湾":"湾",
+		"山":"山",
+		"河":"河",
+        "首钢": "首钢",
+        "太钢": "太钢",
+        "天钢": "天钢",
+        "国丰": "国丰",
+        "临钢": "临钢",
+        "文丰": "文丰",
+        "天铁": "天铁",
+        "普阳": "普阳",
+        "德龙": "德龙",
+        "港陆": "港陆",
+        "新金": "新金",
+        "宝钢": "宝钢",
+        "南钢": "南钢",
+        "梅钢": "梅钢",
+        "马钢": "马钢",
+        "济钢": "济钢",
+        "莱钢": "莱钢",
+        "沙钢": "沙钢",
+        "新钢": "新钢",
+        "泰钢": "泰钢",
+        "萍钢": "萍钢",
+        "华伟": "华伟",
+        "兆泰": "兆泰",
+        "西城": "西城",
+        "宁钢": "宁钢",
+        "春冶": "春冶",
+        "兆顺": "兆顺",
+        "长达": "长达",
+        "飞达": "飞达 ",
+        "武钢": "武钢",
+        "安钢": "安钢",
+        "涟钢": "涟钢",
+        "湘钢": "湘钢",
+        "鄂钢": "鄂钢",
+        "韶钢": "韶钢",
+        "柳钢": "柳钢",
+        "鞍钢": "鞍钢",
+        "本钢": "本钢",
+        "通钢": "通钢",
+        "北台": "北台",
+        "攀钢": "攀钢",
+        "重钢": "重钢",
+        "昆钢": "昆钢",
+        "包钢": "包钢",
+        "酒钢": "酒钢",
+        "八钢": "八钢"
+    }
+}

udpprojectset/src/cleareids.go

@@ -0,0 +1,109 @@
+// cleareids
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"qfw/util"
+	"qfw/util/redis"
+	"sync"
+	"time"
+
+	"github.com/robfig/cron"
+)
+
+var (
+	c            *cron.Cron
+	projectcycle int64
+)
+
+func clearedis() {
+	if credis, ok := Sysconfig["clearedis"].(map[string]interface{}); ok {
+		clearcron := util.ObjToString(credis["clearcron"])
+		projectcycle = util.Int64All(credis["projectcycle"])
+		log.Println(credis)
+		if credis["open"].(bool) {
+			c = cron.New()
+			c.AddFunc(clearcron, clearPKey)
+			c.Start()
+		}
+	}
+}
+
+func clearPKey() {
+	log.Println("开始清理")
+	nowtime := time.Now().Unix()
+	wg := sync.WaitGroup{}
+	for _, pncb := range []*KeyMap{PNKey, PCKey, PBKey} {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			clearPNCBKey(pncb, nowtime)
+		}()
+	}
+	wg.Wait()
+	log.Println("清理结束")
+}
+
+func clearPNCBKey(pncb *KeyMap, nowtime int64) {
+	delkey := clearIdsKeys(pncb, nowtime)
+	pncb.Lock.Lock()
+	for _, k := range delkey {
+		delete(pncb.Map, k)
+	}
+	pncb.Lock.Unlock()
+}
+
+func clearIdsKeys(pKey *KeyMap, nowtime int64) []string {
+	defer util.Catch()
+	delkey := []string{}
+	for k, ma := range pKey.Map {
+		ids := ma.Arr
+		delids := []interface{}{}
+		res := redis.Mget(REDISIDS, *ids)
+		for _, b1 := range res {
+			if b1 != nil {
+				var info ProjectInfo
+				err := json.Unmarshal(b1.([]byte), &info)
+				if err != nil {
+					log.Println(err)
+					continue
+				}
+				publistime := info.Publistime[len(info.Publistime)-1]
+				if projectcycle < (nowtime-publistime)/86400 { //项目周期超时
+					delids = append(delids, info.Id)
+				}
+			}
+		}
+		if len(delids) < 1 {
+			continue
+		}
+		//删除redis相关信息
+		b := redis.Del(REDISIDS, delids...)
+		if b {
+			for _, id := range delids {
+				ids = deleteSliceId(*ids, id.(string))
+			}
+			//log.Println(ids, *ma.Arr)
+			ma.Arr = ids
+			if len(*ma.Arr) < 1 {
+				redis.Del(REDISKEYS, k)
+				delkey = append(delkey, k)
+			} else {
+				//更新REDISKEYS
+				redis.Put(REDISKEYS, k, *ma.Arr, 0)
+			}
+		}
+	}
+	return delkey
+}
+
+func deleteSliceId(a []string, id string) *[]string {
+	ret := make([]string, 0, len(a))
+	for _, val := range a {
+		if val != id {
+			ret = append(ret, val)
+		}
+	}
+	return &ret
+}

udpprojectset/src/compare.go

@@ -0,0 +1,678 @@
+package main
+
+import (
+	"log"
+	"qfw/util"
+	"qfw/util/redis"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+/**
+发布时间（结合信息类型-->）
+信息类型（拟建、招标(变更)、结果(成交、中标、流标、废标、变更)、其他(合同、验收、违规、变更)）
+项目代码（长度）
+采购单位（结合省份）
+招标机构
+所属省份
+信息标题（含有二次、标段、包的情况，结合分包情况处理）
+是否分包：
+**/
+type ProjectInfo struct {
+	Id            string                 `json:"id"`
+	Publistime    []int64                `json:"publistime"` //多条信息的发布时间、跨度
+	InfoType      [][]string             `json:"infotype"`   //多条信息内的 toptype、subtype
+	Ids           []string               `json:"ids"`
+	Topscopeclass []string               `json:"topscopeclass"`
+	Subscopeclass []string               `json:"subscopeclass"`
+	Winners       []string               `json:"winners"`
+	ProjectName   string                 `json:"projectname"`
+	ProjectCode   string                 `json:"projectcode"` //项目代码唯一（纯数字的权重低）
+	Buyer         string                 `json:"buyer"`       //采购单位唯一
+	Buyerperson   string                 `json:"buyerperson"`
+	Buyertel      string                 `json:"buyertel"`
+	Agency        string                 `json:"agency"`     //代理机构唯一
+	Area          string                 `json:"area"`       //地区唯一
+	City          string                 `json:"city"`       //地市
+	HasPackage    bool                   `json:"haspackage"` //是否有分包
+	Package       map[string]interface{} `json:"package"`    //分包的对比对象
+
+	Buyerclass  string   `json:"buyerclass"`  //采购单位分类
+	Bidopentime int64    `json:"bidopentime"` //开标时间
+	District    string   `json:"district"`    //区县
+	Winnerorder []string //中标候选人
+}
+
+type CompareOne struct {
+	ProjectNameType string //项目名称对比结果分类 A相等 B包含 C不相等 Dthis存在对比不存在 Ethis不存在
+	ProjectCodeType string //同上
+	BuyerType       string //同上
+	AreaType        string //同上
+	AgencyType      string //同上
+	PublistimeType  int    //1在时间范围  2不在
+	PackageType     int    //1都是多包 2都不是多包 3招标 4新信息是结果
+	Score           int
+	Parent          *CompareInfo
+	Pinfo           *ProjectInfo
+	Pos             int
+
+	Cresult string //对比结果
+}
+
+type CompareInfo struct {
+	Field string //对比属性 pn/pc/pb
+	Key   string //存放rediskey
+	//Pinfo  []ProjectInfo //存放redis对象
+	Scores []*CompareOne //对比分值 pinfo索引对应分值
+	Bfind  bool          //是否查找到
+	IdArr  []string
+	K      *Key
+	KeyMap *KeyMap
+}
+
+func NewCompareInfo(field, key string, KeyMap *KeyMap) *CompareInfo {
+	return &CompareInfo{
+		Field:  field,
+		Key:    key,
+		Scores: []*CompareOne{},
+		KeyMap: KeyMap,
+	}
+
+}
+
+var numreg2 = regexp.MustCompile("^[0-9]+$")
+var TitleReg = regexp.MustCompile("([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ、\\-~至]+(子|合同|分|施工|监理|标)?[包标段][号段]?[、]?)+|((子|合同|分|施工|监理|标)?[包标段][号段]?[一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ、\\-~至]+[、]?)+|(子|合同|分|施工|监理|标)?[包标段][号段]?[a-zA-Z0-9]+[\\-~－至、](子|合同|分|施工|监理|标)?[包标段][号段]?[a-zA-Z0-9]+")
+
+//对比打分程序
+func Compare(tmp map[string]interface{} /*新信息*/, pici int64) (sflag string) {
+	defer util.Catch()
+	thisinfo := PreThisInfo(tmp) //信息预处理
+	if thisinfo == nil {
+		return
+	}
+	var PN, PC, PB *CompareInfo
+	new_pn, _ := handleprojectname(thisinfo.ProjectName, thisinfo.Buyer, thisinfo.TopType) //作为判断依据
+	if thisinfo.ProjectName != "" {
+		KEY := "pn_" + new_pn
+		PN = NewCompareInfo("pn", KEY, PNKey)
+	}
+	if len(thisinfo.ProjectCode) > 5 && !numreg.MatchString(thisinfo.ProjectCode) { //大于5且不是纯数字bfind
+		KEY := "pc_" + thisinfo.ProjectCode
+		PC = NewCompareInfo("pc", KEY, PCKey)
+	}
+	if len([]rune(thisinfo.Buyer)) > 4 {
+		KEY := "pb_" + thisinfo.Buyer
+		PB = NewCompareInfo("pb", KEY, PBKey)
+	}
+	repeatId := map[string]bool{}
+	IdLock.Lock()
+	//此处加id锁，会引进多线程的死锁，对比三个大map数组，找到key相同的项目id数组,并去重
+	for _, PNCB := range []*CompareInfo{PN, PC, PB} {
+		if PNCB != nil {
+			PNCB.KeyMap.Lock.Lock()
+			K := PNCB.KeyMap.Map[PNCB.Key]
+			if K == nil {
+				K = &Key{&[]string{}, &sync.Mutex{}}
+				PNCB.KeyMap.Map[PNCB.Key] = K
+			}
+			PNCB.K = K
+			PNCB.K.Lock.Lock()
+			PNCB.KeyMap.Lock.Unlock()
+			defer PNCB.K.Lock.Unlock()
+			newarr := []string{}
+			for _, id := range *K.Arr {
+				if !repeatId[id] {
+					newarr = append(newarr, id)
+					repeatId[id] = true
+				}
+			}
+			PNCB.IdArr = newarr
+		}
+	}
+	IdLock.Unlock()
+	updateid, ncid := "", ""
+	tmpArr := []*CompareOne{}
+	//三个对比对象分别和thisinfo比较，打完分的对象放入scores，合并放入tempArr
+	for _, PNCB := range []*CompareInfo{PN, PC, PB} {
+		if PNCB != nil {
+			if len(PNCB.IdArr) > 0 {
+				//PNCB.compute(thisinfo, PNCB.IdArr)
+				ncid = PNCB.ProcessInfo(new_pn, tmp, pici, thisinfo)
+				tmpArr = append(tmpArr, PNCB.Scores...)
+			}
+		}
+	}
+	if ncid == "" {
+		//排序取出最大分值的对象，更新对象
+		if len(tmpArr) > 0 {
+			sort.Slice(tmpArr, func(i, j int) bool {
+				return tmpArr[i].Score > tmpArr[j].Score
+			})
+			max := tmpArr[0]
+			if max.Score > 0 {
+				//获取到对象
+				max.Parent.Bfind = true
+				tmp["cresult"] = max.Cresult
+				updateid = updateinfo(thisinfo, tmp, max.Pinfo, pici)
+				//log.Println("更新项目", updateid)
+			}
+		}
+		if updateid == "" {
+			sflag = "normal"
+			updateid = InsertProject(new_pn, tmp, nil, pici, thisinfo)
+			p1 := NewPinfo(updateid, thisinfo)
+			redis.PutCKV(REDISIDS, updateid, p1)
+			//log.Println("生成项目", updateid)
+		} else {
+			sflag = "repeat"
+		}
+		//更新redis中存入的三大map对象中的id
+		if updateid != "" {
+			PUT := []interface{}{}
+			for _, PNCB := range []*CompareInfo{PN, PC, PB} {
+				if PNCB != nil && !PNCB.Bfind {
+					if BinarySearch(*PNCB.K.Arr, updateid) == -1 {
+						*(PNCB.K.Arr) = append(*(PNCB.K.Arr), updateid)
+						PUT = append(PUT, []interface{}{PNCB.Key, *(PNCB.K.Arr)})
+					}
+				}
+			}
+			if len(PUT) > 0 {
+				redis.BulkPut(REDISKEYS, 0, PUT...)
+			}
+		}
+	} else {
+		log.Println("无效项目", ncid)
+	}
+	return
+}
+
+//没有在redis中查找到，新增项目
+func InsertProject(new_pn string, tmp, mess map[string]interface{}, pici int64, thisinfo *Info) string {
+	e := InitEL("")
+	set := map[string]interface{}{
+		"pici": pici,
+	}
+	set["s_projectname"] = new_pn
+	e.fieldpriority(&tmp, nil, &set)
+	set["extractpos"] = e.GetVal()
+	set["createtime"] = time.Now().Unix()
+	set["sourceinfoid"] = util.BsonIdToSId(tmp["_id"])
+	set["sourceinfourl"] = tmp["href"]
+	set["topscopeclass"] = thisinfo.Topscopeclass
+	set["subscopeclass"] = thisinfo.Subscopeclass
+	if thisinfo.Buyerperson != "" {
+		set["buyerperson"] = thisinfo.Buyerperson
+	}
+	if thisinfo.Buyertel != "" {
+		set["buyertel"] = thisinfo.Buyertel
+	}
+	if thisinfo.Buyerclass != "" {
+		set["buyertel"] = thisinfo.Buyerclass
+	}
+	if thisinfo.District != "" {
+		set["district"] = thisinfo.District
+	}
+	if thisinfo.Bidopentime > 0 {
+		set["bidopentime"] = thisinfo.Bidopentime
+	}
+	if len(thisinfo.Winnerorder) > 0 {
+		set["winnerorder"] = thisinfo.Winnerorder
+	}
+	s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
+	//	if len(s_subscopeclass) > 0 {
+	//		s_subscopeclass = "," + s_subscopeclass + ","
+	//	}
+	set["s_subscopeclass"] = s_subscopeclass
+	s_winner := strings.Join(thisinfo.Winners, ",")
+	//	if len(s_winner) > 0 {
+	//		s_winner = "," + s_winner + ","
+	//	}
+	set["s_winner"] = s_winner
+	if tmp["package"] != nil {
+		set["package"] = tmp["package"] //没定义优先级
+	}
+	push := NewPushInfo(tmp)
+	for tkey, _ := range extractpos {
+		if tmp[tkey] != nil {
+			push[tkey] = tmp[tkey]
+		}
+	}
+	set["list"] = []bson.M{
+		push,
+	}
+	if mess != nil { //不参与对比项目标识
+		for k, v := range mess {
+			set[k] = v
+		}
+	}
+	return MQFW.Save(projectColl, set)
+}
+
+//生成存放在redis数组中的对象
+func NewPinfo(id string, thisinfo *Info) ProjectInfo {
+	p1 := ProjectInfo{
+		Publistime:    []int64{thisinfo.Publishtime},
+		InfoType:      [][]string{[]string{thisinfo.TopType, thisinfo.SubType}},
+		Id:            id,
+		Ids:           []string{thisinfo.Id},
+		Topscopeclass: thisinfo.Topscopeclass,
+		Subscopeclass: thisinfo.Subscopeclass,
+		Winners:       thisinfo.Winners,
+		ProjectName:   thisinfo.ProjectName,
+		ProjectCode:   thisinfo.ProjectCode,
+		Buyer:         thisinfo.Buyer,
+		Agency:        thisinfo.Agency,
+		Area:          thisinfo.Area,
+		HasPackage:    thisinfo.HasPackage,
+		Package:       map[string]interface{}{},
+		Buyerclass:    thisinfo.Buyerclass,
+		Bidopentime:   thisinfo.Bidopentime,
+		District:      thisinfo.District,
+		Winnerorder:   thisinfo.Winnerorder,
+	}
+	for k4, _ := range thisinfo.Package {
+		p1.Package[k4] = ""
+	}
+	return p1
+}
+
+//每一个字段的打分map,获取最高分
+func GetMaxScore(scoremap map[int]int) (find bool, index int) {
+	ks := []int{}
+	vs := []int{}
+	for k, v := range scoremap {
+		mk, mv := -1, -100
+		bf := false
+		for mk, mv = range vs {
+			if mv > v {
+				bf = true
+				break
+			}
+		}
+		if mk == -1 || !bf {
+			vs = append(vs, v)
+			ks = append(ks, k)
+		} else {
+			vs = append(append(vs[:mk], v), vs[mk:]...)
+			ks = append(append(ks[:mk], k), ks[mk:]...)
+		}
+	}
+	if len(ks) > 0 {
+		index = ks[len(ks)-1]
+		find = true
+	}
+	return
+}
+
+//往对应redis中更新信息
+func updateinfo(thisinfo *Info, tmp map[string]interface{}, pInfo *ProjectInfo, pici int64) string {
+	updateid := pInfo.Id
+	if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
+		return updateid
+	}
+	set := map[string]interface{}{
+		"pici": pici,
+	}
+	res, bres := MQFW.FindById(projectColl, pInfo.Id, `{"list":0}`)
+	EqInfoUpdate(thisinfo, pInfo)
+	if bres && res != nil && *res != nil {
+		set["topscopeclass"] = pInfo.Topscopeclass
+		set["subscopeclass"] = pInfo.Subscopeclass
+		s_subscopeclass := strings.Join(pInfo.Subscopeclass, ",")
+		if len(s_subscopeclass) > 0 {
+			s_subscopeclass = "," + s_subscopeclass + ","
+		}
+		set["s_subscopeclass"] = s_subscopeclass
+		s_winner := strings.Join(pInfo.Winners, ",")
+		if len(s_winner) > 0 {
+			s_winner = "," + s_winner + ","
+		}
+		set["s_winner"] = s_winner
+		if pInfo.Buyerperson != "" && pInfo.Buyertel != "" {
+			set["buyerperson"] = pInfo.Buyerperson
+			set["buyertel"] = pInfo.Buyertel
+		}
+		if pInfo.Buyerclass != "" {
+			set["buyerclass"] = pInfo.Buyerclass
+		}
+		if pInfo.District != "" {
+			set["district"] = pInfo.District
+		}
+		if pInfo.Bidopentime > 0 {
+			set["bidopentime"] = pInfo.Bidopentime
+		}
+		if len(pInfo.Winnerorder) > 0 {
+			set["winnerorder"] = pInfo.Winnerorder
+		}
+		if thisinfo.HasPackage {
+			set["multipackage"] = 1
+		} else {
+			set["multipackage"] = 0
+		}
+		e := InitEL(util.ObjToString((*res)["extractpos"]))
+		if thisinfo.dealtype == 1 {
+			var sonpackage map[string]interface{}
+			for _, obj := range tmp["package"].(map[string]interface{}) {
+				sonpackage, _ = obj.(map[string]interface{})
+			}
+			for _, v2 := range []string{"budget", "budget_w", "winner", "winner_w", "bidstatus", "bidstatus_w"} {
+				if sonpackage[v2] != nil {
+					tmp[v2] = sonpackage[v2]
+				}
+			}
+		}
+		e.fieldpriority(&tmp, res, &set)
+		set["extractpos"] = e.GetVal()
+		if thisinfo.HasPackage { //多包处理
+			p1, _ := (*res)["package"].(map[string]interface{})
+			p2, _ := tmp["package"].(map[string]interface{})
+			if p2 != nil {
+				if p1 != nil {
+					for pk2, pv2 := range p2 {
+						if p1[pk2] != nil { //合并
+							item1, _ := p1[pk2].(map[string]interface{})
+							item2, _ := pv2.(map[string]interface{})
+							if item1 != nil && item2 != nil { //原始项
+								for ik1, iv1 := range item2 {
+									if item1[ik1] == nil {
+										item1[ik1] = iv1
+									}
+								}
+							}
+						} else {
+							p1[pk2] = pv2
+						}
+					}
+				} else {
+					p1 = p2
+				}
+			}
+			set["package"] = p1
+		}
+		//中标候选人合并
+
+		update := map[string]interface{}{}
+		if len(set) > 0 {
+			update["$set"] = set
+		}
+		//保留原数据吧
+		push := NewPushInfo(tmp)
+		for tkey, _ := range extractpos {
+			if tmp[tkey] != nil {
+				push[tkey] = tmp[tkey]
+			}
+		}
+		update["$push"] = map[string]interface{}{
+			"list": push,
+		}
+		if len(update) > 0 {
+			MQFW.Update(projectColl, map[string]interface{}{
+				"_id": util.StringTOBsonId(pInfo.Id),
+			}, &update, false, false)
+		}
+	}
+	//再往redis中放 index
+	//往队列中增加时间 -------------->start
+	redis.Put(REDISIDS, updateid, pInfo, 0)
+	return updateid
+}
+
+func EqInfoUpdate(thisinfo *Info, pInfo *ProjectInfo) {
+	var tk int
+	bf1 := false
+	for _k, tv := range pInfo.Publistime {
+		tk = _k
+		if tv > thisinfo.Publishtime {
+			bf1 = true
+			break
+		}
+	}
+	if bf1 {
+		pInfo.Publistime = append(append(pInfo.Publistime[:tk], thisinfo.Publishtime), pInfo.Publistime[tk:]...)
+		pInfo.InfoType = append(append(pInfo.InfoType[:tk], []string{thisinfo.TopType, thisinfo.SubType}), pInfo.InfoType[tk:]...)
+		pInfo.Ids = append(append(pInfo.Ids[:tk], thisinfo.Id), pInfo.Ids[tk:]...)
+	} else {
+		pInfo.Publistime = append(pInfo.Publistime, thisinfo.Publishtime)
+		pInfo.InfoType = append(pInfo.InfoType, []string{thisinfo.TopType, thisinfo.SubType})
+		pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
+	}
+	//增加发布时间结束----------------->end
+
+	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
+		pInfo.Buyer = thisinfo.Buyer
+	}
+	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
+		pInfo.Agency = thisinfo.Agency
+	}
+	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+		pInfo.ProjectCode = thisinfo.ProjectCode
+	}
+
+	if thisinfo.Buyerperson != "" && thisinfo.Buyertel != "" && len([]rune(thisinfo.Buyertel)) > 6 {
+		pInfo.Buyerperson = thisinfo.Buyerperson
+		pInfo.Buyertel = thisinfo.Buyertel
+	}
+	if thisinfo.Buyerclass != "" {
+		pInfo.Buyerclass = thisinfo.Buyerclass
+	}
+	if thisinfo.District != "" {
+		pInfo.District = thisinfo.District
+	}
+	if thisinfo.Bidopentime > 0 {
+		pInfo.Bidopentime = thisinfo.Bidopentime
+	}
+	if len(thisinfo.Topscopeclass) > 0 {
+		sort.Strings(pInfo.Topscopeclass)
+		for _, k := range thisinfo.Topscopeclass {
+			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
+				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
+				sort.Strings(pInfo.Topscopeclass)
+			}
+		}
+	}
+
+	if len(thisinfo.Subscopeclass) > 0 {
+		sort.Strings(pInfo.Subscopeclass)
+		for _, k := range thisinfo.Subscopeclass {
+			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
+				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
+				sort.Strings(pInfo.Subscopeclass)
+			}
+		}
+	}
+	//winner
+	if len(thisinfo.Winners) > 0 {
+		sort.Strings(pInfo.Winners)
+		for _, k := range thisinfo.Winners {
+			if BinarySearch(pInfo.Winners, k) == -1 {
+				pInfo.Winners = append(pInfo.Winners, k)
+				sort.Strings(pInfo.Winners)
+			}
+		}
+	}
+	//winnerorder
+	if len(thisinfo.Winnerorder) > 0 {
+		sort.Strings(pInfo.Winnerorder)
+		for _, k := range thisinfo.Winnerorder {
+			if BinarySearch(pInfo.Winnerorder, k) == -1 {
+				pInfo.Winnerorder = append(pInfo.Winnerorder, k)
+				sort.Strings(pInfo.Winnerorder)
+			}
+		}
+	}
+	//return pInfo
+}
+
+/**
+func (com *CompareInfo) compute(thisinfo *Info, ids []string) {
+	//根据 com的key到redis中查找对应的值（如：项目名称的key为"pn_某某项目",值为一维对象数组 即:[]ProjectInfo）
+	//这些信息id上面已经加过锁
+	defer util.Catch()
+	res := redis.Mget(REDISIDS, ids)
+	//redis.GetInterface("other", com.Key, &com.Pinfo)
+	//redispush := false
+	if len(res) > 0 { //找到项目名称、项目编号或采购单位相同时
+		for _, b1 := range res { //遍历对象数组
+			if b1 != nil {
+				var info ProjectInfo
+				err := json.Unmarshal(b1.([]byte), &info)
+				if err != nil {
+					log.Println(err)
+				}
+				cone := &CompareOne{
+					Parent: com,
+					Pinfo:  &info,
+				}
+				if com.Field != "pn" { //比较字段不是项目名称时，对项目名称要素的打分逻辑
+					//新信息和对比信息的项目名称长度都大于5时
+					if len([]rune(thisinfo.ProjectName)) > 5 && len([]rune(info.ProjectName)) > 5 {
+						if thisinfo.ProjectName == info.ProjectName { //绝对相等加5分
+							cone.ProjectNameType = 1
+							cone.Score += 5
+						} else if strings.Index(info.ProjectName, thisinfo.ProjectName) > -1 || strings.Index(thisinfo.ProjectName, info.ProjectName) > -1 { //包含或被包含关系不加分不减分
+							cone.ProjectNameType = 2
+							//cone.Score += 3
+						} else { //不相等且不包含减5分
+							cone.ProjectNameType = 3
+							cone.Score -= 10
+						}
+					}
+				}
+				if com.Field != "pc" { //比较字段不是项目编号时，对项目编号要素的打分逻辑
+					//新信息和对比信息和项目编号长度大于5
+					if len(thisinfo.ProjectCode) > 5 && len(info.ProjectCode) > 5 {
+						if thisinfo.ProjectCode == info.ProjectCode { //项目编号相等加5分
+							cone.ProjectCodeType = 1
+							cone.Score += 5
+						} else if strings.Index(info.ProjectCode, thisinfo.ProjectCode) > -1 || strings.Index(thisinfo.ProjectCode, info.ProjectCode) > -1 { //包含或被包含关系加3分
+							cone.ProjectCodeType = 2
+							cone.Score += 3
+						} else {
+							//不相等减5分
+							cone.ProjectCodeType = 3
+							cone.Score -= 5
+							if len(thisinfo.ProjectCode) == len(info.ProjectCode) { //值不相等但长度相等减20分
+								cone.Score -= 20
+							}
+						}
+					}
+				}
+				if com.Field != "pb" { //比较字段不是采购单位时，对采购单位要素的打分逻辑
+					//采购单位大于4个字时做比较
+					if len(thisinfo.Buyer) > 12 && len(info.Buyer) > 12 {
+						if thisinfo.Buyer == info.Buyer { //相等加5分
+							cone.BuyerType = 1
+							cone.Score += 5
+						} else if strings.Index(thisinfo.Buyer, info.Buyer) > -1 || strings.Index(info.Buyer, thisinfo.Buyer) > -1 { //包含与被包含加3分
+							cone.BuyerType = 2
+							cone.Score += 3
+						} else { //不相等减5分
+							cone.BuyerType = 3
+							cone.Score -= 5
+						}
+					}
+				}
+				//对发布时间打分，两者信息发布时间在120天以内加2分
+				if math.Abs(float64(thisinfo.Publishtime-info.Publistime[0])) < 120*86400 && math.Abs(float64(thisinfo.Publishtime-info.Publistime[len(info.Publistime)-1])) < 120*86400 {
+					cone.PublistimeType = 1
+					//cone.Score += 2
+				} else { //发布时间相差超过120天减5分
+					cone.PublistimeType = 2
+					cone.Score -= 5
+				}
+
+				//对招标机构进行打分
+				//招标机构长度大于4分字时做比较
+				if len(thisinfo.Agency) > 12 && len(info.Agency) > 12 {
+					if thisinfo.Agency == info.Agency { //相等加3分
+						cone.AgencyType = 1
+						cone.Score += 3
+					} else if strings.Index(thisinfo.Agency, info.Agency) > -1 || strings.Index(info.Agency, thisinfo.Agency) > -1 { //包含关系不加分
+						cone.AgencyType = 2
+					} else { //不相等减5分
+						cone.AgencyType = 3
+						cone.Score -= 5
+					}
+				}
+				//对地区进行打分，地区都不为空，且范围都不是全国时做比较
+				if thisinfo.Area != "" && info.Area != "" && thisinfo.Area != "A" && thisinfo.Area != "全国" && info.Area != "A" && info.Area != "全国" {
+					if thisinfo.Area == info.Area { //地区相同不加分
+						cone.AreaType = 1
+					} else { //地区不同减5分
+						cone.AreaType = 2
+						cone.Score -= 5
+					}
+				}
+				//分包类型相同时
+				if thisinfo.HasPackage == info.HasPackage {
+					if thisinfo.HasPackage {
+						cone.PackageType = 1
+					} else { //都不是多包
+						cone.PackageType = 2
+					}
+					//都是分包或都不是分包加4分
+					cone.Score += 4
+				} else { //分包不一致时
+					if thisinfo.TopType == "招标" { //新信息类型为招标时
+						if thisinfo.HasPackage && len(thisinfo.Package) == 1 { //当前信息是招标类，且是一个包，对比信息不是分包，认为它们都不是分包，加2分
+							cone.PackageType = 3
+							cone.Score += 2
+						} else if !thisinfo.HasPackage { //当前信息不是分包，合并信息是分包（要从标题中查找），不加分不减分
+							cone.PackageType = 31
+						} else { //当前信息是分包，合并信息不是分包，不加减分
+							cone.PackageType = 32
+						}
+					} else if thisinfo.TopType == "结果" { //新信息类型为结果时
+						if thisinfo.HasPackage && len(thisinfo.Package) == 1 { //从标题中查找有没有分包 ，当前信息结果类是一个分包，合并信息不是分包，加2分
+							cone.PackageType = 4
+							cone.Score += 2
+						} else if !thisinfo.HasPackage { //当前信息是结果，不是分包，而合并信息是分包，不加分
+							cone.PackageType = 41
+						} else {
+							cone.PackageType = 42 //当前信息是分包，而合并信息不是分包，不加减分
+						}
+					}
+				}
+				//cone.Pos = k
+				//放入打分数组中
+				com.Scores = append(com.Scores, cone)
+			}
+		}
+	}
+}
+**/
+//二分字符串查找
+func BinarySearch(s []string, k string) int {
+	sort.Strings(s)
+	lo, hi := 0, len(s)-1
+	for lo <= hi {
+		m := (lo + hi) >> 1
+		if s[m] < k {
+			lo = m + 1
+		} else if s[m] > k {
+			hi = m - 1
+		} else {
+			return m
+		}
+	}
+	return -1
+}
+
+//移除数组中重复的元素
+func RemoveDup(arr []string) (newarr []string) {
+	m1 := map[string]bool{}
+	newarr = []string{}
+	for _, k := range arr {
+		if !m1[k] {
+			m1[k] = true
+			newarr = append(newarr, k)
+		}
+	}
+	return
+}

udpprojectset/src/config.json

@@ -0,0 +1,28 @@
+{
+    "mongodbServers": "192.168.3.207:27082",
+    "mongodbPoolSize": 100,
+    "mongodbName": "qfw",
+    "jkmail": {
+        "to": "renzheng@topnet.net.cn",
+        "api": "http://10.171.112.160:19281/_send/_mail"
+    },
+    "thread": 300,
+    "extractColl": "extract_v2.0",
+    "projectColl": "projectset",
+    "lenprojectname": 18,
+    "redisPoolSize": 300,
+    "redisaddrs": "ids=192.168.3.207:3379,keys=192.168.3.207:3378,info=192.168.3.207:3378",
+    "clearedis": {
+        "open": true,
+        "clearcron": "0 10 15 ? * 4",
+        "projectcycle": 180
+    },
+    "udpport": ":1482",
+    "nextNode": [
+        {
+            "addr": "127.0.0.1",
+            "port": 1483,
+            "memo": "创建项目索引"
+        }
+    ]
+}

udpprojectset/src/datamonitor.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"sync"
+	"time"
+)
+
+/**
+增加数据监控服务，每15分钟保存一次抽取情况
+**/
+var IS = &InfoStatus{
+	time.Now().Unix(), 0, map[string]int{}, &sync.Mutex{}}
+
+func init() {
+	go IS.Save()
+}
+
+type InfoStatus struct {
+	Starttime int64
+	Endtime   int64
+	Val       map[string]int
+	Lock      *sync.Mutex
+}
+
+func (is *InfoStatus) Add(t string) {
+	is.Lock.Lock()
+	is.Val[t]++
+	is.Lock.Unlock()
+}
+
+func (is *InfoStatus) Save() {
+	is.Lock.Lock()
+	is.Endtime = time.Now().Unix()
+	save := map[string]interface{}{}
+	all := 0
+	for _, v := range is.Val {
+		all += v
+	}
+	if all > 0 {
+		save["receive"] = all
+		save["starttime"] = is.Starttime
+		save["endtime"] = is.Endtime
+		save["flag"] = "projectser"
+		is.Val = map[string]int{}
+		go MQFW.Save("datamonitor", save)
+	}
+	is.Starttime = is.Endtime
+	is.Lock.Unlock()
+	time.AfterFunc(15*time.Minute, is.Save)
+}

udpprojectset/src/extractarea.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"qfw/util"
+)
+
+var CityConfig map[string]interface{}
+var AreaGet DFA //敏感词
+
+func initarea() {
+	util.ReadConfig("city.json", &CityConfig)
+	cp := CityConfig["city"].(map[string]interface{})
+	citys := []string{}
+	for k, _ := range cp {
+		citys = append(citys, k)
+	}
+	AreaGet = DFA{}
+	AreaGet.AddWord(citys...)
+}
+
+type DFA struct {
+	Link map[string]interface{}
+}
+
+func (d *DFA) AddWordAll(haskey bool, keys ...string) {
+	if d.Link == nil {
+		d.Link = make(map[string]interface{})
+	}
+	for _, key := range keys {
+		nowMap := &d.Link
+		for i := 0; i < len(key); i++ {
+			kc := key[i : i+1]
+			if v, ok := (*nowMap)[kc]; ok {
+				nowMap, _ = v.(*map[string]interface{})
+			} else {
+				newMap := map[string]interface{}{}
+				newMap["YN"] = "0"
+				(*nowMap)[kc] = &newMap
+				nowMap = &newMap
+			}
+			if i == len(key)-1 {
+				(*nowMap)["YN"] = "1"
+				if haskey {
+					(*nowMap)["K"] = key
+				}
+			}
+		}
+	}
+}
+
+func (d *DFA) AddWord(keys ...string) {
+	d.AddWordAll(false, keys...)
+}
+
+//适合一次查找
+func (d *DFA) CheckSensitiveWord(src string) (b bool) {
+	pos := 0
+	nowMap := &d.Link
+	for i := 0; i < len(src); i++ {
+		word := src[i : i+1]
+		nowMap, _ = (*nowMap)[word].(*map[string]interface{})
+		if nowMap != nil { // 存在，则判断是否为最后一个
+			if pos == 0 {
+				pos = i
+			}
+			if "1" == util.ObjToString((*nowMap)["YN"]) { // 如果为最后一个匹配规则,结束循环，返回匹配标识数
+				b = true
+				break
+			}
+		} else {
+			nowMap = &d.Link
+			if pos > 0 {
+				i = pos
+				pos = 0
+			}
+		}
+	}
+	return
+}

udpprojectset/src/handleproject.go

@@ -0,0 +1,205 @@
+package main
+
+import (
+	"fmt"
+	"qfw/util"
+	"regexp"
+	"strings"
+)
+
+var pnreg = regexp.MustCompile("^(及编号[：:])|(项目|采购|招标|中标|成交|结果|[_]|公告)$")
+var numreg = regexp.MustCompile("^[0-9]$")
+
+/**
+0 项目名称不是多包结构,直接按项目名称查找
+1 项目名称多包,截去多包查找，再按不截取的查找
+2 非招标项目名称含单个子包,截取多包查找，再搂
+**/
+func handleprojectname(pn, buyer, toptype string) (string, int) {
+	for i := 0; i < 4; i++ {
+		opn := pn
+		pn = pnreg.ReplaceAllString(opn, "")
+		if opn == pn {
+			break
+		}
+	}
+	//return pn
+	if len([]rune(pn)) > lenprojectname || strings.Index(pn, buyer) > -1 || AreaGet.CheckSensitiveWord(pn) {
+		return pn, 0
+	} else {
+		return buyer + pn, 0
+	}
+}
+
+//对字段的优先级处理
+type ExtractLevel struct {
+	Strarr []string
+}
+
+func InitEL(v string) (e *ExtractLevel) {
+	if v == "" {
+		v = "0999999999"
+		//第1位一下标为0，不用
+	}
+	Strarr := strings.Split(v, "")
+	e = &ExtractLevel{Strarr}
+	return
+}
+
+func (e *ExtractLevel) PutVal(field, val string) {
+	e.Strarr[extractpos[field]] = val
+}
+
+func (e *ExtractLevel) GetVal() string {
+	return strings.Join(e.Strarr, "")
+}
+
+//字段优先级处理
+func (e *ExtractLevel) fieldpriority(info, project, set *map[string]interface{}) {
+	defer util.Catch()
+	for _k, v := range extractpos {
+		breplace := false
+		var _infov interface{}
+		_infopos := fmt.Sprintf("%d", (*info)[_k+"_w"])
+		if !numreg.MatchString(_infopos) {
+			_infopos = "9"
+		}
+		if project == nil {
+			breplace = true
+			_infov = (*info)[_k]
+		} else {
+			ty := cextractpos[_k]
+			switch ty {
+			case 's':
+				_v := util.ObjToString((*project)[_k])
+				_tv := util.ObjToString((*info)[_k])
+				_infov = _tv
+				if _v == "" && _tv != "" {
+					breplace = true
+				} else if _tv != "" { //对比
+					if e.Strarr[v] > _infopos {
+						breplace = true
+					}
+				}
+			case 'l':
+				_v := util.Float64All((*project)[_k])
+				_tv := util.Float64All((*info)[_k])
+				_infov = _tv
+				if _v == 0 && _tv > 0 {
+					breplace = true
+				} else if _tv > 0 {
+					if e.Strarr[v] > _infopos {
+						breplace = true
+					}
+				}
+
+			}
+		}
+		if breplace {
+			(*set)[_k] = _infov
+			e.PutVal(_k, _infopos)
+		}
+	}
+	otherfieldpriority(info, project, set)
+}
+
+//抽取字段
+var cextractpos = map[string]rune{
+	"projectname":  's',
+	"projectcode":  's',
+	"buyer":        's',
+	"winner":       's',
+	"budget":       'l',
+	"bidamount":    'l',
+	"bidstatus":    's',
+	"agency":       's',
+	"projectscope": 's',
+}
+var extractpos = map[string]int{
+	"projectname":  0,
+	"projectcode":  1,
+	"buyer":        2,
+	"winner":       3,
+	"budget":       4,
+	"bidamount":    5,
+	"bidstatus":    6,
+	"agency":       7,
+	"projectscope": 8,
+}
+
+//合并字段
+//"area","city","topscopeclass","subscopeclass","bidtype","zbtime","jgtime","buyerclass"
+var bidtype = map[string]string{
+	"招标": "招标",
+	"邀标": "邀标",
+	"询价": "询价",
+	"竞谈": "竞谈",
+	"单一": "单一",
+	"竞价": "竞价",
+}
+
+func otherfieldpriority(info, project, set *map[string]interface{}) {
+	area2 := util.ObjToString((*info)["area"])
+	infoformat := util.IntAll((*info)["infoformat"])
+	toptype := util.ObjToString((*info)["toptype"])
+	topscopeclass := util.ObjToString((*info)["topscopeclass"])
+	subscopeclass := util.ObjToString((*info)["subscopeclass"])
+	bt := bidtype[util.ObjToString((*info)["subtype"])]
+	if project == nil {
+		if area2 == "" || area2 == "全国" {
+			area2 = "A"
+		}
+		(*set)["area"] = area2
+		(*set)["city"] = util.ObjToString((*info)["city"])
+		(*set)["topscopeclass"] = topscopeclass
+		(*set)["subscopeclass"] = subscopeclass
+		if bt == "" {
+			bt = "招标"
+		}
+		(*set)["bidtype"] = bt
+		if infoformat == 2 {
+			(*set)["njtime"] = (*info)["publishtime"]
+		} else {
+			if toptype == "招标" {
+				(*set)["zbtime"] = (*info)["publishtime"]
+			} else if toptype == "结果" {
+				(*set)["jgtime"] = (*info)["publishtime"]
+			}
+		}
+		if (*set)["zbtime"] == nil {
+			(*set)["zbtime"] = (*info)["publishtime"]
+		}
+	} else {
+		area := util.ObjToString((*project)["area"])
+		if area == "A" && (area2 != "" && area2 != "A" && area2 != "全国") {
+			(*set)["area"] = area2
+		}
+		publishtime := util.Int64All((*info)["publishtime"])
+		if infoformat == 2 {
+			//拟建信息，先不判断
+			if publishtime > 0 {
+				(*set)["njtime"] = publishtime
+			}
+		} else {
+			//招标信息
+			if toptype == "招标" {
+				zbtime := util.Int64All((*project)["zbtime"])
+				if publishtime > 0 && publishtime < zbtime {
+					(*set)["zbtime"] = publishtime
+				}
+			} else if toptype == "结果" {
+				jgtime := util.Int64All((*project)["jgtime"])
+				if publishtime > 0 && publishtime > jgtime {
+					(*set)["jgtime"] = publishtime
+				}
+			}
+		}
+		if subscopeclass != "" {
+			(*set)["topscopeclass"] = topscopeclass
+			(*set)["subscopeclass"] = subscopeclass
+		}
+		if bt != "" && bt != "招标" {
+			(*set)["bidtype"] = bt
+		}
+	}
+}

udpprojectset/src/main.go

@@ -0,0 +1,287 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	//"math"
+	mu "mfw/util"
+	"net"
+	//"os"
+	du "jy/util"
+	"qfw/util"
+	"qfw/util/mongodb"
+	"qfw/util/redis"
+	"sync"
+	"time"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+/*
+从extract_result表中抽到projectset表中，根据3选2判断是不是同一条信息，抽取信息至少有两个字段。
+可以使用redis
+//对时间的判断
+*/
+var MultiThread chan bool
+var (
+	Sysconfig                map[string]interface{}
+	MQFW                     mongodb.MongodbSim
+	extractColl, projectColl string
+	lenprojectname           int
+	udpclient                mu.UdpClient             //udp对象
+	nextNode                 []map[string]interface{} //下节点数组
+	toaddr                   = []*net.UDPAddr{}       //下节点对象
+)
+
+const (
+	REDISIDS    = "ids"
+	REDISKEYS   = "keys"
+	INFOID      = "info"
+	INFOTIMEOUT = 86400 * 30
+)
+
+func init() {
+	initarea()
+	du.SetConsole(false)
+	du.SetRollingDaily("./", "project.log")
+	du.SetLevel(du.DEBUG)
+	util.ReadConfig(&Sysconfig)
+	MultiThread = make(chan bool, util.IntAllDef(Sysconfig["thread"], 200))
+	lenprojectname = util.IntAllDef(Sysconfig["lenprojectname"], 20) - 1
+	redis.InitRedisBySize(Sysconfig["redisaddrs"].(string), util.IntAllDef(Sysconfig["redisPoolSize"], 100), 30, 300)
+	MQFW = mongodb.MongodbSim{
+		MongodbAddr: Sysconfig["mongodbServers"].(string),
+		Size:        util.IntAll(Sysconfig["mongodbPoolSize"]),
+		DbName:      Sysconfig["mongodbName"].(string),
+	}
+	MQFW.InitPool()
+	extractColl = Sysconfig["extractColl"].(string)
+	projectColl = Sysconfig["projectColl"].(string)
+	nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
+	for _, m := range nextNode {
+		toaddr = append(toaddr, &net.UDPAddr{
+			IP:   net.ParseIP(m["addr"].(string)),
+			Port: util.IntAll(m["port"]),
+		})
+	}
+}
+
+type KeyMap struct {
+	Lock sync.Mutex
+	Map  map[string]*Key
+}
+
+type Key struct {
+	Arr  *[]string
+	Lock *sync.Mutex
+}
+
+func NewKeyMap() *KeyMap {
+	//k:=&Key{[]string{}}
+	return &KeyMap{
+		Map: map[string]*Key{},
+	}
+}
+
+//三组lock,对应的（PNKey）key为项目名称,值对应的是此项目名称对应的项目id数组
+var PNKey, PCKey, PBKey = NewKeyMap(), NewKeyMap(), NewKeyMap()
+
+//id-lock
+var IdLock = &sync.Mutex{}
+
+func main() {
+	go checkMapJob()
+	//先加载所有锁
+	log.Println("loading data from redis...")
+	n := 0
+	km := []*KeyMap{PNKey, PCKey, PBKey}
+	for pos, key := range []string{"pn_*", "pc_*", "pb_*"} {
+		res := redis.GetKeysByPattern(REDISKEYS, key)
+		pk := km[pos]
+		if res != nil {
+			//一次500条
+			num := 0
+			arr := []string{}
+			for _, v := range res {
+				n++
+				num++
+				k := string(v.([]uint8))
+				arr = append(arr, k) //根据正则找到key
+				if num == 500 {
+					num = 0
+					ret := redis.Mget(REDISKEYS, arr) //根据key批量取内容
+					if len(ret) > 0 {
+						for k1, v1 := range ret {
+							if v1 != nil {
+								var a1 []string
+								json.Unmarshal(v1.([]uint8), &a1)
+								pk.Map[arr[k1]] = &Key{&a1, &sync.Mutex{}} //pn_项目名称 id数组
+							}
+						}
+					}
+					arr = []string{}
+				}
+			}
+			if num > 0 {
+				ret := redis.Mget(REDISKEYS, arr)
+				if len(ret) > 0 {
+					for k1, v1 := range ret {
+						if v1 != nil {
+							var a1 []string
+							json.Unmarshal(v1.([]uint8), &a1)
+							pk.Map[arr[k1]] = &Key{&a1, &sync.Mutex{}}
+						}
+					}
+				}
+				arr = []string{}
+			}
+		}
+	}
+	log.Println("load data from redis finished.", n)
+	//清理redis
+	//clearedis()
+	updport := Sysconfig["udpport"].(string)
+	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Println("Udp服务监听", updport)
+	time.Sleep(99999 * time.Hour)
+}
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA: //上个节点的数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Println("err:", err, "mapInfo:", mapInfo)
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			key, _ := mapInfo["key"].(string)
+			if key == "" {
+				key = "udpok"
+			}
+			go udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
+			SingleThread <- true
+			go task(data, mapInfo)
+		}
+	case mu.OP_NOOP: //下个节点回应
+		ok := string(data)
+		if ok != "" {
+			log.Println("ok:", ok)
+			udptaskmap.Delete(ok)
+		}
+	}
+}
+
+var SingleThread = make(chan bool, 1)
+
+func task(data []byte, mapInfo map[string]interface{}) {
+	defer func() {
+		<-SingleThread
+	}()
+	defer util.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": map[string]interface{}{
+				"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	sess := MQFW.GetMgoConn()
+	defer MQFW.DestoryMongoConn(sess)
+	//数据正序处理
+	it := sess.DB(MQFW.DbName).C(extractColl).Find(&q).Iter() //.Sort("publishtime")
+	count, index := 0, 0
+	pici := time.Now().Unix()
+	//log.Println(q, MQFW.DbName, extractColl)
+	wg := &sync.WaitGroup{}
+	idmap := &sync.Map{}
+	for tmp := make(map[string]interface{}); it.Next(tmp); {
+		//		if util.IntAll(tmp["extracttype"]) == -1 {
+		//			tmp = make(map[string]interface{})
+		//			continue
+		//		}
+		if index%10000 == 0 {
+			log.Println(index, tmp["_id"])
+		}
+		index++
+		if util.IntAll(tmp["repeat"]) == 1 {
+			tmp = make(map[string]interface{})
+			continue
+		}
+		count++
+		thisid := util.BsonIdToSId(tmp["_id"])
+		b, err := redis.Exists(INFOID, thisid)
+		if err != nil {
+			log.Println("checkid err", err.Error())
+		}
+		if !b { // && util.IntAll(tmp["infoformat"]) < 2 && tmp["detail"] != nil {
+			//增加判重逻辑,重复id不再生成
+			wg.Add(1)
+			idmap.Store(tmp["_id"], true)
+			MultiThread <- true
+			go func(tmp map[string]interface{}, thisid string) {
+				defer func() {
+					<-MultiThread
+					wg.Done()
+					idmap.Delete(tmp["_id"])
+				}()
+				sflag := Compare(tmp, pici)
+				redis.Put(INFOID, thisid, 1, INFOTIMEOUT)
+				go IS.Add(sflag)
+			}(tmp, thisid)
+		}
+		if count%500 == 0 {
+			log.Println("count:", count)
+		}
+		tmp = make(map[string]interface{})
+	}
+	for {
+		time.Sleep(5 * time.Second)
+		n := 0
+		idmap.Range(func(key interface{}, v interface{}) bool {
+			n++
+			log.Println(key, v)
+			return true
+		})
+		if n < 1 {
+			break
+		}
+	}
+	wg.Wait()
+	log.Println("task over...", index, count)
+	//发送udp,调用生成项目索引
+	mapInfo["stype"] = "project"
+	if mapInfo["stop"] == nil && len(toaddr) > 0 {
+		for n, to := range toaddr {
+			key := fmt.Sprintf("%d-%s-%d", pici, "project", n)
+			mapInfo["query"] = map[string]interface{}{
+				"pici": pici,
+			}
+			mapInfo["key"] = key
+			datas, _ := json.Marshal(mapInfo)
+			node := &udpNode{datas, to, time.Now().Unix(), 0}
+			udptaskmap.Store(key, node)
+			udpclient.WriteUdp(datas, mu.OP_TYPE_DATA, to)
+		}
+	}
+}
+
+func NewPushInfo(tmp map[string]interface{}) bson.M {
+	return bson.M{
+		"comeintime":  tmp["comeintime"],
+		"publishtime": tmp["publishtime"],
+		"title":       tmp["title"],
+		"toptype":     tmp["toptype"],
+		"subtype":     tmp["subtype"],
+		"infoformat":  tmp["infoformat"],
+		"infoid":      util.BsonIdToSId(tmp["_id"]),
+		"href":        tmp["href"],
+		"area":        tmp["area"],
+		"city":        tmp["city"],
+		"cresult":     tmp["cresult"],
+	}
+}

udpprojectset/src/main_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"testing"
+)
+
+func Test_main(t *testing.T) {
+	m1 := map[string]*[]string{}
+	s := m1["dd"]
+	log.Println("aaaaa1")
+	func() {
+		if s == nil {
+			defer func() {
+				log.Println("defer")
+			}()
+			s = &[]string{}
+			m1["dd"] = s
+			log.Println("aaa3")
+		}
+	}()
+	n := 0
+	for {
+		defer func(n int) {
+			log.Println("for", n)
+		}(n)
+		n++
+		log.Println(n)
+		if n > 2 {
+			break
+		}
+	}
+	log.Println("aaa2")
+	*s = append(*s, "aa")
+	log.Println("---m1[dd]:", m1["dd"])
+
+	//arr := []string{"aa", "bb", "cc", "dd", "ee", "abdsfsdfsd"}
+	//log.Println("二分查找", binarySearch(arr, "bb"))
+}
+func Test_3t2(t *testing.T) {
+	pcb := [][]string{
+		[]string{"A", "B"},
+		[]string{"A", "B"},
+		[]string{"A", "B"},
+	}
+	for _, v1 := range pcb[0] {
+		tmp := []string{}
+		for _, v2 := range pcb[1] {
+			for _, v3 := range pcb[1] {
+				key := fmt.Sprintf("%s%s%s", v1, v2, v3)
+				tmp = append(tmp, key)
+			}
+		}
+		log.Println(tmp)
+	}
+
+}

udpprojectset/src/scores.go

@@ -0,0 +1,231 @@
+// scores
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"qfw/util"
+	"qfw/util/redis"
+	"strings"
+)
+
+//有效值三选一、三选二
+var ThreeToTow, ThreeToOne map[string]bool
+
+func init() {
+	ThreeToTow = map[string]bool{}
+	ThreeToOne = map[string]bool{
+		"AAA": true,
+		"AAB": true,
+	}
+	tows := [][][]string{
+		[][]string{
+			[]string{"A", "B"},
+			[]string{"A", "B"},
+			[]string{"A", "B"},
+		},
+		[][]string{
+			[]string{"A", "B"},
+			[]string{"D", "E"},
+			[]string{"A", "B"},
+		},
+		[][]string{
+			[]string{"A", "B"},
+			[]string{"A", "B"},
+			[]string{"D", "E"},
+		},
+		[][]string{
+			[]string{"D", "E"},
+			[]string{"A", "B"},
+			[]string{"A", "B"},
+		},
+	}
+	for _, tow := range tows {
+		for _, v1 := range tow[0] {
+			for _, v2 := range tow[1] {
+				for _, v3 := range tow[2] {
+					key := fmt.Sprintf("%s%s%s", v1, v2, v3)
+					ThreeToTow[key] = true
+				}
+			}
+		}
+	}
+}
+
+//三选一打分
+func (com *CompareInfo) ComputeOne(thisinfo *Info, ids []string) []*CompareOne {
+	defer util.Catch()
+	res := redis.Mget(REDISIDS, ids)
+	scores := []*CompareOne{}
+	if len(res) > 0 { //找到项目名称、项目编号或采购单位相同时
+		for _, b1 := range res { //遍历对象数组
+			if b1 != nil {
+				var info ProjectInfo
+				err := json.Unmarshal(b1.([]byte), &info)
+				if err != nil {
+					log.Println(err)
+				}
+				cone := &CompareOne{
+					Parent: com,
+					Pinfo:  &info,
+				}
+				if com.Field == "pn" { //比较字段项目名称
+					if len(info.ProjectName) > 0 {
+						if thisinfo.ProjectName == info.ProjectName { //A
+							cone.Score += 2
+							cone.ProjectNameType = "A"
+						} else if strings.Index(info.ProjectName, thisinfo.ProjectName) > -1 || strings.Index(thisinfo.ProjectName, info.ProjectName) > -1 { //B
+							cone.Score += 1
+							cone.ProjectNameType = "B"
+						} else { //C
+							cone.Score -= 2
+							cone.ProjectNameType = "C"
+						}
+					} else { //D不计分
+						cone.ProjectNameType = "D"
+					}
+				}
+				if com.Field == "pc" { //比较字段项目编号
+					if len(info.ProjectCode) > 0 {
+						if thisinfo.ProjectCode == info.ProjectCode { //A
+							cone.Score += 2
+							cone.ProjectCodeType = "A"
+						} else if strings.Index(info.ProjectCode, thisinfo.ProjectCode) > -1 || strings.Index(thisinfo.ProjectCode, info.ProjectCode) > -1 { //B
+							cone.Score += 1
+							cone.ProjectCodeType = "B"
+						} else { //C
+							cone.Score -= 2
+							cone.ProjectCodeType = "C"
+						}
+					} else { //D不计分
+						cone.ProjectCodeType = "D"
+					}
+				}
+				if thisinfo.Area != "A" && thisinfo.Area != "全国" && info.Area != "A" && info.Area != "全国" {
+					if thisinfo.Area == info.Area && thisinfo.District == info.District {
+						cone.Score += 2
+						cone.AreaType = "A"
+					} else {
+						cone.Score -= 1
+						cone.AreaType = "C"
+					}
+				} else {
+					cone.Score += 1
+					cone.AreaType = "B"
+				}
+				if len([]rune(info.Agency)) > 0 {
+					if thisinfo.Agency == info.Agency { //A
+						cone.Score += 2
+						cone.AgencyType = "A"
+					} else if strings.Index(info.Agency, thisinfo.Agency) > -1 || strings.Index(thisinfo.Agency, info.Agency) > -1 { //B
+						cone.Score += 1
+						cone.AgencyType = "B"
+					} else {
+						if len(thisinfo.Agency) < 1 { //E
+							cone.Score -= 1
+							cone.AgencyType = "E"
+						} else { //C
+							cone.Score -= 2
+							cone.AgencyType = "C"
+						}
+					}
+				} else { //D不计分
+					cone.AgencyType = "D"
+				}
+				scores = append(scores, cone)
+			}
+		}
+	}
+	return scores
+}
+
+//三选二打分
+func (com *CompareInfo) ComputeTow(thisinfo *Info, ids []string, hasbyer bool) []*CompareOne {
+	defer util.Catch()
+	res := redis.Mget(REDISIDS, ids)
+	scores := []*CompareOne{}
+	if len(res) > 0 { //找到项目名称、项目编号或采购单位相同时
+		for _, b1 := range res { //遍历对象数组
+			if b1 != nil {
+				var info ProjectInfo
+				err := json.Unmarshal(b1.([]byte), &info)
+				if err != nil {
+					log.Println(err)
+				}
+				cone := &CompareOne{
+					Parent: com,
+					Pinfo:  &info,
+				}
+				cone.BuyerType, cone.Score = fieldPCBScore(thisinfo.Buyer, info.Buyer, cone.BuyerType, cone.Score)
+				if hasbyer {
+					cone.ProjectNameType, cone.Score = fieldPCBScore(thisinfo.ProjectName, info.ProjectName, cone.ProjectNameType, cone.Score)
+					cone.ProjectCodeType, cone.Score = fieldPCBScore(thisinfo.ProjectCode, info.ProjectCode, cone.ProjectCodeType, cone.Score)
+				} else { //无采购单位，打分考虑长度
+					if len([]rune(thisinfo.ProjectName)) > 5 {
+						cone.ProjectNameType, cone.Score = fieldPCBScore(thisinfo.ProjectName, info.ProjectName, cone.ProjectNameType, cone.Score)
+					} else {
+						cone.ProjectNameType = "D"
+					}
+					if len(thisinfo.ProjectCode) > 6 {
+						cone.ProjectCodeType, cone.Score = fieldPCBScore(thisinfo.ProjectCode, info.ProjectCode, cone.ProjectCodeType, cone.Score)
+					} else {
+						cone.ProjectCodeType = "D"
+					}
+				}
+				//省市打分
+				if thisinfo.Area != "A" && thisinfo.Area != "全国" && info.Area != "A" && info.Area != "全国" {
+					if thisinfo.Area == info.Area && thisinfo.City == info.City {
+						cone.Score += 2
+					} else {
+						cone.Score -= 1
+					}
+				} else {
+					cone.Score += 1
+				}
+				//代理机构打分
+				if len([]rune(info.Agency)) > 0 {
+					if thisinfo.Agency == info.Agency { //A
+						cone.Score += 2
+					} else if strings.Index(info.Agency, thisinfo.Agency) > -1 || strings.Index(thisinfo.Agency, info.Agency) > -1 { //B
+						cone.Score += 1
+					} else {
+						if len(thisinfo.Agency) < 1 { //E
+							cone.Score -= 1
+						} else { //C
+							cone.Score -= 2
+						}
+					}
+				} else { //D不计分
+					//
+				}
+				scores = append(scores, cone)
+			}
+		}
+	}
+	return scores
+}
+
+//项目名称、项目编号、采购单位打分
+func fieldPCBScore(this, info, ctype string, score int) (string, int) {
+	if len(info) > 0 {
+		if this == info { //A
+			score += 5
+			ctype = "A"
+		} else if strings.Index(info, this) > -1 || strings.Index(this, info) > -1 { //B
+			score += 2
+			ctype = "B"
+		} else {
+			if len(this) < 1 { //E
+				score -= 1
+				ctype = "E"
+			} else { //C
+				score -= 2
+				ctype = "C"
+			}
+		}
+	} else { //D不计分
+		ctype = "D"
+	}
+	return ctype, score
+}

udpprojectset/src/thisinfo.go

@@ -0,0 +1,322 @@
+// thisinfo
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	du "jy/util"
+	"log"
+	"qfw/util"
+	"strings"
+
+	"gopkg.in/mgo.v2/bson"
+)
+
+//抽取信息映射实体类
+type Info struct {
+	Id            string                 `json:"_id"`
+	Href          string                 `json:"href"`
+	Publishtime   int64                  `json:"publishtime"`
+	Title         string                 `json:"title"`
+	TopType       string                 `json:"toptype"`
+	SubType       string                 `json:"subtype"`
+	ProjectName   string                 `json:"projectname"`
+	ProjectCode   string                 `json:"projectcode"`
+	Buyer         string                 `json:"buyer"`
+	Buyerperson   string                 `json:"buyerperson"`
+	Buyertel      string                 `json:"buyertel"`
+	Agency        string                 `json:"agency"`
+	Area          string                 `json:"area"`
+	City          string                 `json:"city"`
+	HasPackage    bool                   `json:"haspackage"`
+	Package       map[string]interface{} `json:"package"`
+	PNum          string                 `json:"pnum"`
+	Topscopeclass []string               `json:"topscopeclass"`
+	Subscopeclass []string               `json:"subscopeclass"`
+	Winners       []string
+	dealtype      int
+
+	Buyerclass  string `json:"buyerclass"`
+	Bidopentime int64  `json:"bidopentime"`
+	District    string `json:"district"`
+	Winnerorder []string
+}
+
+//pcb三选值
+type PCBV struct {
+	Val    int  //1一项有效值,2二项有效值,3三项有效值
+	Pname  bool //有项目名称
+	Cname  bool //有项目编号
+	Bname  bool //有采购单位
+	Area   bool //区域
+	City   bool //市
+	Agency bool //代理机构
+
+	PnameLen int //值长度
+	CnameLen int //值长度
+	BnameLen int //值长度
+}
+
+func PreThisInfo(tmp map[string]interface{} /*新信息*/) *Info {
+	bys, _ := json.Marshal(tmp)
+	var thisinfo *Info
+	json.Unmarshal(bys, &thisinfo)
+	if thisinfo == nil {
+		return nil
+	}
+	if len(thisinfo.Topscopeclass) == 0 {
+		thisinfo.Topscopeclass = []string{}
+	}
+	if len(thisinfo.Subscopeclass) == 0 {
+		thisinfo.Subscopeclass = []string{}
+	}
+	//去重
+	thisinfo.Subscopeclass = RemoveDup(thisinfo.Subscopeclass)
+
+	if len(thisinfo.Package) > 0 { //信息是否分包
+		thisinfo.HasPackage = true
+	} else if thisinfo.TopType == "结果" && TitleReg.MatchString(thisinfo.Title) {
+		//当信息类型是结果时，并且标题中包含分包字样，找到包号，用以后面比较打分
+		res := TitleReg.FindAllStringSubmatch(thisinfo.Title, -1)
+		pnum := du.PackageNumberConvert(res[0][0])
+		//du.Debug(pnum, res)
+		thisinfo.PNum = pnum
+	}
+	if thisinfo.SubType == "变更" || strings.Index(thisinfo.Title, "变更公告") > -1 || strings.Index(thisinfo.Title, "更正公告") > -1 {
+		//当信息类型是变更或标题中含变更时
+		if thisinfo.TopType == "招标" {
+			//招标的变更公告过，不作处理
+			//du.Debug(thisinfo.Id, thisinfo.Href, thisinfo.ProjectName, thisinfo.ProjectCode)
+			return nil
+		} else if thisinfo.TopType == "结果" {
+			thisinfo.SubType = "变更"
+		}
+	}
+
+	//计算中标人
+	winner, _ := tmp["winner"].(string)
+	m1 := map[string]bool{}
+	winners := []string{}
+	if winner != "" {
+		m1[winner] = true
+		winners = append(winners, winner)
+	}
+	if thisinfo.HasPackage {
+		packageM, _ := tmp["package"].(bson.M)
+		for _, p := range packageM {
+			pm, _ := p.(map[string]interface{})
+			pw, _ := pm["winner"].(string)
+			if pw != "" {
+				m1[pw] = true
+				winners = append(winners, pw)
+			}
+		}
+	}
+	thisinfo.Winners = winners
+	m1 = nil
+	//中标候选人
+	winnerorder := []string{}
+	if winorders, ok := tmp["winnerorder"].([]interface{}); ok {
+		for _, wins := range winorders {
+			if win, ok := wins.(map[string]interface{}); ok {
+				entname := util.ObjToString(win["entname"])
+				if entname != "" && len([]rune(entname)) > 6 {
+					winnerorder = append(winnerorder, entname)
+				}
+			}
+		}
+	}
+	thisinfo.Winnerorder = winnerorder
+	return thisinfo
+}
+
+//流程处理
+func (com *CompareInfo) ProcessInfo(new_pn string, tmp map[string]interface{}, pici int64, info *Info) (id string) {
+	pcbv := PCBValue(info)
+	if info.SubType == "变更" {
+		if pcbv.Val > 1 {
+			id = com.Process(new_pn, tmp, pici, info, pcbv)
+		} else {
+			//信息无效
+			MQFW.UpdateById(extractColl, info.Id,
+				map[string]interface{}{
+					"$set": map[string]interface{}{
+						"ext_mess": info.SubType + ",不满足三选二",
+					},
+				})
+			du.Debug("信息无效", info.Id)
+		}
+	} else { //非变更
+		id = com.Process(new_pn, tmp, pici, info, pcbv)
+	}
+	return id
+}
+
+//流程
+func (com *CompareInfo) Process(new_pn string, tmp map[string]interface{}, pici int64, info *Info, pcbv PCBV) (id string) {
+	if pcbv.Bname { //有采购单位
+		if pcbv.Pname || pcbv.Cname {
+			//三选二打分
+			scores := com.ComputeTow(info, com.IdArr, true)
+			for _, v := range scores {
+				skey := fmt.Sprintf("%s%s%s", v.BuyerType, v.ProjectNameType, v.ProjectCodeType)
+				if ThreeToTow[skey] { //判断是否进入打分合并逻辑
+					log.Println(info.Id, skey)
+					v.Cresult = skey
+					com.Scores = append(com.Scores, v)
+				}
+			}
+			if len(scores) < 1 {
+				//新建项目，（Compare方法中创建）不在此创建，其他同理
+				//du.Debug("有采购单位,新建项目")
+			}
+		} else {
+			//生成项目，不参与后续对比
+			id = InsertProject(new_pn, tmp,
+				map[string]interface{}{
+					"ext_mess": map[string]interface{}{
+						"type": "有采购单位,不满足三选二",
+						"mess": "项目名称或项目编号不满足",
+					},
+				}, pici, info)
+			//du.Debug("生成项目，不参与后续对比", id)
+		}
+	} else { //无采购单位
+		if pcbv.Pname {
+			if pcbv.Cname {
+				//三选二打分
+				scores := com.ComputeTow(info, com.IdArr, false)
+				for _, v := range scores {
+					skey := fmt.Sprintf("%s%s%s", v.BuyerType, v.ProjectNameType, v.ProjectCodeType)
+					if ThreeToTow[skey] { //判断是否进入打分合并逻辑
+						v.Cresult = skey
+						com.Scores = append(com.Scores, v)
+					}
+				}
+				if len(scores) < 1 {
+					//新建项目
+					//du.Debug("无采购单位,新建项目")
+				}
+			} else {
+				if pcbv.PnameLen > 5 {
+					if pcbv.Area && pcbv.City {
+						//三选一打分
+						scores := com.ComputeOne(info, com.IdArr)
+						for _, v := range scores { // v.ProjectNameType, v.ProjectCodeType二者有其一
+							skey := fmt.Sprintf("%s%s%s%s", v.ProjectNameType, v.ProjectCodeType, v.AreaType, v.AgencyType)
+							if ThreeToOne[skey] { //判断是否进入打分合并逻辑
+								v.Cresult = skey
+								com.Scores = append(com.Scores, v)
+							}
+						}
+						if len(scores) < 1 {
+							//新建项目
+							du.Debug("新建项目")
+						}
+					} else {
+						//生成项目，不参与后续对比
+						id = InsertProject(new_pn, tmp,
+							map[string]interface{}{
+								"ext_mess": map[string]interface{}{
+									"type": "无采购单位,不满足三选一",
+									"mess": "省市不满足",
+								},
+							},
+							pici, info)
+						//du.Debug("生成项目，不参与后续对比", id)
+					}
+				} else {
+					//生成项目，不参与后续对比
+					id = InsertProject(new_pn, tmp,
+						map[string]interface{}{"ext_mess": map[string]interface{}{
+							"type": "无采购单位,不满足三选一",
+							"mess": "项目名称不满足",
+						},
+						}, pici, info)
+					//du.Debug("生成项目，不参与后续对比", id)
+				}
+			}
+		} else {
+			if pcbv.Cname {
+				if pcbv.CnameLen > 8 {
+					if pcbv.Area && pcbv.City {
+						//三选一打分
+						scores := com.ComputeOne(info, com.IdArr)
+						for _, v := range scores { // v.ProjectNameType, v.ProjectCodeType二者有其一
+							skey := fmt.Sprintf("%s%s%s%s", v.ProjectNameType, v.ProjectCodeType, v.AreaType, v.AgencyType)
+							if ThreeToOne[skey] { //判断是否进入打分合并逻辑
+								v.Cresult = skey
+								com.Scores = append(com.Scores, v)
+							}
+						}
+						if len(scores) < 1 {
+							//新建项目
+							//du.Debug("无采购单位，新建项目")
+						}
+					} else {
+						//生成项目，不参与后续对比
+						id = InsertProject(new_pn, tmp,
+							map[string]interface{}{
+								"ext_mess": map[string]interface{}{
+									"type": "无采购单位,不满足三选一",
+									"mess": "省市不满足",
+								},
+							}, pici, info)
+						//du.Debug("生成项目，不参与后续对比", id)
+					}
+				} else {
+					//生成项目，不参与后续对比
+					id = InsertProject(new_pn, tmp,
+						map[string]interface{}{
+							"ext_mess": map[string]interface{}{
+								"type": "无采购单位,不满足三选一",
+								"mess": "项目编号不满足",
+							},
+						}, pici, info)
+					//du.Debug("生成项目，不参与后续对比", id)
+				}
+			} else {
+				//信息无效
+				MQFW.UpdateById(extractColl, info.Id,
+					map[string]interface{}{
+						"$set": map[string]interface{}{
+							"ext_mess": info.SubType + ",不满足三选一",
+						},
+					})
+				//du.Debug("信息无效", info.Id)
+			}
+		}
+	}
+	return id
+}
+func PCBValue(info *Info) PCBV {
+	pcbv := PCBV{}
+	if len([]rune(info.ProjectName)) > 1 {
+		pcbv.Val += 1
+		pcbv.Pname = true
+		pcbv.PnameLen = len([]rune(info.ProjectName))
+
+	}
+	if len([]rune(info.ProjectCode)) > 1 {
+		pcbv.Val += 1
+		pcbv.Cname = true
+		pcbv.CnameLen = len([]rune(info.ProjectCode))
+
+	}
+	if len([]rune(info.Buyer)) > 1 {
+		pcbv.Val += 1
+		pcbv.Bname = true
+		pcbv.BnameLen = len([]rune(info.Buyer))
+	}
+	//省市代理机构
+	if info.Area != "" {
+		pcbv.Area = true
+	}
+	if info.City != "" {
+		pcbv.City = true
+	}
+	if info.Agency != "" {
+		pcbv.Agency = true
+	}
+	return pcbv
+}

udpprojectset/src/udptaskmap.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	mu "mfw/util"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+)
+
+var udptaskmap = &sync.Map{}
+var tomail string
+var api string
+
+type udpNode struct {
+	data      []byte
+	addr      *net.UDPAddr
+	timestamp int64
+	retry     int
+}
+
+func checkMapJob() {
+	//阿里云内网无法发送邮件
+	jkmail, _ := Sysconfig["jkmail"].(map[string]interface{})
+	if jkmail != nil {
+		tomail, _ = jkmail["to"].(string)
+		api, _ = jkmail["api"].(string)
+	}
+	log.Println("start checkMapJob", tomail, Sysconfig["jkmail"])
+	for {
+		udptaskmap.Range(func(k, v interface{}) bool {
+			now := time.Now().Unix()
+			node, _ := v.(*udpNode)
+			if now-node.timestamp > 120 {
+				node.retry++
+				if node.retry > 5 {
+					log.Println("udp重试失败", k)
+					udptaskmap.Delete(k)
+					res, err := http.Get(fmt.Sprintf("%s?to=%s&title=%s&body=%s", api, tomail, "extract-send-fail", k.(string)))
+					if err == nil {
+						defer res.Body.Close()
+						read, err := ioutil.ReadAll(res.Body)
+						log.Println("邮件发发送：", string(read), err)
+					}
+				} else {
+					log.Println("udp重发", k)
+					udpclient.WriteUdp(node.data, mu.OP_TYPE_DATA, node.addr)
+				}
+			} else if now-node.timestamp > 10 {
+				log.Println("udp任务超时中..", k)
+			}
+			return true
+		})
+		time.Sleep(60 * time.Second)
+	}
+}

udprepairdata/biddingindex.go

@@ -0,0 +1,402 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	mu "mfw/util"
+	qutil "qfw/util"
+	elastic "qfw/util/elastic"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"qfw/util/redis"
+
+	"gopkg.in/mgo.v2/bson"
+	es "gopkg.in/olivere/elastic.v1"
+)
+
+var BulkSize = 100
+var ESLEN = 32766
+
+//对字段处理 bidamount  budget
+//招标数据表和抽取表一一对应开始更新
+var esMap = map[string]bool{}
+var esLock = sync.Mutex{}
+
+func delEsTask() {
+	time.AfterFunc(30*time.Second, delEsTask)
+	esLock.Lock()
+	delEsAndRedis()
+	esLock.Unlock()
+}
+
+func delEsAndRedis() {
+	if len(esMap) > 0 {
+		client := elastic.GetEsConn()
+		defer elastic.DestoryEsConn(client)
+		if client != nil {
+			defer qutil.Catch()
+			req := client.Bulk()
+			var keys = []interface{}{}
+			str := ""
+			for k, _ := range esMap {
+				str += " " + k
+				keys = append(keys, fmt.Sprintf("jypcdetail__rec%s", k))
+				req = req.Add(es.NewBulkDeleteRequest().Index(index).Type(itype).Id(k))
+			}
+			res, err := req.Do()
+			if err != nil {
+				log.Println("批量删除es出错", err.Error())
+			}
+			log.Println("del es", len(res.Succeeded()))
+			log.Println(str)
+			log.Println("del redis", redis.Del("other", keys...))
+		} else {
+			log.Println("es client get failed")
+		}
+		esMap = map[string]bool{}
+	}
+}
+
+func biddingTask(data []byte, mapInfo map[string]interface{}) {
+	defer qutil.Catch()
+	q, _ := mapInfo["query"].(map[string]interface{})
+	bkey, _ := mapInfo["bkey"].(string)
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": bson.M{
+				"$gt":  qutil.StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": qutil.StringTOBsonId(mapInfo["lteid"].(string)),
+			},
+		}
+	}
+	//连接信息
+	//线程池
+	UpdatesLock := sync.Mutex{}
+
+	//bidding库
+	session := mgo.GetMgoConn(86400)
+	defer mgo.DestoryMongoConn(session)
+	count, _ := session.DB(db).C(c).Find(&q).Count()
+	log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
+	//查询招标数据
+	query := session.DB(db).C(c).Find(q).Select(bson.M{
+		"projectinfo.attachment": 0,
+		"contenthtml":            0,
+	}).Sort("_id").Iter()
+	//查询抽取结果
+	log.Println("查询抽取结果..")
+	//extract库
+	n := 0
+
+	//更新数组
+	arr := [][]map[string]interface{}{}
+	arrEs := []map[string]interface{}{}
+	//对比两张表数据，减少查询次数
+	var compare map[string]interface{}
+	log.Println("开始迭代..")
+	con := 0
+	for tmp := make(map[string]interface{}); query.Next(tmp); n++ {
+		if qutil.IntAll(tmp["extracttype"]) == -1 || ((tmp["buyerclass"] != nil || tmp["projectname"] != nil || tmp["keywords"] != nil) && force == 0) {
+			tmp = make(map[string]interface{})
+			con++
+			if con%500 == 0 {
+				log.Println("跳过:", con)
+			}
+			continue
+		}
+		//删除索引和缓存
+		tid := qutil.BsonIdToSId(tmp["_id"])
+		if tid != "" {
+			esLock.Lock()
+			esMap[tid] = true
+			if len(esMap) > 100 {
+				delEsAndRedis()
+			}
+			esLock.Unlock()
+			//log.Println("del es ", tid, elastic.DelById(index, itype, tid))
+			//删除缓存
+			//redis.DelByCodePattern("other", fmt.Sprintf("*%s*", tid))
+		}
+		update := map[string]interface{}{}
+		//对比方法----------------
+		compare = nil
+		obj, bres := mgo.FindById(extractc, tid, nil)
+		if bres && obj != nil {
+			compare = *obj
+		}
+		//extractsession.DB(extractdb).C(extractc).FindId(tmp["_id"]).One(&compare)
+		//下面可以多线程跑的--->
+		//处理分类
+		if compare != nil {
+			for _, k := range fields {
+				v1 := compare[k]
+				v2 := tmp[k]
+				if v2 == nil && v1 != nil {
+					update[k] = v1
+				} else if v2 != nil && v1 != nil {
+					update[k] = v1
+				}
+			}
+			subscopeclass, _ := compare["subscopeclass"].([]interface{})
+			if subscopeclass != nil {
+				//str := ","
+				m1 := map[string]bool{}
+				newclass := []string{}
+				for _, sc := range subscopeclass {
+					sclass, _ := sc.(string)
+					if !m1[sclass] {
+						m1[sclass] = true
+						//str += sclass + ","
+						newclass = append(newclass, sclass)
+					}
+				}
+				update["s_subscopeclass"] = strings.Join(newclass, ",")
+				update["subscopeclass"] = newclass
+			}
+			//处理中标企业
+			winner, _ := compare["winner"].(string)
+			m1 := map[string]bool{}
+			if winner != "" {
+				m1[winner] = true
+			}
+			package1 := compare["package"]
+			if package1 != nil {
+				packageM, _ := package1.(map[string]interface{})
+				for _, p := range packageM {
+					pm, _ := p.(map[string]interface{})
+					pw, _ := pm["winner"].(string)
+					if pw != "" {
+						m1[pw] = true
+					}
+				}
+			}
+			compare = nil
+			if len(m1) > 0 {
+				//str := ","
+				winnerarr := []string{}
+				for k, _ := range m1 {
+					//str += k + ","
+					winnerarr = append(winnerarr, k)
+				}
+				update["s_winner"] = strings.Join(winnerarr, ",")
+			}
+		}
+		//------------------对比结束
+
+		//处理key descript
+		if bkey == "" {
+			DealInfo(&tmp, &update)
+		}
+		//同时保存到elastic
+		for tk, tv := range update {
+			tmp[tk] = tv
+		}
+		//对projectscope字段的索引处理
+		ps, _ := tmp["projectscope"].(string)
+		if ps == "" {
+			tmp["projectscope"] = "" //= tmp["detail"]
+		}
+		if len(ps) > ESLEN {
+			tmp["projectscope"] = string(([]rune(ps))[:4000])
+		}
+		if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
+			tmp["budget"] = nil
+		} else if sbd, ok := tmp["budget"].(string); ok {
+			tmp["budget"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+		}
+		if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
+			tmp["bidamount"] = nil
+		} else if sbd, ok := tmp["bidamount"].(string); ok {
+			tmp["bidamount"] = ObjToMoney([]interface{}{sbd, sbd})[0]
+		}
+		UpdatesLock.Lock()
+		//		for k1, _ := range tmp {
+		//			if strings.HasSuffix(k1, "_b") || k1 == "contenthtml" {
+		//				delete(tmp, k1)
+		//			}
+		//		}
+		newTmp := map[string]interface{}{}
+		for _, v := range biddingIndexFields {
+			//			if tmp[v] != nil {
+			//				newTmp[v] = tmp[v]
+			//			}
+			if tmp[v] != nil {
+				if "projectinfo" == v {
+					mp, _ := tmp[v].(map[string]interface{})
+					if mp != nil {
+						newmap := map[string]interface{}{}
+						for _, v1 := range projectinfoFields {
+							if mp[v1] != nil {
+								newmap[v1] = mp[v1]
+							}
+						}
+						newTmp[v] = newmap
+					}
+				} else {
+					if v == "detail" {
+						detail, _ := tmp[v].(string)
+						newTmp[v] = FilterDetail(detail)
+					} else {
+						newTmp[v] = tmp[v]
+					}
+				}
+			}
+		}
+		arrEs = append(arrEs, newTmp)
+		if len(update) > 0 {
+			arr = append(arr, []map[string]interface{}{
+				map[string]interface{}{
+					"_id": tmp["_id"],
+				},
+				map[string]interface{}{
+					"$set": update,
+				},
+			})
+		}
+		if len(arr) >= BulkSize-1 {
+			mgo.UpdateBulkAll(db, c, arr...)
+			arr = [][]map[string]interface{}{}
+		}
+		if len(arrEs) >= BulkSize-1 {
+			tmps := arrEs
+			elastic.BulkSave(index, itype, &tmps, true)
+			arrEs = []map[string]interface{}{}
+		}
+		UpdatesLock.Unlock()
+		if n%100 == 0 {
+			log.Println("current:", n)
+		}
+		tmp = make(map[string]interface{})
+	}
+	UpdatesLock.Lock()
+	if len(arr) > 0 {
+		mgo.UpdateBulkAll(db, c, arr...)
+	}
+	if len(arrEs) > 0 {
+		tmps := arrEs
+		elastic.BulkSave(index, itype, &tmps, true)
+	}
+	UpdatesLock.Unlock()
+	log.Println(mapInfo, "create bidding index...over", n, con)
+}
+
+var client *mu.Client
+var reg = regexp.MustCompile("^[0-9a-zA-Z-.]+$")
+var reg_space = regexp.MustCompile("(?ism)(<style.*?>.*?</style>)|([.#]?\\w{1,20}\\{.*?\\})|(<.*?>)|(\\\\t)+|\\t|( +)|(　+)|(" + string(rune(160)) + "+)")
+var reg_row = regexp.MustCompile("(?i)<(tr|div|p)[^>]*?>|(\\n)+")
+var reg_dh = regexp.MustCompile("[,]+")
+var reg_newdb = regexp.MustCompile("([:,、：，。.;])[,]")
+var reg_no = regexp.MustCompile("^[0-9]*$")
+var MSG_SERVER = "123.56.236.148:7070"
+var DesLen = 120
+
+func inits() {
+	ser := qutil.ObjToString(Sysconfig["msg_server"])
+	if ser != "" {
+		MSG_SERVER = ser
+	}
+	cf := &mu.ClientConfig{
+		ClientName:      "剑鱼抽关键词",
+		EventHandler:    func(p *mu.Packet) {},
+		MsgServerAddr:   MSG_SERVER,
+		CanHandleEvents: []int{},
+		OnConnectSuccess: func() {
+			log.Println("c.")
+		},
+		ReadBufferSize:  10,
+		WriteBufferSize: 10,
+	}
+	client, _ = mu.NewClient(cf)
+
+}
+
+var keypool = make(chan bool, 1)
+
+func DealInfo(obj, update *map[string]interface{}) {
+	defer qutil.Catch()
+	if (*obj)["keywords"] != nil && (*obj)["description"] != nil {
+		return
+	} else {
+		(*update)["keywords"] = ""
+		(*update)["description"] = ""
+	}
+	title := qutil.ObjToString((*obj)["title"])
+	var m [][]string
+	select {
+	case <-func() <-chan bool {
+		select {
+		case keypool <- true:
+			defer func() {
+				<-keypool
+			}()
+			ret, _ := client.Call("", mu.UUID(8), 4010, mu.SENDTO_TYPE_RAND_RECIVER, title, 1)
+			json.Unmarshal(ret, &m)
+		case <-time.After(10 * time.Millisecond):
+		}
+		ch := make(chan bool, 1)
+		ch <- true
+		return ch
+	}():
+	case <-time.After(50 * time.Millisecond):
+	}
+	arr := []string{}
+	keyword := []string{}
+	keywordnew := []string{}
+	for _, tmp := range m {
+		if reg.MatchString(tmp[0]) {
+			arr = append(arr, tmp[0])
+		} else {
+			if len(arr) > 0 {
+				str := strings.Join(arr, "")
+				keyword = append(keyword, str)
+				arr = []string{}
+			}
+			if len(tmp[0]) > 3 && (strings.HasPrefix(tmp[1], "n") || tmp[1] == "v" || tmp[1] == "vn" || strings.HasPrefix(tmp[1], "g")) {
+				keyword = append(keyword, tmp[0])
+			}
+		}
+	}
+	for _, v := range keyword {
+		v = reg_no.ReplaceAllString(v, "")
+		if len(v) > 0 {
+			keywordnew = append(keywordnew, v)
+		}
+	}
+	keywords := strings.Join(keywordnew, ",")
+	(*update)["keywords"] = keywords
+	content := ""
+	if (*obj)["detail_bak"] != nil {
+		content = qutil.ObjToString((*obj)["detail_bak"])
+	} else {
+		content = qutil.ObjToString((*obj)["detail"])
+	}
+	//内容替换
+	content = strings.Replace(content, " ", "", -1)
+	content = reg_space.ReplaceAllString(content, "")
+	content = reg_row.ReplaceAllString(content, ",")
+	content = reg_dh.ReplaceAllString(content, ",")
+	content = reg_newdb.ReplaceAllString(content, "$1")
+	if strings.HasPrefix(content, ",") {
+		content = content[1:]
+	}
+	//log.Println(content)
+	tc := []rune(content)
+	ltc := len(tc)
+	description := content
+	if ltc > DesLen {
+		description = string(tc[:DesLen])
+	}
+	(*update)["description"] = description
+	//保存到数据库
+	return
+}
+
+var filterReg = regexp.MustCompile("<[^>]+>")
+
+func FilterDetail(text string) string {
+	return filterReg.ReplaceAllString(text, "")
+}

udprepairdata/config.json

@@ -0,0 +1,65 @@
+{
+    "udpport": ":1483",
+    "msg_server": "123.56.236.148:7070",
+	"redisaddr":"other=10.172.228.50:1713",
+    "biddingback": {
+        "db": "qfw",
+        "collect": "bidding",
+        "index": "bidding",
+        "type": "bidding"
+    },
+    "bidding": {
+        "db": "qfw",
+        "collect": "bidding",
+        "index": "bidding",
+        "type": "bidding",
+        "extractdb": "qfw",
+        "extractcollect": "extract_v2.1",
+        "indexfields": [
+            "_id",
+            "s_winner",
+            "winner",
+            "buyerclass",
+            "title",
+            "detail",
+            "area",
+            "site",
+            "bidopendate",
+            "bidopentime",
+            "buyer",
+            "city",
+            "comeintime",
+            "href",
+            "infoformat",
+            "projectcode",
+            "projectname",
+            "publishtime",
+            "s_sha",
+            "spidercode",
+            "subtype",
+            "toptype",
+            "agency",
+            "budget",
+            "bidamount",
+            "s_subscopeclass",
+            "projectscope",
+            "bidstatus",
+            "projectinfo",
+            "buyertel",
+            "buyerperson",
+            "projectid"
+        ],
+        "fields": "buyerclass,projectname,projectcode,bidamount,budget,agency,amount,winner,buyer,bidopendate,bidopentime,bidstatus,projectscope,buyertel,buyerperson,city,area",
+        "projectinfo": "approvecode,approvecontent,approvestatus,approvetime,industry",
+        "multiIndex": ""
+    },
+    "mongodb": {
+        "addr": "192.168.3.207:27082",
+        "pool": 10,
+        "db": "qfw"
+    },
+    "elastic": {
+        "addr": "http://192.168.3.207:9600",
+        "pool": 12
+    }
+}

udprepairdata/main.go

@@ -0,0 +1,138 @@
+package main
+
+/**
+修复没有合并的抽取数据到bidding表中，并删除缓存和生成索引
+./sendtask -ip 127.0.0.1 -p 14899 -stype bidding -gtid 5b582c12a5cb26b9b77fcce6 -lteid 5b582c12a5cb26b9b77fcd01
+./sendtask -ip 127.0.0.1 -p 14899 -stype bidding -q "{'comeintime':{'\$gte':1537246800,'\$lte':1537254000}}"
+**/
+
+import (
+	"encoding/json"
+	"flag"
+	"log"
+	mu "mfw/util"
+	"net"
+	"qfw/util"
+	elastic "qfw/util/elastic"
+	"qfw/util/mongodb"
+	"qfw/util/redis"
+	"strings"
+	"time"
+)
+
+var (
+	Sysconfig                                    map[string]interface{} //配置文件
+	mgo                                          *mongodb.MongodbSim    //mongodb操作对象
+	udpclient                                    mu.UdpClient           //udp对象
+	updport                                      string
+	winner, bidding, biddingback, project, buyer map[string]interface{}
+	savesizei                                    = 500
+	biddingIndexFields                           = []string{"_id", "buyerclass", "s_winner", "title", "detail", "detail_bak", "area", "areaval", "site", "type", "amount", "bidopendate", "bidopentime", "buyer", "channel", "city", "comeintime", "contenthtml", "descript", "description", "extracttype", "href", "infoformat", "keywords", "projectcode", "projectname", "publishtime", "s_sha", "spidercode", "subtype", "summary", "toptype", "urltop", "winner", "agency", "budget", "bidamount", "s_subscopeclass", "projectscope", "bidstatus"}
+	projectinfoFields                            []string
+	force                                        = 0
+)
+
+func init() {
+	util.ReadConfig(&Sysconfig)
+	inits()
+	updport, _ = Sysconfig["updport"].(string)
+	bidding, _ = Sysconfig["bidding"].(map[string]interface{})
+	index, _ = bidding["index"].(string)
+	itype, _ = bidding["type"].(string)
+	c, _ = bidding["collect"].(string)
+	extractc, _ = bidding["extractcollect"].(string)
+	db, _ = bidding["db"].(string)
+	//extractdb, _ := bidding["extractdb"].(string)
+	fields = strings.Split(bidding["fields"].(string), ",")
+	biddingback, _ = Sysconfig["biddingback"].(map[string]interface{})
+	mconf, _ := Sysconfig["mongodb"].(map[string]interface{})
+	mgo = &mongodb.MongodbSim{
+		MongodbAddr: mconf["addr"].(string),
+		Size:        util.IntAllDef(mconf["pool"], 5),
+		DbName:      mconf["db"].(string),
+	}
+	mgo.InitPool()
+	econf := Sysconfig["elastic"].(map[string]interface{})
+	elastic.InitElasticSize(econf["addr"].(string), util.IntAllDef(econf["pool"], 5))
+	redisaddr, _ := Sysconfig["redisaddr"].(string)
+	redis.InitRedisBySize(redisaddr, 10, 5, 240)
+	if bidding["indexfields"] != nil {
+		biddingIndexFields = util.ObjArrToStringArr(bidding["indexfields"].([]interface{}))
+	}
+	if bidding["projectinfo"] != nil {
+		pf := util.ObjToString(bidding["projectinfo"])
+		if pf != "" {
+			projectinfoFields = strings.Split(pf, ",")
+		}
+	}
+	log.Println(projectinfoFields)
+}
+
+var lastId = ""
+
+//func CheckMgoTask() {
+//	if lastId != "" {
+//		res, bres := mgo.Find(extractc, `{}`, `{"_id":-1}`, `{"_id":1}`, true, -1, -1)
+//		if bres && res != nil && len(*res) == 1 {
+//			id := util.BsonIdToSId(((*res)[0])["_id"])
+//			if id > lastId {
+//				mapInfo := map[string]interface{}{
+//					"gtid":  lastId,
+//					"lteid": id,
+//				}
+//				biddingTask(nil, mapInfo)
+//				log.Println("task over!", lastId, id)
+//				lastId = id
+//			}
+//		}
+//	}
+//	time.AfterFunc(5*time.Minute, CheckMgoTask)
+//}
+
+var (
+	index, itype, c, extractc, db string
+	fields                        []string
+)
+
+func main() {
+	flag.IntVar(&force, "f", 0, "是否强制执行")
+	flag.StringVar(&lastId, "id", "", "上次执行id")
+	flag.Parse()
+	updport := Sysconfig["udpport"].(string)
+	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Println("Udp服务监听", updport)
+	go delEsTask()
+	time.Sleep(99999 * time.Hour)
+}
+
+var pool = make(chan bool, 20)
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA: //上个节点的数据
+		//从表中开始处理生成企业数据
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		log.Println("err:", err, "mapInfo:", mapInfo, string(data))
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			udpclient.WriteUdp([]byte("ok,run"), mu.OP_NOOP, ra)
+			tasktype, _ := mapInfo["stype"].(string)
+			log.Println("tasktype:", tasktype)
+			switch tasktype {
+			case "bidding":
+				pool <- true
+				go func() {
+					defer func() {
+						<-pool
+					}()
+					biddingTask(data, mapInfo)
+				}()
+			}
+		}
+	case mu.OP_NOOP: //下个节点回应
+		log.Println("发送成功", string(data))
+	}
+}

udprepairdata/tonumber.go

@@ -0,0 +1,280 @@
+// tonumber
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var contentUnit *regexp.Regexp          //全文检索单位：万元
+var regOperator *regexp.Regexp          //运算符号
+var regNumFloat *regexp.Regexp          //提取整数或浮点数
+var moneyRegChar *regexp.Regexp         //提取中文数字
+var regStrUnit *regexp.Regexp           //提取单位
+var moneyChar = map[string]interface{}{ //"〇": "0", "零": "0",
+	"一": float64(1), "壹": float64(1), "二": float64(2), "贰": float64(2), "三": float64(3), "叁": float64(3), "四": float64(4), "肆": float64(4), "五": float64(5), "伍": float64(5),
+	"六": float64(6), "陆": float64(6), "七": float64(7), "柒": float64(7), "八": float64(8), "捌": float64(8), "九": float64(9), "玖": float64(9), "十": float64(10), "拾": float64(10),
+	"百": float64(100), "佰": float64(100), "千": float64(1000), "仟": float64(1000), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000),
+	"零": float64(0), "点": ".", "角": float64(0.1), "分": float64(0.01),
+}
+var moneyUnit = map[string]float64{
+	"元": float64(1), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000), //单位
+}
+var spaces = []string{"\u3000", "\u2003", "\u00a0"}
+var cutAllSpace = regexp.MustCompile(`\s*`)
+
+func init() {
+	regOperator, _ = regexp.Compile(`[*|+|)*)]`)
+	regNumFloat, _ = regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
+	regStrUnit, _ = regexp.Compile(`[元|万|亿]`)
+
+	regStrChar := `[〇|零|点|.|．|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
+	moneyRegChar, _ = regexp.Compile(regStrChar)
+
+	contentUnit, _ = regexp.Compile(`(万元|单位/万)`)
+}
+
+//转int
+func ObjToInt(data []interface{}) []interface{} {
+	tmp, err := strconv.Atoi(fmt.Sprint(data[0]))
+	if err != nil {
+		data[0] = 0
+		return data
+	} else {
+		data[0] = tmp
+		return data
+	}
+}
+
+//转float,精度小数点4位
+func ObjToFloat(data []interface{}) []interface{} {
+	tmp, err := strconv.ParseFloat(fmt.Sprint(data[0]), 64)
+	if err != nil {
+		return []interface{}{float64(0), data[1]}
+	} else {
+		tmp, err = strconv.ParseFloat(strconv.FormatFloat(tmp, 'f', 4, 64), 64)
+		if err != nil {
+			return []interface{}{float64(0), data[1]}
+		} else {
+			return []interface{}{tmp, data[1]}
+		}
+	}
+}
+
+//金额转换
+func ObjToMoney(data []interface{}) []interface{} {
+	isfindUnit := true
+	ret := capitalMoney(data)[0]
+	if ret.(float64) < float64(10000) || ret.(float64) > float64(50000000000) {
+		ret2, b := numMoney(data)
+		isfindUnit = b
+		if ret2[0].(float64) > ret.(float64) {
+			ret = ret2[0]
+		}
+	}
+	f, _ := strconv.ParseFloat(strconv.FormatFloat(ret.(float64), 'f', 4, 64), 64)
+	if f < 1 {
+		f = 0
+	}
+	//若果金额小于50，全文检索单位：万
+	if f < 50 && f > 0 && isfindUnit {
+		rep := contentUnit.FindAllStringIndex(fmt.Sprint(data[1]), -1)
+		if len(rep) > 0 {
+			f = f * 10000
+		}
+	}
+	data[0] = f
+	return data
+}
+
+//清理所有空白符
+func CutAllSpace(data []interface{}) []interface{} {
+	tmp := cutAllSpace.ReplaceAllString(fmt.Sprint(data[0]), "")
+	tmp = replaceSymbol(tmp, spaces)
+	data[0] = tmp
+	return data
+}
+
+//数字金额转换
+func numMoney(data []interface{}) ([]interface{}, bool) {
+	tmp := fmt.Sprint(data[0])
+	tmp = replaceSymbol(tmp, []string{",", "，", "（", "）", "(", ")", "：", "\n"})
+	tmp = replaceString(tmp, []string{"万元", "亿元", "．"}, []string{"万", "亿", "."})
+	tmp = fmt.Sprint(CutAllSpace([]interface{}{tmp, data[1]})[0])
+	rets := regNumFloat.FindAllString(tmp, -1)
+	fnums := []float64{}
+	unitstrs := []string{}
+	if len(rets) > 0 {
+		pindex := 0 //单位前置
+		for k, v := range rets {
+			f, err := strconv.ParseFloat(v, 64)
+			if err == nil {
+				fnums = append(fnums, f)
+				index := strings.Index(tmp, v)
+				//单位后置
+				start := index + len(v)
+				end := start + 3
+				//log.Println("vvv", tmp, v, pindex, index, start)
+				if k > 0 {
+					if start >= pindex+3 {
+						pstart := pindex + 3
+						if pstart >= index {
+							pstart = index
+						}
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:])
+						}
+					} else {
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				} else {
+					if len(tmp) > end {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:end])
+						}
+					} else {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				}
+				pindex = start
+			}
+		}
+	}
+	//log.Println("unitstrs", fnums, unitstrs)
+	unit := float64(0)
+	fnum := float64(0)
+	for k, v := range fnums {
+		fnum = v
+		units := regStrUnit.FindAllString(unitstrs[k], -1)
+		for _, v := range units {
+			if moneyUnit[v] != 0 {
+				unit = moneyUnit[v]
+				break
+			}
+		}
+		if unit != float64(0) { //取第一个
+			break
+		}
+	}
+	if unit == float64(0) {
+		data[0] = fnum
+	} else {
+		data[0] = fnum * unit
+	}
+	if unit == 10000 {
+		return data, false
+	} else {
+		return data, true
+	}
+}
+
+//大写数子金额转换
+func capitalMoney(data []interface{}) []interface{} {
+	nodes := []float64{}
+	node := float64(0)
+	tmp := float64(0)
+	decimals := 0.0
+	ishaspoint := false //是否含小数点
+	fnum := float64(0)
+	moneyRegChar.ReplaceAllStringFunc(fmt.Sprint(data[0]), func(key string) string {
+		if key == "元" || key == "圆" || key == "点" {
+			ishaspoint = true
+		}
+		if v, ok := moneyChar[key].(float64); ok {
+			if (ishaspoint && v > 10) || key == "整" || key == "正" { //排除后面有其他的单位
+				return ""
+			}
+			//log.Println(key, v, fnum)
+			if v < 10 && v >= 0 {
+				if ishaspoint { //小数部分
+					if v >= 1 {
+						fnum = v
+					} else if v < 1 && v > 0 {
+						decimals += fnum * v
+					}
+				} else {
+					if tmp != float64(0) {
+						node += tmp
+					}
+					tmp = float64(v)
+				}
+			} else if v == 10000 || v == 100000000 { //单位万、亿
+				if tmp != float64(0) {
+					node += tmp
+					tmp = float64(0)
+				}
+				nodes = append(nodes, node*float64(v))
+				node = float64(0)
+			} else {
+				if v == 10 && tmp == 0 {
+					tmp = 1
+				}
+				tmp = tmp * float64(v)
+				node += tmp
+				tmp = float64(0)
+			}
+		}
+		return ""
+	})
+	nodes = append(nodes, node, tmp)
+	ret := float64(0)
+	for _, v := range nodes {
+		ret = ret + v
+	}
+	return []interface{}{ret + decimals, data[1]}
+}
+
+//过滤符号
+func replaceSymbol(con string, rep []string) string {
+	for _, v := range rep {
+		con = strings.Replace(con, v, "", -1)
+	}
+	return con
+}
+
+//符号替换
+func replaceString(con string, ret, rep []string) string {
+	for k, v := range ret {
+		if len(rep) > k {
+			con = strings.Replace(con, v, rep[k], -1)
+		}
+	}
+	return con
+}
+
+//费率转小数
+func RateToFloat(con []interface{}) []interface{} {
+	tmp := fmt.Sprint(CutAllSpace(con)[0])
+	if strings.Contains(tmp, "%") || strings.Contains(tmp, "％") {
+		tmp = strings.Replace(tmp, "%", "", -1)
+		tmp = strings.Replace(tmp, "％", "", -1)
+		rep := ObjToFloat([]interface{}{tmp, con[1]})[0]
+		con[0] = rep.(float64) / 100
+		return con
+	} else {
+		return ObjToFloat([]interface{}{tmp, con[1]})
+	}
+}
+
+//大于一万亿的过滤掉
+func ClearMaxAmount(data []interface{}) []interface{} {
+	value, _ := data[0].(float64)
+	if value >= 1000000000000 {
+		data[0] = float64(0)
+	}
+	return data
+}