3

src/jy/extract/extractInit.go

@@ -89,10 +89,10 @@ type ExtractTask struct {
 	ResultChanel chan bool //抽取结果详情
 	sync.RWMutex
 	ResultArr [][]map[string]interface {
-	} //抽取结果详情
+	}                   //抽取结果详情
 	BidChanel chan bool //抽取结果
 	BidArr    [][]map[string]interface {
-	} //抽取结果
+	}            //抽取结果
 	BidTotal int //结果数量
 
 	RecogFieldMap map[string]map[string]interface {
@@ -100,7 +100,7 @@ type ExtractTask struct {
 	FidClassMap map[string][]map[string]interface {
 	} //分类
 	CidRuleMap map[string][]map[string]interface {
-	} //规则
+	}                    //规则
 	AuditFields []string //需要审核的字段名称
 
 	SiteCityMap          map[string]*SiteCity //站点对应的省市区
@@ -1000,30 +1000,14 @@ func InitProvince(version string) map[string]interface{} {
 	return fn
 }
 
-//加载城市简称
-func InitCitySim(version string) map[string]map[string]interface{} {
+//加载所有
+func InitProvincesx() []map[string]interface{} {
 	defer qu.Catch()
-	list, _ := db.Mgo.Find("tagdetailinfo", `{"s_type":"citysim","s_version":"`+version+`","delete":false}`, nil, nil, false, -1, -1)
-	fn := map[string]map[string]interface{}{}
-	for _, v := range *list {
-		name := qu.ObjToString(v["s_name"])
-		tmp := v["content"].(map[string]interface{})
-		fn[name] = tmp
-	}
-	return fn
-}
-
-//加载城市全称
-func InitCityAll(version string) map[string]map[string]interface{} {
-	defer qu.Catch()
-	list, _ := db.Mgo.Find("tagdetailinfo", `{"s_type":"cityall","s_version":"`+version+`","delete":false}`, nil, nil, false, -1, -1)
-	fn := map[string]map[string]interface{}{}
-	for _, v := range *list {
-		name := qu.ObjToString(v["s_name"])
-		tmp := v["content"].(map[string]interface{})
-		fn[name] = tmp
-	}
-	return fn
+	provinces := make([]map[string]interface{}, 0)
+	ju.AddrsSess.Find(map[string]interface{}{
+		"Remarks": nil,
+	}).All(&provinces)
+	return provinces
 }
 
 //加载站点库site城市信息
@@ -1065,106 +1049,183 @@ func (e *ExtractTask) InitCityInfo() {
 			e.ProvinceMap[p1] = k             //华中科技大学:湖北
 		}
 	}
+	alldata := InitProvincesx()
+
+	fnx := make([]map[string]interface{}, 0)
+	citys_maps := make(map[string][]map[string]interface{}, 0)
+	districts_maps := make(map[string]map[string][]map[string]interface{}, 0)
+	towns_maps := make(map[string]map[string]map[string][]map[string]interface{}, 0)
+	jwhs_maps := make(map[string]map[string]map[string]map[string][]map[string]interface{}, 0)
+	for _, v := range alldata {
+		codenum := len(v["code"].(string))
+		province := qu.ObjToString(v["province"])
+		city := qu.ObjToString(v["city"])
+		district := qu.ObjToString(v["district"])
+		town := qu.ObjToString(v["town"])
+		if codenum == 2 {
+			fnx = append(fnx, v)
+		} else if codenum == 4 {
+			citys_maps[province] = append(citys_maps[province], v)
+		} else if codenum == 6 {
+			if districts_maps[province] == nil {
+				districts_maps[province] = make(map[string][]map[string]interface{}, 0)
+			}
+			districts_maps[province][city] = append(districts_maps[province][city], v)
+		} else if codenum == 9 {
+			if towns_maps[province] == nil {
+				towns_maps[province] = make(map[string]map[string][]map[string]interface{}, 0)
+			}
+			if towns_maps[province][city] == nil {
+				towns_maps[province][city] = make(map[string][]map[string]interface{}, 0)
+			}
+			towns_maps[province][city][district] = append(towns_maps[province][city][district], v)
+		} else if codenum == 12 {
+			if jwhs_maps[province] == nil {
+				jwhs_maps[province] = make(map[string]map[string]map[string][]map[string]interface{}, 0)
+			}
+			if jwhs_maps[province][city] == nil {
+				jwhs_maps[province][city] = make(map[string]map[string][]map[string]interface{}, 0)
+			}
+			if jwhs_maps[province][city][district] == nil {
+				jwhs_maps[province][city][district] = make(map[string][]map[string]interface{}, 0)
+			}
+			jwhs_maps[province][city][district][town] = append(jwhs_maps[province][city][district][town], v)
+		}
+	}
+
 	//初始化城市全称
-	fn2 := InitCityAll(e.TaskInfo.Version)
-	for k, v := range fn2 {
+	for _, provinces := range fnx {
+		all_province := qu.ObjToString(provinces["all_province"]) //省全称
+		jc_province := qu.ObjToString(provinces["province"])      //省简称
 		//加载省信息
-		e.Trie_Full_Province.AddWords(k) //加入省全称Trie(k:浙江省)
+		e.Trie_Full_Province.AddWords(all_province) //加入省全称Trie(k:浙江省)
 		p := &Province{}
-		p.Name = k                            //省全称:浙江省
-		p.Brief = v["brief"].(string)         //省简称:浙江
-		e.Trie_Sim_Province.AddWords(p.Brief) //加入省简称Trie(k:浙江)
-		e.ProvinceMap[k] = p.Brief            //浙江省:浙江
-		e.ProvinceBriefMap[p.Brief] = p       //浙江:省信息{}
-		p.Cap = v["captial"].(string)         //省会(杭州)
+		p.Name = all_province                     //省全称:浙江省
+		p.Brief = jc_province                     //省简称:浙江
+		e.Trie_Sim_Province.AddWords(jc_province) //加入省简称Trie(k:浙江)
+		e.ProvinceMap[all_province] = jc_province //浙江省:浙江
+		e.ProvinceBriefMap[jc_province] = p       //浙江:省信息{}
+		if province_alias, ok := provinces["province_alias"].([]interface{}); ok {
+			for _, vprovince_alias := range province_alias {
+				e.ProvinceBriefMap[qu.ObjToString(vprovince_alias)] = p
+			}
+		}
 		//加载市信息
-		city, _ := v["city"].(map[string]interface{})
-		for k1, v1 := range city {
-			e.Trie_Full_City.AddWords(k1) //加入市全称Trie(k:杭州市)
-			v1m, _ := v1.(map[string]interface{})
+		citys := citys_maps[jc_province]
+		isok := make(map[string]bool)
+		for _, vcity := range citys {
+			qc_city := qu.ObjToString(vcity["city"])
+			jc_city := qu.ObjToString(vcity["brief_city"])
+			e.Trie_Full_City.AddWords(qc_city) //加入市全称Trie(k:杭州市)
 			c := &City{}
-			c.Name = k1                       //市全称:杭州市
-			c.Brief = v1m["brief"].(string)   //市简称:杭州
-			e.Trie_Sim_City.AddWords(c.Brief) //加入市简称Trie(k:杭州)
-			e.CityMap[k1] = c.Brief           //杭州市:杭州
-			e.CityBriefMap[c.Brief] = c       //杭州:市信息{}
-			e.CityFullMap[k1] = c             //杭州市:市信息{}
+			c.Name = qc_city //市全称:杭州市
+			if jc_city != "" {
+				c.Brief = jc_city                 //市简称:杭州
+				e.Trie_Sim_City.AddWords(c.Brief) //加入市简称Trie(k:杭州)
+				e.CityMap[qc_city] = c.Brief      //杭州市:杭州
+				e.CityBriefMap[c.Brief] = c       //杭州:市信息{}
+				e.CityFullMap[qc_city] = c        //杭州市:市信息{}
+			}
 			c.P = p
-			if c.Name == p.Cap {
-				p.Captial = c //加载province中的省会市信息{}
+			if city_alias, ok := vcity["city_alias"].([]interface{}); ok {
+				for _, vcity_alias := range city_alias {
+					strvcity_alias := qu.ObjToString(vcity_alias)
+					if isok[jc_province+"_"+strvcity_alias] {
+						continue
+					}
+					e.CityBriefMap[strvcity_alias] = c
+					e.initDistricts(jc_province, strvcity_alias, c, jc_city, districts_maps, towns_maps, jwhs_maps)
+					isok[jc_province+"_"+strvcity_alias] = true
+				}
 			}
-			//区县
-			districtmap, _ := v1m["area"].(map[string]interface{}) //区或县
-			for district, streets := range districtmap {
-				d := &District{}
-				d.Name = district
-				d.C = c
-				//省直辖市，河南济源市没有区一级，目前区一级写的还是济源市
-				//匹配时，如果匹配到区，拿区和市比对，相同则代表是省直辖市，不要区一级
-				e.Trie_Full_District.AddWords(district) //加入区或县全称Trie
-				ctmp := e.NewDistrictCityMap[district]
+			if isok[jc_province+"_"+qc_city] {
+				continue
+			}
+			e.initDistricts(jc_province, qc_city, c, jc_city, districts_maps, towns_maps, jwhs_maps)
+		}
+	}
+
+	e.Trie_Fulls = []*ju.Trie{e.Trie_Full_Province, e.Trie_Full_City, e.Trie_Full_District, e.Trie_Full_Street, e.Trie_Full_Community}
+	e.Trie_Sims = []*ju.Trie{e.Trie_Sim_Province, e.Trie_Sim_City, e.Trie_Sim_District}
+}
+
+//加载区县
+func (e *ExtractTask) initDistricts(jc_province string, qc_city string, c *City,
+	jc_city string, districts_maps map[string]map[string][]map[string]interface{},
+	towns_maps map[string]map[string]map[string][]map[string]interface{},
+	jwhs_maps map[string]map[string]map[string]map[string][]map[string]interface{}) {
+	districts := districts_maps[jc_province][qc_city]
+	for _, vdistricts := range districts {
+		qc_district := qu.ObjToString(vdistricts["district"])
+		jc_district := qu.ObjToString(vdistricts["brief_district"])
+		d := &District{}
+		d.Name = qc_district
+		d.C = c
+		e.Trie_Full_District.AddWords(qc_district) //加入区或县全称Trie
+		if jc_district != "" {
+			e.Trie_Sim_District.AddWords(jc_district) //加入区或县简称Trie
+			//初始化城市简称
+			c := e.CityBriefMap[jc_city]
+			dfullarr := e.NewDistrictSimAndAll[jc_district]
+			dfullcity := map[string]*City{qc_district: c}
+			if len(dfullarr) == 0 {
+				tmparr := []map[string]*City{dfullcity}
+				e.NewDistrictSimAndAll[jc_district] = tmparr
+			} else {
+				e.NewDistrictSimAndAll[jc_district] = append(e.NewDistrictSimAndAll[jc_district], dfullcity)
+			}
+		}
+		ctmp := e.NewDistrictCityMap[qc_district]
+		if len(ctmp) == 0 {
+			tmpcarr := []*City{c}
+			e.NewDistrictCityMap[qc_district] = tmpcarr
+		} else {
+			e.NewDistrictCityMap[qc_district] = append(e.NewDistrictCityMap[qc_district], c)
+		}
+		if district_alias, ok := vdistricts["district_alias"].([]interface{}); ok {
+			for _, vdistrict_alias := range district_alias {
+				strvdistrict_alias := qu.ObjToString(vdistrict_alias)
+				e.Trie_Full_District.AddWords(strvdistrict_alias) //加入区或县全称Trie
+				ctmp := e.NewDistrictCityMap[strvdistrict_alias]
 				if len(ctmp) == 0 {
 					tmpcarr := []*City{c}
-					e.NewDistrictCityMap[district] = tmpcarr
+					e.NewDistrictCityMap[strvdistrict_alias] = tmpcarr
 				} else {
-					e.NewDistrictCityMap[district] = append(e.NewDistrictCityMap[district], c)
-				}
-				//街道
-				streetmap, _ := streets.(map[string]interface{})
-				for street, communitys := range streetmap {
-					s := &Street{}
-					s.Name = street
-					s.D = d
-					e.Trie_Full_Street.AddWords(street) //加入街道全称Trie
-					dtmp := e.NewStreetDistrictMap[street]
-					if len(dtmp) == 0 {
-						tmpdarr := []*District{d}
-						e.NewStreetDistrictMap[street] = tmpdarr
-					} else {
-						e.NewStreetDistrictMap[street] = append(e.NewStreetDistrictMap[street], d)
-					}
-					//村、居委会
-					for _, ct := range qu.ObjArrToStringArr(communitys.([]interface{})) {
-						e.Trie_Full_Community.AddWords(ct) //加入居委会、村全称Trie
-						cttmp := e.CommunityDistrictMap[ct]
-						if len(cttmp) == 0 {
-							tmpdarr := []*District{d}
-							e.CommunityDistrictMap[ct] = tmpdarr
-						} else {
-							e.CommunityDistrictMap[ct] = append(e.CommunityDistrictMap[ct], d)
-						}
-					}
+					e.NewDistrictCityMap[strvdistrict_alias] = append(e.NewDistrictCityMap[strvdistrict_alias], c)
 				}
 			}
 		}
-	}
-
-	//初始化城市简称
-	fn3 := InitCitySim(e.TaskInfo.Version)
-	for _, v := range fn3 {
-		city, _ := v["city"].(map[string]interface{})
-		for _, v1 := range city {
-			v1m, _ := v1.(map[string]interface{})
-			cb := v1m["brief"].(string)                 //市简称
-			arr := v1m["area"].(map[string]interface{}) //区或县简称
-			for districtsim, districtall := range arr {
-				dfullstr, _ := districtall.(string)
-				e.Trie_Sim_District.AddWords(districtsim) //加入区或县简称Trie
-				c := e.CityBriefMap[cb]
-				dfullarr := e.NewDistrictSimAndAll[districtsim]
-				dfullcity := map[string]*City{dfullstr: c}
-				if len(dfullarr) == 0 {
-					tmparr := []map[string]*City{dfullcity}
-					e.NewDistrictSimAndAll[districtsim] = tmparr
+		//街道
+		towns := towns_maps[jc_province][qc_city][qc_district]
+		for _, vtown := range towns {
+			strvtown := qu.ObjToString(vtown["town"])
+			s := &Street{}
+			s.Name = strvtown
+			s.D = d
+			e.Trie_Full_Street.AddWords(strvtown) //加入街道全称Trie
+			dtmp := e.NewStreetDistrictMap[strvtown]
+			if len(dtmp) == 0 {
+				tmpdarr := []*District{d}
+				e.NewStreetDistrictMap[strvtown] = tmpdarr
+			} else {
+				e.NewStreetDistrictMap[strvtown] = append(e.NewStreetDistrictMap[strvtown], d)
+			}
+			//村、居委会
+			jwhs := jwhs_maps[jc_province][qc_city][qc_district][strvtown]
+			for _, vjwh := range jwhs {
+				strvillage := qu.ObjToString(vjwh["village"])
+				e.Trie_Full_Community.AddWords(strvillage) //加入居委会、村全称Trie
+				cttmp := e.CommunityDistrictMap[strvillage]
+				if len(cttmp) == 0 {
+					tmpdarr := []*District{d}
+					e.CommunityDistrictMap[strvillage] = tmpdarr
 				} else {
-					e.NewDistrictSimAndAll[districtsim] = append(e.NewDistrictSimAndAll[districtsim], dfullcity)
+					e.CommunityDistrictMap[strvillage] = append(e.CommunityDistrictMap[strvillage], d)
 				}
 			}
 		}
-	}
 
-	e.Trie_Fulls = []*ju.Trie{e.Trie_Full_Province, e.Trie_Full_City, e.Trie_Full_District, e.Trie_Full_Street, e.Trie_Full_Community}
-	e.Trie_Sims = []*ju.Trie{e.Trie_Sim_Province, e.Trie_Sim_City, e.Trie_Sim_District}
+	}
 }
 
 func (e *ExtractTask) InitVar() {
@@ -1226,128 +1287,6 @@ func (e *ExtractTask) InitVar() {
 
 }
 
-//初始化城市省份敏感词
-func (e *ExtractTask) InitCityDFA() {
-	defer qu.Catch()
-	e.CityAllGet = &ju.DFA{}
-	e.CitySimGet = &ju.DFA{}
-	e.DistrictAllGet = &ju.DFA{}
-	e.DistrictSimGet = &ju.DFA{}
-	e.ProvinceAllGet = &ju.DFA{}
-	e.ProvinceSimGet = &ju.DFA{}
-	e.StreetGet = &ju.DFA{}
-	//初始化map
-	if e.ProvinceMap == nil {
-		e.ProvinceMap = make(map[string]string)
-	}
-	if e.CityMap == nil {
-		e.CityMap = make(map[string]string)
-	}
-	if e.DistrictSimAndAll == nil {
-		e.DistrictSimAndAll = make(map[string]string)
-	}
-	if e.CityBriefMap == nil {
-		e.CityBriefMap = make(map[string]*City)
-	}
-	if e.CityFullMap == nil {
-		e.CityFullMap = make(map[string]*City)
-	}
-	if e.ProvinceBriefMap == nil {
-		e.ProvinceBriefMap = make(map[string]*Province)
-	}
-	if e.DistrictCityMap == nil {
-		e.DistrictCityMap = make(map[string]*City)
-	}
-	if e.StreetDistrictMap == nil {
-		e.StreetDistrictMap = make(map[string]*District)
-	}
-	//初始化省
-	fn1 := InitProvince(e.TaskInfo.Version)
-	for k, v := range fn1 {
-		for _, p := range v.([]interface{}) {
-			p1, _ := p.(string)
-			e.ProvinceAllGet.AddWord(p1) //华中科技大学
-			e.ProvinceMap[p1] = k        //华中科技大学:湖北
-		}
-	}
-
-	//初始化城市全称
-	fn2 := InitCityAll(e.TaskInfo.Version)
-	for k, v := range fn2 {
-		//加载省信息
-		e.ProvinceAllGet.AddWord(k) //加入省全称dfa(k:浙江省)
-		p := &Province{}
-		p.Name = k                        //省全称:浙江省
-		p.Brief = v["brief"].(string)     //省简称:浙江
-		e.ProvinceSimGet.AddWord(p.Brief) //加入省简称dfa(k:浙江)
-		e.ProvinceMap[k] = p.Brief        //浙江省:浙江
-		e.ProvinceBriefMap[p.Brief] = p   //浙江:省信息{}
-		p.Cap = v["captial"].(string)     //省会(杭州)
-
-		//加载市信息
-		city, _ := v["city"].(map[string]interface{})
-		for k1, v1 := range city {
-			e.CityAllGet.AddWord(k1) //加入市全称dfa(k:杭州市)
-			v1m, _ := v1.(map[string]interface{})
-			c := &City{}
-			c.Name = k1                     //市全称:杭州市
-			c.Brief = v1m["brief"].(string) //市简称:杭州
-			e.CitySimGet.AddWord(c.Brief)   //加入市简称dfa(k:杭州)
-			e.CityMap[k1] = c.Brief         //杭州市:杭州
-			e.CityBriefMap[c.Brief] = c     //杭州:市信息{}
-			e.CityFullMap[k1] = c           //杭州市:市信息{}
-			c.P = p
-			if c.Name == p.Cap {
-				p.Captial = c //加载province中的省会市信息{}
-			}
-
-			//区县
-			districtmap := v1m["area"].(map[string]interface{}) //区或县
-			for district, streetarr := range districtmap {
-				d := &District{}
-				d.Name = district
-				d.C = c
-				//省直辖市，河南济源市没有区一级，目前区一级写的还是济源市
-				//匹配时，如果匹配到区，拿区和市比对，相同则代表是省直辖市，不要区一级？
-				e.DistrictAllGet.AddWord(district) //加入区或县全称dfa
-				ctmp := e.DistrictCityMap[district]
-				if ctmp == nil {
-					e.DistrictCityMap[district] = c
-				}
-				//街道
-				for _, s := range qu.ObjArrToStringArr(streetarr.([]interface{})) {
-					e.StreetGet.AddWord(s) //加入街道敏感词
-					dtmp := e.StreetDistrictMap[s]
-					if dtmp == nil {
-						e.StreetDistrictMap[s] = d
-					}
-				}
-			}
-		}
-	}
-	//初始化城市简称
-	fn3 := InitCitySim(e.TaskInfo.Version)
-	for _, v := range fn3 {
-		city, _ := v["city"].(map[string]interface{})
-		for _, v1 := range city {
-			v1m, _ := v1.(map[string]interface{})
-			cb := v1m["brief"].(string)                 //市简称
-			arr := v1m["area"].(map[string]interface{}) //区或县简称
-			for districtsim, districtall := range arr {
-				e.DistrictSimAndAll[districtsim] = districtall.(string)
-				d := &District{}
-				d.Name = districtsim
-				d.C = e.CityBriefMap[cb]
-				e.DistrictSimGet.AddWord(districtsim) //加入区或县简称敏感词
-				ctmp := e.DistrictCityMap[districtsim]
-				if ctmp == nil {
-					e.DistrictCityMap[districtsim] = e.CityBriefMap[cb]
-				}
-			}
-		}
-	}
-}
-
 //初始化邮编库
 func (e *ExtractTask) InitPostCode() {
 	defer qu.Catch()

src/jy/extract/extractcity2.go

@@ -1,410 +0,0 @@
-package extract
-
-import (
-	db "jy/mongodbutil"
-	ju "jy/util"
-	"log"
-	qu "qfw/util"
-	"strings"
-)
-
-var ProvinceMap2 map[string]string
-var CityBrief2 map[string]*City         //只加载一次即可
-var ProvinceBrief2 map[string]*Province //只加载一次
-var AreaToCity2 map[string][]*City      //两个文件共用
-var DistrictCityMap2 map[string]*City
-var StreetDistrictMap2 map[string]*District
-var AreaGet2 *ju.DFA         //市全称
-var AreaDistrict2 *ju.DFA    //区或县
-var AreaProvinceGet2 *ju.DFA //省
-var AreaSimGet2 *ju.DFA      //市简称
-var AreaStreet2 *ju.DFA      //街道
-
-func InitDFA2() {
-	defer qu.Catch()
-	AreaGet2 = &ju.DFA{}
-	AreaDistrict2 = &ju.DFA{}
-	AreaProvinceGet2 = &ju.DFA{}
-	AreaStreet2 = &ju.DFA{}
-	//初始化map
-	if ProvinceMap2 == nil {
-		ProvinceMap2 = make(map[string]string)
-	}
-	if CityBrief2 == nil {
-		CityBrief2 = make(map[string]*City)
-	}
-	if ProvinceBrief2 == nil {
-		ProvinceBrief2 = make(map[string]*Province)
-	}
-	if AreaToCity2 == nil {
-		AreaToCity2 = make(map[string][]*City)
-	}
-	if DistrictCityMap2 == nil {
-		DistrictCityMap2 = make(map[string]*City)
-	}
-	if StreetDistrictMap2 == nil {
-		StreetDistrictMap2 = make(map[string]*District)
-	}
-	//初始化省
-	fn1 := InitProvince("v3.0")
-	for k, v := range fn1 {
-		for _, p := range v.([]interface{}) {
-			p1, _ := p.(string)
-			AreaProvinceGet2.AddWord(p1)
-			ProvinceMap2[p1] = k
-		}
-	}
-
-	//初始化城市全称
-	fn2 := InitCityAll("v3.0")
-	for k, v := range fn2 {
-		AreaProvinceGet2.AddWord(k) //省全称
-		p := &Province{}
-		p.Name = k
-		p.Brief = v["brief"].(string)
-		ProvinceMap2[k] = p.Brief
-		//
-		ProvinceBrief2[p.Brief] = p
-		p.Cap = v["captial"].(string)
-		city, _ := v["city"].(map[string]interface{})
-		for k1, v1 := range city {
-			v1m, _ := v1.(map[string]interface{})
-			c := &City{}
-			c.Name = k1
-			//			if v1m["brief"] == nil {
-			//			}
-			c.Brief = v1m["brief"].(string)
-			//
-			CityBrief2[c.Brief] = c
-			c.P = p
-			if c.Brief == p.Cap {
-				p.Captial = c
-			}
-			//加入到城市map中
-			cs := AreaToCity2[k1]
-			AreaGet2.AddWord(k1) //市全称
-			if cs != nil {
-				cs = append(cs, c)
-			} else {
-				cs = []*City{c}
-			}
-			AreaToCity2[k1] = cs
-			//区县
-			districtmap := v1m["area"].(map[string]interface{}) //区或县
-			for district, streetarr := range districtmap {
-				d := &District{}
-				d.Name = district
-				d.C = c
-				AreaDistrict2.AddWord(district) //加入区或县敏感词
-				ctmp := DistrictCityMap2[district]
-				if ctmp == nil {
-					DistrictCityMap2[district] = c
-				}
-				//街道
-				for _, s := range qu.ObjArrToStringArr(streetarr.([]interface{})) {
-					AreaStreet2.AddWord(s) //加入街道敏感词
-					dtmp := StreetDistrictMap2[s]
-					if dtmp == nil {
-						StreetDistrictMap2[s] = d
-					}
-				}
-			}
-		}
-	}
-	//初始化城市简称
-	fn3 := InitCitySim("v3.0")
-	AreaSimGet2 = &ju.DFA{}
-	for k, v := range fn3 {
-		pb := v["brief"].(string)
-		p := ProvinceBrief2[pb]
-		//加载
-		for _, ss := range []string{k, pb} {
-			cs := AreaToCity2[ss]
-			if cs != nil {
-				cs = append(cs, p.Captial)
-			} else {
-				cs = []*City{p.Captial}
-			}
-			AreaToCity2[ss] = cs
-			AreaSimGet2.AddWord(ss) //省全称和省简称
-		}
-		city, _ := v["city"].(map[string]interface{})
-		for k1, v1 := range city {
-			v1m, _ := v1.(map[string]interface{})
-			if v1m["brief"] == nil {
-			}
-			cb := v1m["brief"].(string)
-			c := AreaToCity2[k1][0]
-			//加入到城市map中
-			for _, ss := range []string{cb, k + cb, pb + cb} { //杭州 浙江省杭州  浙江杭州
-				AreaSimGet2.AddWord(ss)
-				cs := AreaToCity2[ss]
-				if cs != nil {
-					cs = append(cs, c)
-				} else {
-					cs = []*City{c}
-				}
-				AreaToCity2[ss] = cs
-			}
-			arr := v1m["area"].([]interface{})
-			for _, k2 := range arr {
-				s := k2.(string)
-				for n, ss := range []string{s, cb + s, pb + s, k + s} { //淳安 杭州淳安 浙江淳安 浙江省淳安
-					cs := AreaToCity2[ss]
-					AreaSimGet2.AddWord(ss)
-					if cs != nil {
-						cs = append(cs, c)
-					} else {
-						cs = []*City{c}
-					}
-					AreaToCity2[ss] = cs
-
-					//只加入简称
-					if n == 0 {
-						d := &District{}
-						d.Name = ss
-						d.C = c
-						AreaDistrict2.AddWord(ss) //加入区或县简称敏感词
-						ctmp := DistrictCityMap2[ss]
-						if ctmp == nil {
-							DistrictCityMap2[ss] = c
-						}
-					}
-				}
-			}
-		}
-	}
-}
-
-func FindBuyer() {
-	list, _ := db.Mgo.Find("buyer", nil, nil, `{"name":1}`, false, -1, -1)
-	for _, l := range *list {
-		val := qu.ObjToString(l["name"])
-		if val != "" {
-			//开始抽取城市省份
-			bres, c, p := ExtractProvinceCity2("", "", qu.BsonIdToSId(l["_id"]), []string{val})
-			bres, p, c, d := ExtractDistrict2([]string{val}, bres, c, p, qu.BsonIdToSId(l["_id"])) //抽取区或县
-			log.Println("---------------------", bres, c, p, d)
-		}
-	}
-}
-
-//抽取城市、省份
-func ExtractProvinceCity2(province, city, id string, text []string) (bres bool, c, p string) {
-	defer qu.Catch()
-	bc := true //是否继续抽取
-	if city != "" {
-		lock.Lock()
-		citybrief := CityBrief2[city]
-		//log.Println("citybrief========", citybrief)
-		lock.Unlock()
-		if citybrief == nil { //简称不存在
-			log.Println("city err:", city, id)
-		} else { //简称存在
-			lock.Lock()
-			pbrief := CityBrief2[city].P.Brief
-			//log.Println("pbrief========", pbrief)
-			lock.Unlock()
-			if province != pbrief { //省份不配对
-				log.Println("province err:", city, province, id)
-			} else {
-				bc = false
-				//城市省份都正确
-			}
-		}
-	}
-	//有省份
-	bp := false
-	lock.Lock()
-	provincebrief := ProvinceBrief2[province]
-	//log.Println("provincebrief========", provincebrief)
-	lock.Unlock()
-	if provincebrief != nil { //省份简称正确
-		bp = true
-	} else { //没有省份，先识别省份
-		for _, str := range text { //没有省的简称，从配置的字段信息中抽取省
-			word := AreaProvinceGet2.CheckSensitiveWord(str) //省全称DFA中匹配
-			if word != "" {
-				lock.Lock()
-				province = ProvinceMap2[word]
-				lock.Unlock()
-				bp = true
-				break
-			}
-		}
-	}
-	//匹配城市
-	if bc { //城市简称不存在CityBrief[city]==nil,或城市简称存在但省份不配对，继续抽取
-		for pos, GET := range []*ju.DFA{AreaGet2, AreaSimGet2} { //AreaGet市全称，AreaSimGet省全称和简称
-			ws := make([]string, 5)
-			for n, str := range text {
-				if str != "" {
-					word := GET.CheckSensitiveWord(str)
-					if pos == 1 { //用简称 后辍为路、集团替换
-						str1 := strings.Replace(str, word+"路", "", 1)
-						if str1 != str {
-							word = GET.CheckSensitiveWord(str1)
-						}
-					}
-					ws[n] = word
-					if word != "" {
-						lock.Lock()
-						res := AreaToCity2[word]
-						lock.Unlock()
-						if len(res) == 1 {
-							//判断省份
-							if !bp || province == res[0].P.Brief { //省份不存在或一致直接返回(!bp:省的简称)
-								bres = true
-								c = res[0].Brief
-								p = res[0].P.Brief
-								break
-							} else { //不一致时。。暂时不处理
-							}
-						} else { //多个时(出现这种情况是多个省中的市，市名相同。现在的配置文件中已经将市名，县名重复的全部去掉)
-						}
-					}
-				}
-			}
-			if !bres { //没有匹配到
-				mc := map[string]int{}
-				for _, w := range ws {
-					lock.Lock()
-					res := AreaToCity2[w]
-					lock.Unlock()
-					for _, ct := range res {
-						log.Println("ct===", ct)
-						if ct == nil {
-							continue
-						}
-						if bp { //有省份
-							if ct.P != nil && ct.P.Brief == province {
-								mc[ct.Brief]++
-							}
-						} else { //没有省份
-							mc[ct.Brief]++
-						}
-					}
-				}
-				//计算mc中最大值且大于1
-				max := 1
-				v := ""
-				for mk, mv := range mc {
-					if mv > max {
-						v = mk
-					}
-				}
-				if v != "" {
-					bres = true
-					lock.Lock()
-					ctb := CityBrief2[v]
-					lock.Unlock()
-					c = ctb.Brief
-					p = ctb.P.Brief
-				} else if len(mc) > 0 {
-					//取级别更大的
-					v := ""
-					for mk, _ := range mc {
-						lock.Lock()
-						cb := CityBrief2[mk]
-						lock.Unlock()
-						if cb.P.Cap == mk {
-							bres = true
-							c = cb.Brief
-							p = cb.P.Brief
-							break
-						} else {
-							v = mk
-						}
-					}
-					if !bres {
-						bres = true
-						lock.Lock()
-						cbb := CityBrief2[v]
-						c = cbb.Brief
-						p = cbb.P.Brief
-						lock.Unlock()
-					}
-				}
-			}
-			if bres {
-				break
-			}
-		}
-	} else {
-		return
-	}
-	if !bres {
-		//取默认省会
-		lock.Lock()
-		pbp := ProvinceBrief2[province]
-		lock.Unlock()
-		if pbp != nil {
-			bres = true
-			c = pbp.Cap
-			p = province
-		}
-	}
-	return
-}
-func ExtractDistrict2(field []string, bres bool, c, p, id string) (bool, string, string, string) {
-	d := ""
-	for _, str := range field {
-		//log.Println("field===========", str)
-		for pos, GET := range []*ju.DFA{AreaDistrict2, AreaStreet2} { //先匹配区或县再匹配街道
-			word := GET.CheckSensitiveWord(str)
-			//log.Println("word================", word)
-			if word != "" {
-				if pos == 0 { //区或县匹配
-					//log.Println("县直接匹配到====", word)
-					lock.Lock()
-					city := DistrictCityMap2[word]
-					lock.Unlock()
-					//log.Println("city================", city)
-					if city != nil {
-						d = word
-						ctmp := city.Brief
-						ptmp := city.P.Brief
-						//log.Println("ctmpptmp================", ptmp, ctmp)
-						if !bres { //城市省份没有抽到，通过区或县定位市和省
-							c = ctmp
-							p = ptmp
-							bres = true
-						} else { //对比抽到的城市省份是否一致
-							if c != ctmp || p != ptmp {
-								//log.Println("str---", str, "====", word)
-								//log.Println("district: City And Province, Inconsistent Before And After,Id:", id, c, p, ctmp, ptmp, d)
-								c = ctmp
-								p = ptmp
-							}
-						}
-					}
-				} else { //街道匹配
-					//log.Println("匹配到街道====", word)
-					lock.Lock()
-					district := StreetDistrictMap2[word]
-					lock.Unlock()
-					//log.Println("district================", district)
-					if district != nil {
-						d = district.Name
-						ctmp := district.C.Brief
-						ptmp := district.C.P.Brief
-						//log.Println("districtptmp================", ctmp, ptmp)
-						if !bres { //城市省份没有抽到，通过区或县定位市和省
-							c = ctmp
-							p = ptmp
-							bres = true
-						} else { //对比抽到的城市省份是否一致
-							if c != ctmp || p != ptmp {
-								//log.Println("street: City And Province, Inconsistent Before And After,Id:", id, c, p, ctmp, ptmp, d)
-								c = ctmp
-								p = ptmp
-							}
-						}
-					}
-				}
-				return bres, p, c, d
-			}
-		}
-	}
-	return bres, p, c, d
-}

src/jy/extract/extractudp.go

@@ -230,7 +230,7 @@ func ExtractByUdp(sid, eid string, ra *net.UDPAddr, instanceId ...string) {
 				//	log.Debug(index, qu.BsonIdToSId(v["_id"]), "//去除含敏感词数据")
 				//	continue
 				//}
-				if qu.ObjToString(v["spidercode"]) == "a_gjggzyjypt_gcjs_kbjl" { //临时开标记录
+				if qu.ObjToString(v["spidercode"]) == "a_gjggzyjypt_gcjs_kbjl"  || "a_hbszbtbggfwpt_kbjl" == qu.ObjToString(v["spidercode"]) { //临时开标记录
 					log.Debug(index, qu.BsonIdToSId(v["_id"]), "//开标记录")
 					continue
 				}

src/main_blocktest.go

@@ -1,264 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"jy/extract"
-	"jy/mongodbutil"
-	"jy/pretreated"
-	ju "jy/util"
-	"log"
-	"os"
-	qu "qfw/util"
-	"regexp"
-	"time"
-)
-
-var f *os.File
-var m = map[string]bool{}
-
-func main1() {
-	//winnerorder()
-	//return
-	//log.Println(pretreated.ProcTitle("以上公告内容如有变动将在相关网络媒体上另行通知凡购买本招标文件的单位必须就此采购项目的相关事宜详细咨询否则参与投标即被视为已经充分了解了招标方的需求中标后承担该文件范围内的所有要求投标前如对招标文件存有疑问请在投标截止日期前三个工作日以实名制书面文件向我公司询问否则视为接受已报名购买招标文件的投标商未递交投标文件或虽递交投标文件但未参加开标大会的投标商不得再参加该项目的采购活动"))
-	//return
-	//f, _ = os.OpenFile("./title.txt", os.O_RDWR|os.O_CREATE, 777)
-	//all()
-	one()
-}
-func all() {
-	m := mongodbutil.MgoFactory(3, 3, 120, "127.0.0.1:27092", "extract_kf")
-	sess := m.Get()
-	defer m.Close(sess)
-	it := sess.DB("extract_kf").C("bidding201901").Find(nil).Iter()
-	pool := make(chan bool, 5)
-	count := 0
-	for temp := make(map[string]interface{}); it.Next(&temp); {
-		pool <- true
-		count++
-		go func(d map[string]interface{}) {
-			defer func() {
-				<-pool
-			}()
-			com(d)
-		}(temp)
-		temp = make(map[string]interface{})
-		if count%200 == 0 {
-			log.Println(count)
-		}
-	}
-	log.Println("over...")
-	time.Sleep(time.Hour)
-}
-func one() {
-	m := mongodbutil.MgoFactory(3, 3, 120, "127.0.0.1:27092", "extract_kf")
-	d, _ := m.FindById("bidding_data2", "5e59321150b5ea296ed84985", extract.Fields)
-	com(*d)
-}
-func com(doc map[string]interface{}) {
-	detail := GetDetail(doc)
-	doc["detail"] = detail
-	toptype := qu.ObjToString(doc["toptype"])
-	subtype := qu.ObjToString(doc["subtype"])
-	if qu.ObjToString(doc["type"]) == "bid" {
-		toptype = "结果"
-	}
-	if toptype == "" {
-		toptype = "*"
-	}
-	e := &extract.ExtractTask{
-		TaskInfo: &extract.TaskInfo{
-			Version:     "v3.6",
-			VersionId:   "5e02be5869841446c0005a93",
-			ProcessPool: make(chan bool, 1),
-		},
-	}
-	e.Id = qu.ObjToString(ju.Config["udptaskid"])
-	e.InitTaskInfo()
-	//d.TaskInfo.FDB = db.MgoFactory(3, 5, 600, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
-	//d.TaskInfo.TDB = db.MgoFactory(3, 5, 600, ext.TaskInfo.ToDbAddr, ext.TaskInfo.ToDB)
-	e.InitSite()
-	e.InitRulePres()
-	e.InitRuleBacks(false)
-	//e.InitRuleBacks(true)
-	e.InitRuleCore(false)
-	//e.InitRuleCore(true)
-	e.InitBlockRule()
-	e.InitPkgCore()
-	e.InitTag(false)
-	//e.InitTag(true)
-	e.InitClearFn(false)
-	//e.InitClearFn(true)
-	if e.IsExtractCity { //版本上控制是否开始城市抽取
-		//初始化城市DFA信息
-		e.InitCityDFA()
-		e.InitAreaCode()
-		e.InitPostCode()
-	}
-	//质量审核
-	e.InitAuditFields()
-	e.InitAuditRule()
-	e.InitAuditClass()
-	e.InitAuditRecogField()
-
-	//品牌抽取是否开启
-	ju.IsBrandGoods, _ = ju.Config["brandgoods"].(bool)
-
-	j := &ju.Job{
-		SourceMid:      qu.BsonIdToSId(doc["_id"]),
-		Category:       toptype,
-		CategorySecond: subtype,
-		Content:        qu.ObjToString(doc["detail"]),
-		SpiderCode:     qu.ObjToString(doc["spidercode"]),
-		Site:           qu.ObjToString(doc["site"]),
-		//Domain:     qu.ObjToString(doc["domain"]),
-		//Href:       qu.ObjToString(doc["href"]),
-		Title:     qu.ObjToString(doc["title"]),
-		Data:      &doc,
-		City:      qu.ObjToString(doc["city"]),
-		Province:  qu.ObjToString(doc["area"]),
-		Result:    map[string][]*ju.ExtField{},
-		BuyerAddr: qu.ObjToString(doc["buyeraddr"]),
-		RuleBlock: e.RuleBlock,
-		Dataging:  qu.IntAll(doc["dataging"]),
-	}
-	e.TaskInfo.ProcessPool <- true
-	pretreated.AnalyStart(j, false, "")
-	e.ExtractProcess(j, nil, false)
-	log.Println("=============块信息================")
-	for _, v := range j.Block {
-		log.Println("----", v.Title, v.Titles, "----")
-		if v.ColonKV != nil {
-			for kk, vv := range v.ColonKV.KvTags {
-				for _, vvv := range vv {
-					log.Println("ColonKV", kk, "---", vvv.Key, "---", vvv.Value, "---", vvv.Weight)
-				}
-			}
-		}
-		if v.SpaceKV != nil {
-			for kk, vv := range v.SpaceKV.KvTags {
-				for _, vvv := range vv {
-					log.Println("SpaceKV", kk, "---", vvv.Key, "---", vvv.Value, "---", vvv.Weight)
-				}
-			}
-		}
-		if v.TableKV != nil {
-			for kk, vv := range v.TableKV.KvTags {
-				for _, vvv := range vv {
-					log.Println("TableKV", kk, "---", vvv.Key, "---", vvv.Value, "---", vvv.Weight)
-				}
-			}
-		}
-		//log.Println("Classify", v.Classify)
-		//log.Println("Tag", v.Tag)
-	}
-	log.Println("=============抽取结果================")
-	//	log.Println(e.ResultArr)
-	set := (e.ResultArr[0][1]["$set"]).(map[string]interface{})
-	for k, v := range set {
-		//if k == "budget" || k == "bidamount" || k == "winner" || k == "amount" || k == "projectname" || k == "projectcode" || k == "buyer" || k == "buyerperson" || k == "buyertel" || k == "agency" {
-		log.Println(k, "---", v)
-		//}
-	}
-	log.Println("=============抽取结果 result================")
-	return
-	for k, v := range set["result"].(map[string][]*ju.ExtField) {
-		if k != "winner" {
-			continue
-		}
-		for _, vv := range v {
-			log.Println(k, fmt.Sprintf("%+v", vv))
-			for kkk, vvv := range vv.ScoreItem {
-				log.Println("--", kkk, k, fmt.Sprintf("%+v", vvv))
-			}
-			//log.Println("\n")
-		}
-	}
-	log.Println("=============中标候选人================")
-	for _, v := range j.Winnerorder {
-		log.Println(v)
-	}
-	log.Println("=============分包================")
-	for k, v := range j.BlockPackage {
-		log.Println(k, v)
-	}
-	log.Println("=============正文================")
-	//log.Println(j.Content)
-	return
-	for _, v := range j.Block {
-		if v.ColonKV != nil && v.ColonKV.KvTags != nil {
-			for kk, vv := range v.ColonKV.KvTags {
-				for _, vvv := range vv {
-					log.Println(kk, vvv.Weight, vvv.Value)
-				}
-			}
-		}
-		if v.TableKV != nil && v.TableKV.KvTags != nil {
-			for kk, vv := range v.TableKV.KvTags {
-				for _, vvv := range vv {
-					log.Println(kk, vvv.Weight, vvv.Value)
-				}
-			}
-		}
-		if v.SpaceKV != nil && v.SpaceKV.KvTags != nil {
-			for kk, vv := range v.SpaceKV.KvTags {
-				for _, vvv := range vv {
-					log.Println(kk, vvv.Weight, vvv.Value)
-				}
-			}
-		}
-	}
-	log.Println(len(j.Block))
-	return
-	for _, v := range j.Block {
-		if m[v.Title] || v.Title == "" {
-			continue
-		}
-		if !regexp.MustCompile("或|和|以?及|与|、|或").MatchString(v.Title) {
-			//continue
-		}
-		m[v.Title] = true
-		f.WriteString(j.SourceMid + "-----" + v.Title + "---" + fmt.Sprint(v.Titles) + "\n")
-		continue
-		for _, kv := range v.ColonKV.Kvs {
-			//log.Println("\n")
-			log.Println(kv.Key, "---", kv.Value)
-			log.Println(kv.Line)
-			log.Println("=======================")
-		}
-	}
-}
-
-func GetDetail(doc map[string]interface{}) (detail string) {
-	detail = ""
-	d1, _ := doc["detail"].(string)
-	d2, _ := doc["contenthtml"].(string)
-	if len(d1) >= len(d2) || d2 == "" {
-		detail = d1
-	} else {
-		detail = d2
-	}
-	detail = ju.CutLableStr(detail)
-	detail = ju.NewCut().ClearHtml(detail)
-	tabs, ration := pretreated.ComputeConRatio(detail, 1)
-	if len(tabs) > 0 {
-		newcon, newtabs, newration := pretreated.FindBigText(detail, ration, tabs)
-		//log.Println(newcon, newtabs, newration)
-		if newcon != "" && newration == 0 {
-			detail = newcon
-			tabs = newtabs
-			ration = newration
-		}
-	}
-	return detail
-}
-func winnerorder() {
-	text := `评审专家名单：1
-吴殿波、韩屹、孙胜进、郑丹、李海波
- 
-中标标的名称、规格型号、数量、单价、服务要求：
-2019年沈阳惠涌供热有限责任公司、沈阳圣达热力供暖有限责任公司、沈阳惠盛供热有限责任公司PE管保温
-第一入围供货商：沈阳曲暖鼎盛保温安装有限公司 、总单价：11.833300
-第二入围供货商：沈阳国盛防腐保温有限公司、总单价：11.102100
-第三入围供货商：沈阳泰豪管材有限公司、总单价：13.258100`
-	log.Println((&pretreated.WinnerOrderEntity{}).Find(text, true, 1, false, ""))
-}

src/main_test.go

@@ -1,141 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"jy/admin/track"
-	"jy/clear"
-	"jy/extract"
-	. "jy/mongodbutil"
-	"log"
-	"os"
-	"qfw/util"
-	"regexp"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-)
-
-func Test_han(t *testing.T) {
-	str := `[\u4e00-\u9fa5]` //"[\u4e00-\u9fa5]"
-	//var rg = regexp.MustCompile(`[\u4e00-\u9fa5]`)会出错
-	if strings.Contains(str, "\\u") {
-		pattern, _ := strconv.Unquote(`"` + str + `"`)
-		log.Println(pattern)
-	}
-	var rg = regexp.MustCompile(str)
-	fmt.Println(rg.MatchString(str))
-	os.Exit(0)
-}
-func Test_task(t *testing.T) {
-	Mgo = MgoFactory(1, 3, 120, "192.168.3.207:27092", "extract_kf")
-	//extract.StartExtractTaskId("5b8f804025e29a290415aee1")5c528686698414055c47b115
-	//extract.StartExtractTestTask("5e103206234ddc34b406c5d1", "5df59ee3e9d1f601e46fc3f9", "1", "mxs_v1", "mxs_v1")
-	extract.StartExtractTestTask("5cdd3025698414032c8322b1", "5df50776e9d1f601e4964179", "1", "mxs_v1", "mxs_v2")
-	//extract.StartExtractTestTask("5c3d75c96984142998eb00e1", "5c2a3d28a5cb26b9b76144dd", "100", "mxs_v3", "mxs_v3")
-	time.Sleep(5 * time.Second)
-}
-func Test_extractcity(t *testing.T) {
-	Mgo = MgoFactory(1, 3, 120, "192.168.3.207:27082", "extract_kf")
-	extract.InitDFA2()
-	//查询采购单位信息
-	extract.FindBuyer()
-}
-func Test_reg(t *testing.T) {
-	reg1, _ := regexp.Compile("((([（(]\\d{3,4}[)）])?(\\d{6,12}([×―—－\\-]+\\d{3,4})?|\\d{3,5}[×―—－\\-]+[\u3000\u2003\u00a0\\s]*\\d{6,12}([×―—－\\-]+\\d{3,})?|(\\d{2}[×―—－\\-])+\\d{8}[×―—－\\-](\\d{3}[、])+)(转\\d{3,4})?[或/、,，;；\u3000\u2003\u00a0\\s]*)+(\\d{3,})?)")
-	log.Println("---", reg1.FindAllString("05939-5365001(兰陵县芦柞镇人民政府)", -1))
-	reg2, _ := regexp.Compile("^\\d*[×―—－\\-]*[\u3000\u2003\u00a0\\s]*\\d*$")
-	log.Println("---", reg2.MatchString("张女士/"))
-	filterK := regexp.MustCompile("[(（\\[【].*?[)）\\]】]|<[^>].+?>|[①②③￥·；;‘“'’”,*<>(（\\[【、)）/\\]】？?,。．\".\\s\u3000\u2003\u00a0]+|^[一二三四五六七八九十0-9１２３４５６７８９０]+")
-	log.Println(filterK.FindString("二）采购项目联系人（代理机构）"))
-}
-
-func Test_reg1(t *testing.T) {
-	context := `sss<input  name="AgentCode" size="30" maxsize="50" value="91370800688271668P" class="textbox">
-    dfdf<input type="hidden" name="AgentCode" size="30" maxsize="50" value="tttt" class="textbox"></input>`
-	reg := regexp.MustCompile(`<\s*input.*value=['"](.[^'"]+).+>`)
-	tmp := reg.ReplaceAllString(context, "$1")
-	log.Println(tmp)
-}
-
-func Test_paths(t *testing.T) {
-	Mgo = MgoFactory(1, 3, 120, "192.168.3.207:27082", "extract_v3")
-	tracks := track.GetTrackPath("5b8dd276a5cb26b9b7faaa7c", "projectname", "rack_v3", "result_v3")
-	for code, v := range tracks {
-		if tmp, ok := v.([]map[string]interface{}); ok {
-			for k, v := range tmp {
-				if k == 0 {
-					log.Println(v)
-				} else {
-					log.Println(code, v["code"], v["value"])
-				}
-			}
-		}
-		break
-	}
-}
-
-func Test_clear(t *testing.T) {
-	value := "法拉(盛（客{）户)端副科级沙发俩括号的"
-	log.Println("pre---", value)
-	startChars := []string{"[（(]", "[\\[【]", "[{｛]", "[<《]", "〔"}
-	endChars := []string{"[)）]", "[\\]】]", "[}｝]", "[>》]", "〕"}
-	for k, v := range startChars {
-		sReg := regexp.MustCompile(v)
-		eReg := regexp.MustCompile(endChars[k])
-		sIndex := sReg.FindAllStringIndex(value, -1)
-		eIndex := eReg.FindAllStringIndex(value, -1)
-		sCount := len(sIndex)
-		eCount := len(eIndex)
-		if sCount == eCount {
-			continue
-		}
-		log.Println("value1---", value, sCount, eCount)
-		//清理前面
-		if sCount > eCount {
-			value = value[sIndex[eCount][1]:]
-		}
-		log.Println("value2---", value)
-		//清理后面
-		if sCount < eCount {
-			value = value[:eIndex[sCount][0]]
-		}
-		log.Println("value3---", value)
-	}
-	log.Println("value4---", value)
-	//交叉出现情况处理
-	sReplReg := regexp.MustCompile("[（(\\[【{｛〔<《][^）)\\]】}｝〕>》]*$")
-	eReplReg := regexp.MustCompile("^[^（(\\[【{｛〔<《]*[）)\\]】}｝〕>》]")
-	if sReplReg.MatchString(value) || eReplReg.MatchString(value) {
-		value = sReplReg.ReplaceAllString(value, "")
-		value = eReplReg.ReplaceAllString(value, "")
-		//value = fmt.Sprint(childCutNotPrs([]interface{}{value, data[1]}, count+1)[0])
-	}
-	log.Println("result---", value)
-}
-
-func Test_buyer(t *testing.T) {
-	Mgo = MgoFactory(1, 3, 120, "192.168.3.207:27092", "extract_kf")
-	demo, _ := Mgo.Find("demo_data", nil, `{"_id:1"}`, `{"buyer":1,"title":1}`, false, -1, -1)
-	result, _ := Mgo.Find("mxs_buyer", nil, `{"_id:1"}`, `{"buyer":1}`, false, -1, -1)
-	for _, d := range *demo {
-		id1 := util.BsonIdToSId(d["_id"])
-		buyer1 := util.ObjToString(d["buyer"])
-		//title := util.ObjToString(d["title"])
-		for _, r := range *result {
-			id2 := util.BsonIdToSId(r["_id"])
-			buyer2 := util.ObjToString(r["buyer"])
-			if id1 == id2 {
-				if buyer1 != buyer2 {
-					util.Debug(id1, buyer1, buyer2)
-				}
-				break
-			}
-		}
-	}
-}
-
-func Test_util1(t *testing.T) {
-	data := clear.CutSymbol([]interface{}{"----------123123", "-----123123"})
-	fmt.Println(data)
-}