备份

1、抽取城市新增逻辑校验
2、控制中心bug解决（互斥锁）
3、抽取平台页面展示table列表  宽度适配调整
4、城市服务接口备份

extcity/src/ext/extInit.go

@@ -4,11 +4,13 @@ import (
 	log "github.com/donnie4w/go-logger/logger"
 	"github.com/go-ego/gse"
 	"github.com/sensitive"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	qu "qfw/util"
 	ul "util"
 )
 
 var Ext *ExtractTask
+var RegionsConfig map[string]interface{}
 var DefaultRegions, AdjustmentRegions = []string{}, []string{}
 
 type ExtractTask struct {
@@ -52,6 +54,49 @@ type ExtractTask struct {
 	Seg_SV              *gse.Segmenter //分词
 }
 
+func ConvertInterface(t interface{}) []string {
+	p_list := []string{}
+	if yl_list_1, ok_1 := t.(primitive.A); ok_1 {
+		p_list = qu.ObjArrToStringArr(yl_list_1)
+	} else {
+		if yl_list_2, ok_2 := t.([]interface{}); ok_2 {
+			p_list = qu.ObjArrToStringArr(yl_list_2)
+		}
+	}
+	return p_list
+}
+
+//初始化邮编库
+func (e *ExtractTask) InitPostCode() {
+	defer qu.Catch()
+	e.PostCodeMap = make(map[string]*PostCode)
+	query := map[string]interface{}{}
+	list, _ := ul.ExtMgo.Find("postcode", query, nil, nil)
+	for _, l := range list {
+		pc := &PostCode{}
+		pc.Code = qu.ObjToString(l["code"])
+		pc.P = qu.ObjToString(l["province"])
+		pc.C = qu.ObjToString(l["city"])
+		pc.D = ConvertInterface(l["district"])
+		e.PostCodeMap[pc.Code] = pc
+	}
+}
+
+//初始化区号库
+func (e *ExtractTask) InitAreaCode() {
+	defer qu.Catch()
+	e.AreaCodeMap = make(map[string]*AreaCode)
+	query := map[string]interface{}{}
+	list, _ := ul.ExtMgo.Find("areacode", query, nil, nil)
+	for _, l := range list {
+		ac := &AreaCode{}
+		ac.Code = qu.ObjToString(l["code"])
+		ac.P = qu.ObjToString(l["province"])
+		ac.C = ConvertInterface(l["city"])
+		e.AreaCodeMap[ac.Code] = ac
+	}
+}
+
 //加载所有
 func InitProvincesx() []map[string]interface{} {
 	defer qu.Catch()
@@ -109,11 +154,12 @@ func (e *ExtractTask) InitUpdateSite() {
 
 func (e *ExtractTask) InitCityInfo() {
 	defer qu.Catch()
-	e.InitVar() //初始化变量
-	//新疆兵团数据
-	e.InitXjbtCityInfo()
-	//site站点信息
-	e.InitUpdateSite()
+	e.InitVar()          //初始化变量
+	e.InitXjbtCityInfo() //新疆兵团数据
+	e.InitUpdateSite()   //site站点信息
+	e.InitPostCode()     //邮编信息
+	e.InitAreaCode()     //区号信息
+
 	//初始化省信息
 	alldata := InitProvincesx()
 	fnx := make([]map[string]interface{}, 0)
@@ -348,6 +394,8 @@ func (e *ExtractTask) InitVar() {
 
 	//初始化城市相关
 	e.SiteCityMap = make(map[string]*SiteCity)
+	e.PostCodeMap = make(map[string]*PostCode)
+	e.AreaCodeMap = make(map[string]*AreaCode)
 	e.ProvinceMap = make(map[string]string)
 	e.CityMap = make(map[string]string)
 	e.DistrictSimAndAll = make(map[string][]map[string]*City)

extcity/src/main.go

@@ -1,9 +1,13 @@
 package main
 
 import (
+	"bytes"
+	"encoding/json"
 	"ext"
 	log "github.com/donnie4w/go-logger/logger"
+	"io/ioutil"
 	"net/http"
+	qu "qfw/util"
 	"service"
 	"time"
 	ul "util"
@@ -15,9 +19,23 @@ func init() {
 func main() {
 	ext.Ext = &ext.ExtractTask{}
 	ext.Ext.InitCityInfo() //初始化-抽取城市
+	qu.ReadConfig("./res/regions.json", &ext.RegionsConfig)
+	ext.DefaultRegions = ext.ConvertInterface(ext.RegionsConfig["default_regions"])
+	ext.AdjustmentRegions = ext.ConvertInterface(ext.RegionsConfig["adjustment_regions"])
 	log.Debug("地域模型数据加载完毕~~~")
 	service.InitRegionsService()
 	log.Debug("开始监听~", ul.Port)
 	http.ListenAndServe(ul.Port, nil)
 	time.Sleep(99999 * time.Hour)
 }
+
+func test() {
+	client := &http.Client{Timeout: 30 * time.Second}
+	data := map[string]interface{}{"detail": "我是正文郑州"}
+	jsonStr, _ := json.Marshal(data)
+	resp, _ := client.Post("http://127.0.0.1:9991/service/region", "application/json", bytes.NewBuffer(jsonStr))
+	res_info, _ := ioutil.ReadAll(resp.Body)
+	info := map[string]interface{}{}
+	json.Unmarshal(res_info, &info)
+	log.Debug("接收返回值~", info)
+}

extcity/src/service/service.go

@@ -3,6 +3,8 @@ package service
 import (
 	"encoding/json"
 	"ext"
+	log "github.com/donnie4w/go-logger/logger"
+	"io/ioutil"
 	"net/http"
 	qu "qfw/util"
 	"sync"
@@ -13,10 +15,12 @@ var datalock sync.Mutex
 func InitRegionsService() {
 	http.HandleFunc("/service/region", func(w http.ResponseWriter, r *http.Request) {
 		datalock.Lock()
-		tmp := map[string]interface{}{
-			"detail": "金水区",
+		data, _ := ioutil.ReadAll(r.Body)
+		tmp := map[string]interface{}{}
+		err := json.Unmarshal([]byte(data), &tmp)
+		if err != nil {
+			log.Debug(err)
 		}
-		//需要构建解析的模型~
 		ext.Ext.ExtractRegionInfo(&tmp, true)
 		area := qu.ObjToString(tmp["area"])
 		city := qu.ObjToString(tmp["city"])

extcity/src/util/init.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"log"
 	qu "qfw/util"
 )
@@ -14,7 +15,6 @@ var (
 
 func InitExt() {
 	qu.ReadConfig("./res/config.json", &Sysconfig)
-	log.Println(Sysconfig)
 	if len(Sysconfig) == 0 {
 		log.Fatalln("init config err")
 	}
@@ -46,3 +46,15 @@ func initMgo() {
 	}
 	SiteMgo.InitPool()
 }
+
+func convertInterface(t interface{}) []string {
+	p_list := []string{}
+	if yl_list_1, ok_1 := t.(primitive.A); ok_1 {
+		p_list = qu.ObjArrToStringArr(yl_list_1)
+	} else {
+		if yl_list_2, ok_2 := t.([]interface{}); ok_2 {
+			p_list = qu.ObjArrToStringArr(yl_list_2)
+		}
+	}
+	return p_list
+}

src/config.json

@@ -3,7 +3,7 @@
     "mgodb": "127.0.0.1:27017",
     "dbsize": 3,
     "dbname": "extract_local",
-    "dbname_addrs": "mixdata",
+    "dbname_addrs": "extract_service",
     "dbname_addrs_c": "address_new_2020",
     "site_addr": "127.0.0.1:27017",
     "site_dbname": "extract_local",

src/jy/extract/extract.go

@@ -699,11 +699,10 @@ func (e *ExtractTask) ExtractDetail(j *ju.Job, isSite bool, codeSite string) {
 				if !ju.Logic(vc.LuaLogic, tmp) {
 					continue
 				}
-
 				if vc.Field == "winner" {
 					//log.Debug("调试抽取字段")
 				}
-				////抽取-前置规则
+				//抽取-前置规则
 				//for _, v := range vc.RulePres {
 				//	tmp = ExtRegPre(tmp, j, v, e.TaskInfo)
 				//}
@@ -2466,7 +2465,6 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 				delete(tmp,k)
 			}
 		}*/
-
 		//检查字段
 		tmp = checkFields(tmp, *j.Data)
 
@@ -2480,6 +2478,13 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 			}
 		}
 
+		//临时保存指定字段数据
+		//new_tmp := map[string]interface{}{}
+		//new_tmp["area"] = qu.ObjToString(tmp["area"])
+		//new_tmp["city"] = qu.ObjToString(tmp["city"])
+		//new_tmp["district"] = qu.ObjToString(tmp["district"])
+		//tmp = new_tmp
+
 		if e.TaskInfo.TestColl == "" {
 			if len(tmp) > 0 { //保存抽取结果
 				delete(tmp, "_id")

src/jy/extract/extractInit.go

@@ -3,6 +3,7 @@ package extract
 
 import (
 	"github.com/sensitive"
+	"gopkg.in/mgo.v2/bson"
 	db "jy/mongodbutil"
 	ju "jy/util"
 	qu "qfw/util"
@@ -13,8 +14,6 @@ import (
 	"sync"
 	"time"
 
-	"gopkg.in/mgo.v2/bson"
-
 	log "github.com/donnie4w/go-logger/logger"
 	"github.com/go-ego/gse"
 )
@@ -1299,7 +1298,8 @@ func (e *ExtractTask) InitPostCode() {
 		pc.Code = qu.ObjToString(l["code"])
 		pc.P = qu.ObjToString(l["province"])
 		pc.C = qu.ObjToString(l["city"])
-		pc.D = qu.ObjArrToStringArr(l["district"].([]interface{}))
+		pc.D = ju.ConvertInterface(l["district"])
+		//pc.D = qu.ObjArrToStringArr(l["district"].([]interface{}))
 		e.PostCodeMap[pc.Code] = pc
 	}
 }
@@ -1313,7 +1313,8 @@ func (e *ExtractTask) InitAreaCode() {
 		ac := &AreaCode{}
 		ac.Code = qu.ObjToString(l["code"])
 		ac.P = qu.ObjToString(l["province"])
-		ac.C = qu.ObjArrToStringArr(l["city"].([]interface{}))
+		ac.C = ju.ConvertInterface(l["city"])
+		//ac.C = qu.ObjArrToStringArr(l["city"].([]interface{}))
 		e.AreaCodeMap[ac.Code] = ac
 	}
 }

src/jy/extract/extractcity_new.go

@@ -80,6 +80,15 @@ func (e *ExtractTask) ExtractRegionInfo(j *ju.Job, tmp *map[string]interface{},
 	if f_city == "" {
 		e.LinkSpecialRuleFullStep(projectname, &f_area, &f_city, &f_district)
 	}
+	//企业补充城市校验逻辑
+	if buyer != "" && f_city == "" && (f_area == "全国" || f_area == "") {
+		LinkSpecialQyxyStep(buyer, &f_area, &f_city, &f_district)
+		if f_area != "" && f_area != "全国" && isLog {
+			LogProcessRecordingForSupplement(&logRecordInfo, map[string]interface{}{
+				"sup_qy_link": f_area + "~" + f_city + "~" + f_district,
+			})
+		}
+	}
 	if f_city == "" {
 		e.LinkSpecialRuleBriefStep(projectname, &f_area, &f_city, &f_district)
 	}

src/jy/extract/extractcity_way.go

@@ -1,6 +1,7 @@
 package extract
 
 import (
+	ju "jy/util"
 	qu "qfw/util"
 	"regexp"
 	"strings"
@@ -595,6 +596,27 @@ func LinkSuppleRules(regions map[string]map[string]map[string]string, area *stri
 
 }
 
+//链路补充~企业校验步骤
+func LinkSpecialQyxyStep(buyer string, area *string, city *string, district *string) {
+	qyxy_arr := make([]map[string]interface{}, 0)
+	ju.QyxySess.Find(map[string]interface{}{
+		"company_name": buyer,
+	}).All(&qyxy_arr)
+	if qyxy_arr != nil && len(qyxy_arr) > 0 {
+		for _, v := range qyxy_arr {
+			qy_area := qu.ObjToString(v["company_area"])
+			qy_city := qu.ObjToString(v["company_city"])
+			qy_district := qu.ObjToString(v["company_district"])
+			if qy_area != "" && qy_area != "全国" {
+				*area = qy_area
+				*city = qy_city
+				*district = qy_district
+				break
+			}
+		}
+	}
+}
+
 //同组合并的地域数据
 //func MergeGroupRegionInfo(datas_1, datas_2 map[string]map[string]map[string]string) map[string]map[string]map[string]string {
 //	regions := map[string]map[string]map[string]string{}

src/jy/util/util.go

@@ -27,6 +27,7 @@ type Trie struct {
 var syncint chan bool //获取下标锁
 var Config map[string]interface{}
 var RegionsConfig map[string]interface{}
+
 var Se = qu.SimpleEncrypt{Key: "topnet@extract"}
 
 var BrandRules map[string]map[string]string
@@ -82,8 +83,8 @@ func UtilInit() {
 		PriceNumberReg[k] = regexp.MustCompile(v)
 	}
 
-	DefaultRegions = convertInterface(RegionsConfig["default_regions"])
-	AdjustmentRegions = convertInterface(RegionsConfig["adjustment_regions"])
+	DefaultRegions = ConvertInterface(RegionsConfig["default_regions"])
+	AdjustmentRegions = ConvertInterface(RegionsConfig["adjustment_regions"])
 
 	//定时更新站点信息
 	IsUpdateSite = false
@@ -237,7 +238,7 @@ func FloatFormat(val float64, length int) float64 {
 	}
 }
 
-func convertInterface(t interface{}) []string {
+func ConvertInterface(t interface{}) []string {
 	p_list := []string{}
 	if yl_list_1, ok_1 := t.(primitive.A); ok_1 {
 		p_list = qu.ObjArrToStringArr(yl_list_1)

src/web/templates/admin/distribution.html

@@ -76,11 +76,11 @@ $(function () {
 		"columns": [
             { "data": "TaskName"},
             { "data": "InstanceName"},
-			{ "data": "InstanceId","width":"15%"},
-			{ "data": "ip_nw","width":"17%",render:function(val,a,row,pos){
+			{ "data": "InstanceId","width":"10%"},
+			{ "data": "ip_nw","width":"10%",render:function(val,a,row,pos){
                 return val+"("+row.ip_ww+")"
             }},
-			{ "data": "AutoReleaseTime","width":"15%"},
+			{ "data": "AutoReleaseTime","width":"10%"},
             { "data": "Status"},
             { "data": "extstatus",render:function(val,a,row,pos){
                if(val){

udpcontrol/src/updprocess.go

@@ -16,6 +16,7 @@ var (
 	nextNode                                           []map[string]interface{}
 	udpclient                                          mu.UdpClient
 	udplock                                            sync.Mutex
+	nextlock                                           sync.Mutex
 	extractAction                                      map[string]map[string]interface{}
 	heartAction                                        map[string]interface{}
 	isAction                                           bool
@@ -52,7 +53,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 		}
 	case mu.OP_NOOP: //下个节点回应
 		//抽取多节点
-		udplock.Lock()
+		nextlock.Lock()
 		str := string(data)
 		if isAction {
 			if strings.Contains(str, "heart_extract") {
@@ -64,7 +65,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 			log.Debug("其他节点回应：", str)
 			udptaskmap.Delete(str)
 		}
-		udplock.Unlock()
+		nextlock.Unlock()
 	}
 }