xiugai

jyinfomatch/src/config.json

@@ -4,11 +4,12 @@
     "durationMinutes": 10,
     "mgoSize": 50,
     "mgoAddr": "192.168.3.18:27080",
-	"mongodbName": "qyfw",
-	"collection": "matchresult",
+	"mongodbName": "jyqyfw",
+	"collection": "usermail",
     "elasticPoolSize": 10,
     "elasticsearch": "http://192.168.3.18:9800",
-    "maxSearch": 400,
+    "maxSearch": 600,
 	"saveSize": 100,
-	"poolSize": 100
+	"poolSize": 100,
+	"fields":["_id","title","detail","projectscope","publishtime","toptype","subtype","type","area","href","projectname","projectcode","s_winner","buyer","budget","bidamount","bidopentime","s_subscopeclass","bidstatus","agency","projectinfo"]
 }

jyinfomatch/src/luascript/dev.lua

@@ -1,5 +1,5 @@
 --用户唯一标识
-userId="1"
+appid="jyMi1XQgMABQNcSkBMIhBq"
 --匹配词
 keys1={
 	{"政务云","不动产登记","涉密网","机要网","质量监督管理局","政法网","电子政务","警务云"},
@@ -22,7 +22,7 @@ function filterValidate(data)
 	--print(title)
 	if title ~= nil and title ~= "" then
 		--标题匹配
-		y_p,y_k,y_ok = contain(title)
+		y_p,y_k,y_ok = match(title)
 		--print("title包含--",y_p,y_k,y_ok)
 		--如果标题匹配上，进行标题排除
 		if y_ok then
@@ -45,7 +45,7 @@ function filterValidate(data)
 		--如果标题没有匹配上，匹配正文
 		if not y_ok then
 			--排除匹配上的词，对应的保留词，是否成功
-			y_p,y_k,y_ok = contain(detail)
+			y_p,y_k,y_ok = match(detail)
 			--print("detail包含--",y_p,y_k,y_ok)
 		end
 		--匹配上标题或者正文，都要进行正文排除
@@ -69,34 +69,31 @@ function filterValidate(data)
 	return data
 end
 --包含
-function contain(value)
+function match(value)
 	for k,keys in pairs(keys1) do
-		for i,key in pairs(keys) do
-			if string.find(value,key) ~= nil then
-				return keys1_1[k],key,true
-			end
+		local ok,matchkeys = contain(value,keys)
+		if ok then
+			return keys1_1[k],matchkeys,true
 		end
 	end
-	for k,key in pairs(keys2) do
-		if string.find(value,key) ~= nil then
-			return "",key,true
-		end
+	local ok,matchkeys = contain(value,keys2)
+	if ok then
+		return "",matchkeys,true
 	end
 	return "","",false
 end
 --排除
 function exclude(value)
-	for k,key in pairs(notkey1) do
-		if string.find(value,key) ~= nil then
-			return key,"",true
-		end
+	local ok,matchkeys = contain(value,notkey1)
+	if ok then
+		return matchkeys,"",true
 	end
 	for k,key in pairs(notkey2) do
-		if string.find(value,key) ~= nil then
-			for i,n in pairs(notkey2_2[k]) do
-				if string.find(value,n) ~= nil then
-					return key,n,false
-				end
+		local ok_1,matchkeys_1 = contain(value,{key})
+		if ok_1 then
+			local ok_2,matchkeys_2 = contain(value,notkey2_2[k])
+			if ok_2 then
+				return key,matchkeys_2,false
 			end
 			return key,"",true
 		end

jyinfomatch/src/luascript/dev.lua.bak

@@ -0,0 +1,105 @@
+--用户唯一标识
+appid="jyFhJXQgMAAwZfQUNKDzx6"
+--匹配词
+keys1={
+	{"政务云","不动产登记","涉密网","机要网","质量监督管理局","政法网","电子政务","警务云"},
+	{"三通两平台","云学堂","云课堂","云教室","薄改","改薄","校园网","教育云"},
+	{"卫生云","卫生信息平台","医疗卫生机构管理信息系统","医疗机构管理信息系统","工业4.0","智能制造","全民健康","中医馆","MES","HIS","HANA","高性能计算"},
+	{"交通一卡通","两网融合","智慧交通","智慧高速","智慧公交"}
+}
+keys1_1={"政府","教育","企业","公共事业"}
+keys2={"交换机","路由器","防火墙","网络安全","负载均衡","服务器","存储","小机","小型机","无线网","大数据","云计算","云平台","等保","等级保护","物联网"}
+--排除词
+notkey1={"监理","设计","施工","装修","维修","维护","维保","运维","打印","扫描","投影","数据整合","缆","土建","空调","电脑"}
+notkey2={"软件","办公","服务"}
+notkey2_2={{"数据中心","平台","网络"},{"办公网"},{"服务器","硬件集成","云平台建设","网络"}}
+--脚本主入口方法
+function filterValidate(data)
+	--匹配上的父节点，对应的子节点，是否成功
+	local y_p,y_k,y_ok = "","",false
+	--标题处理
+	local title = data["title"]
+	--print(title)
+	if title ~= nil and title ~= "" then
+		--标题匹配
+		y_p,y_k,y_ok = match(title)
+		--print("title包含--",y_p,y_k,y_ok)
+		--如果标题匹配上，进行标题排除
+		if y_ok then
+			--排除匹配上的词，对应的保留词，是否成功
+			local n_p,n_k,n_ok = exclude(title)
+			if n_p ~= "" and n_k ~= "" then
+				--print("title排除--","排除",n_p,"，保留",n_k,"，",n_ok)
+			else
+				--print("title排除--",n_p,n_k,n_ok)
+			end
+			if n_ok then
+				return nil
+			end
+		end
+	end
+	--正文处理
+	local detail = data["detail"]
+	--print(detail)
+	if detail ~= nil and detail ~= "" then
+		--如果标题没有匹配上，匹配正文
+		if not y_ok then
+			--排除匹配上的词，对应的保留词，是否成功
+			y_p,y_k,y_ok = match(detail)
+			--print("detail包含--",y_p,y_k,y_ok)
+		end
+		--匹配上标题或者正文，都要进行正文排除
+		--排除匹配上的词，对应的保留词，是否成功
+		if y_ok then
+			local n_p,n_k,n_ok = exclude(detail)
+			if n_p ~= "" and n_k ~= "" then
+				--print("detail排除--","排除",n_p,"，保留",n_k,"，",n_ok)
+			else
+				--print("detail排除--",n_p,n_k,n_ok)
+			end
+			if n_ok then
+				return nil
+			end
+		end
+	end
+	--没有匹配上
+	if not y_ok then
+		return nil
+	end
+	return data
+end
+--包含
+function match(value)
+	for k,keys in pairs(keys1) do
+		for i,key in pairs(keys) do
+			if string.find(value,key) ~= nil then
+				return keys1_1[k],key,true
+			end
+		end
+	end
+	for k,key in pairs(keys2) do
+		if string.find(value,key) ~= nil then
+			return "",key,true
+		end
+	end
+	return "","",false
+end
+--排除
+function exclude(value)
+	for k,key in pairs(notkey1) do
+		if string.find(value,key) ~= nil then
+			return key,"",true
+		end
+	end
+	for k,key in pairs(notkey2) do
+		if string.find(value,key) ~= nil then
+			for i,n in pairs(notkey2_2[k]) do
+				if string.find(value,n) ~= nil then
+					return key,n,false
+				end
+			end
+			return key,"",true
+		end
+	end
+	return "","",false
+end

jyinfomatch/src/main.go

@@ -9,6 +9,7 @@ import (
 	"qfw/util/elastic"
 	"qfw/util/mongodb"
 	"qyfw"
+	"strings"
 )
 
 //初始化
@@ -20,6 +21,7 @@ func init() {
 	qyfw.Collection = qyfw.SysConfig["collection"].(string)
 	qyfw.SaveSize = util.IntAllDef(qyfw.SysConfig["saveSize"], 200)
 	qyfw.PoolSize = util.IntAllDef(qyfw.SysConfig["poolSize"], 100)
+	qyfw.ShowFields = `"` + strings.Join(util.ObjArrToStringArr(qyfw.SysConfig["fields"].([]interface{})), `","`) + `"`
 }
 
 //主应用，定时任务

jyinfomatch/src/main_test.go

@@ -11,13 +11,14 @@ import (
 func Test_main(t *testing.T) {
 	mongodb.InitMongodbPool(1, "192.168.3.18:27080", "qyfw")
 	list := []map[string]interface{}{}
-	data := mongodb.FindById("matchresult", "5a055045cbac61d2bcb1db12", nil)
+	data := mongodb.FindById("matchresult", "5a056a3dcbac61d2bcb1e7fb", nil)
 	list = append(list, *data)
 	qyfw.IsSave = false
 	job := &qyfw.Job{}
 	job.Results = &[]map[string]interface{}{}
 	job.Name = "test"
-	luafile, err := ioutil.ReadFile("luascript/dev.lua")
+	job.EachListPool = make(chan bool, 10)
+	luafile, err := ioutil.ReadFile("luascript/test.lua")
 	if err != nil {
 		log.Println(err)
 		return

jyinfomatch/src/qyfw/handler.go

@@ -25,7 +25,7 @@ func LoadAllLuaScript() {
 			log.Println(err)
 			return err
 		}
-		sp := NewLuaScript(strings.TrimRight(info.Name(), ".lua"), string(bs))
+		sp := NewLuaScript(info.Name(), string(bs))
 		if sp != nil {
 			AllJobs[sp.Name] = sp
 			log.Println("成功加载脚本", info.Name())
@@ -39,7 +39,7 @@ func LoadAllLuaScript() {
 func runJob() {
 	util.Try(func() {
 		LoadDatasByEs(&SysConfig)
-		util.WriteSysConfig(SysConfig)
+		//util.WriteSysConfig(SysConfig)
 	}, func(e interface{}) {})
 	time.AfterFunc(time.Duration(util.IntAll(SysConfig["durationMinutes"]))*time.Minute, runJob)
 }
@@ -57,5 +57,10 @@ func NewLuaScript(name, luafile string) *Job {
 	if !script.LoadScript(name, luafile, true) {
 		return nil
 	}
+	job.Appid = script.GetVar("appid")
+	if job.Appid == "" {
+		log.Println("error：从脚本", name, "中获取到Appid为空！")
+		return nil
+	}
 	return job
 }

jyinfomatch/src/qyfw/job.go

@@ -22,7 +22,8 @@ var (
 )
 
 type Job struct {
-	Name         string                    //名称
+	Name         string                    //脚本名称
+	Appid        string                    //用户唯一标识
 	Results      *[]map[string]interface{} //最终要存库的数据
 	Lock         sync.Mutex
 	WaitGroup    sync.WaitGroup
@@ -45,16 +46,11 @@ func (j *Job) Start(list *[]map[string]interface{}) {
 			if !script.LoadScript(j.Name, j.ScriptFile, false) {
 				return
 			}
-			userId := script.GetVar("userId")
-			if userId == "" {
-				log.Println("error：从脚本", j.Name, "中获取到UserId为空！")
-				return
-			}
 			//调用lua，匹配
 			result := j.ExecJob(script, &info)
 			//保存
 			if result != nil && len(*result) > 0 && IsSave {
-				j.Save(result, userId, false)
+				j.Save(result, j.Appid, false)
 			}
 		}(v)
 		if count%200 == 0 {
@@ -104,12 +100,12 @@ func (j *Job) ExecJob(script *Script, info *map[string]interface{}) *map[string]
 }
 
 //保存到mongodb
-func (j *Job) Save(result *map[string]interface{}, userId string, flag bool) {
+func (j *Job) Save(result *map[string]interface{}, appid string, flag bool) {
 	j.Lock.Lock()
 	defer j.Lock.Unlock()
 	if result != nil {
 		(*result)["createtime"] = time.Now().Unix()
-		(*result)["userid"] = userId
+		(*result)["appid"] = appid
 		(*result)["id"] = (*result)["_id"]
 		delete(*result, "_id")
 		*j.Results = append(*j.Results, *result)

jyinfomatch/src/qyfw/loadDatas.go

@@ -11,8 +11,6 @@ import (
 )
 
 const (
-	//industry
-	ShowField   = `"_id","title","detail","projectscope","publishtime","toptype","subtype","type","area","href","projectname","winner","buyer","budget","bidamount","bidopentime","s_subscopeclass"`
 	SortQuery   = `{"publishtime":"desc"}`
 	DB          = "bidding"
 	IDRange     = `{"range":{"id":{"gt":"%s","lte":"%s"}}},{"range":{"publishtime":{"gt": %d}}}`
@@ -22,6 +20,7 @@ const (
 )
 
 var (
+	ShowFields      = ``
 	eachLuaPool     = make(chan bool, 100)
 	searchpool      = make(chan bool, 8)
 	eachpool        = make(chan bool, 100)
@@ -103,7 +102,11 @@ func initBiddingCache(_id, lastid string, lastTime int64, startTime, endTime int
 					searchWaitGroup.Done()
 					<-searchpool
 				}()
-				r := elastic.GetAllByNgram(DB, DB, c_query, "", SortQuery, ShowField, start*OnceMax, OnceMax, 0, false)
+				size := OnceMax
+				if start == totalPage-1 && count%OnceMax != 0 {
+					size = count % OnceMax
+				}
+				r := elastic.GetAllByNgram(DB, DB, c_query, "", SortQuery, ShowFields, start*OnceMax, size, 0, false)
 				res = append(res, *r...)
 				log.Println("第", start+1, "页数据加载完成！")
 			}(i)

jyinfomatch/src/qyfw/script.go

@@ -9,6 +9,7 @@ package qyfw
 import (
 	"dfa"
 	qutil "qfw/util"
+	"strings"
 	"util"
 
 	lujson "github.com/yuin/gopher-json"
@@ -37,14 +38,15 @@ func (s *Script) LoadScript(name, script_file string, isValidate bool) bool {
 	if isValidate {
 		return true
 	}
-	s.L.SetGlobal("indexOf", s.L.NewFunction(func(S *lua.LState) int {
+	s.L.SetGlobal("contain", s.L.NewFunction(func(S *lua.LState) int {
 		keywords := util.TableToArray(S.ToTable(-1))
 		text := S.ToString(-2)
 		var d *dfa.DFA = &dfa.DFA{}
 		d.AddWord(keywords...)
-		ok := len(d.Analy(text)) > 0
-		S.Push(lua.LBool(ok))
-		return 1
+		result := d.Analy(text)
+		S.Push(lua.LBool(len(result) > 0))
+		S.Push(lua.LString(strings.Join(util.ArrayDuplicate(result), ",")))
+		return 2
 	}))
 	return true
 }

jyinfomatch/src/util/util.go

@@ -6,6 +6,7 @@ import (
 	"github.com/yuin/gopher-lua"
 )
 
+//table转map
 func TableToMap(tab *lua.LTable) *map[string]interface{} {
 	tmp := make(map[string]interface{})
 	tab.ForEach(func(k, v lua.LValue) {
@@ -24,6 +25,8 @@ func TableToMap(tab *lua.LTable) *map[string]interface{} {
 	})
 	return &tmp
 }
+
+//table转数组
 func TableToArray(tab *lua.LTable) []string {
 	array := []string{}
 	tab.ForEach(func(k, v lua.LValue) {
@@ -33,3 +36,17 @@ func TableToArray(tab *lua.LTable) []string {
 	})
 	return array
 }
+
+//数组去重
+func ArrayDuplicate(array []string) []string {
+	temp := []string{}
+	m := map[string]bool{}
+	for _, v := range array {
+		if m[v] {
+			continue
+		}
+		m[v] = true
+		temp = append(temp, v)
+	}
+	return temp
+}