xiugai

jyinfomatch/src/config.json

@@ -9,6 +9,6 @@
     "elasticPoolSize": 10,
     "elasticsearch": "http://192.168.3.18:9800",
     "maxSearch": 500,
-	"saveSize": 200,
+	"saveSize": 100,
 	"poolSize": 100
 }

jyinfomatch/src/dfa/interestanalysis.go

@@ -0,0 +1,142 @@
+/**
+ *兴趣分析
+ *
+ */
+package dfa
+
+import (
+	"log"
+	"strings"
+)
+
+//DFA实现
+type DFA struct {
+	link        map[string]interface{} //存放or
+	linkAnd     map[string]int         //存放and
+	linkAndWord map[string]interface{} //存放and中的拆分词
+
+}
+
+//添加词组，用于初始化,该方法是可以调用多次的
+func (d *DFA) AddWord(words ...string) {
+	if d.link == nil {
+		d.link = make(map[string]interface{})
+		d.linkAnd = make(map[string]int)
+		d.linkAndWord = make(map[string]interface{})
+	}
+	var nowMap *map[string]interface{}
+	for _, key := range words {
+		keys := strings.Split(key, "+")
+		lenkeys := len(keys)
+		if lenkeys > 1 {
+			d.linkAnd[key] = lenkeys
+			for k := 0; k < lenkeys; k++ {
+				minKey := keys[k]
+				nowMap = &d.linkAndWord
+				for i := 0; i < len(minKey); i++ {
+					kc := minKey[i : i+1]
+					if v, ok := (*nowMap)[kc]; ok {
+						nowMap, _ = v.(*map[string]interface{})
+					} else {
+						newMap := map[string]interface{}{}
+						newMap["YN"] = "N"
+						(*nowMap)[kc] = &newMap
+						nowMap = &newMap
+					}
+					if i == len(minKey)-1 {
+						(*nowMap)["YN"] = "Y"
+						if (*nowMap)["key"] == nil {
+							(*nowMap)["key"] = make(map[string]int)
+						}
+						(*nowMap)["key"].(map[string]int)[key] = k
+					}
+				}
+			}
+		} else {
+			nowMap = &d.link
+			for i := 0; i < len(key); i++ {
+				kc := key[i : i+1]
+				if v, ok := (*nowMap)[kc]; ok {
+					nowMap, _ = v.(*map[string]interface{})
+				} else {
+					newMap := map[string]interface{}{}
+					newMap["YN"] = "N"
+					(*nowMap)[kc] = &newMap
+					nowMap = &newMap
+				}
+
+				if i == len(key)-1 {
+					(*nowMap)["YN"] = "Y"
+				}
+			}
+		}
+	}
+}
+func (d *DFA) Clear() {
+	d.link = nil
+}
+
+//从给定的内容中找出匹配上的关键词
+func (d *DFA) Analy(src string) []string {
+	if d.link == nil {
+		log.Println("请先添加词组")
+		return []string{}
+	}
+	keywords := []string{}
+	tempMap := make(map[string][]bool)
+	for i := 0; i < len(src); i++ {
+		nowMap := &d.link
+		length := 0 // 匹配标识数默认为0
+		//flag := false // 敏感词结束标识位：用于敏感词只有1位的情况
+		for j := i; j < len(src); j++ {
+			word := src[j : j+1]
+			nowMap, _ = (*nowMap)[word].(*map[string]interface{})
+			if nowMap != nil {
+				length = length + 1
+				tag, _ := (*nowMap)["YN"].(string)
+				if "Y" == tag {
+					//flag = true
+					keywords = append(keywords, src[i:i+length])
+				}
+			} else {
+				break
+			}
+		}
+		nowMap = &d.linkAndWord
+		length = 0
+		for j := i; j < len(src); j++ {
+			word := src[j : j+1]
+			nowMap, _ = (*nowMap)[word].(*map[string]interface{})
+			if nowMap != nil {
+				length = length + 1
+				tag, _ := (*nowMap)["YN"].(string)
+				if "Y" == tag {
+					mkeys := (*nowMap)["key"].(map[string]int)
+					for k, v := range mkeys {
+						tempBool := tempMap[k]
+						if tempBool == nil {
+							tempBool = make([]bool, d.linkAnd[k])
+							tempMap[k] = tempBool
+						}
+						tempBool[v] = true
+					}
+				}
+			} else {
+				break
+			}
+		}
+	}
+	for k, v := range tempMap {
+		ball := true
+		for _, m := range v {
+			if !m {
+				ball = false
+				break
+			}
+		}
+		if ball {
+			keywords = append(keywords, k)
+		}
+	}
+	return keywords
+}

jyinfomatch/src/dfa/interestanalysis_test.go

@@ -0,0 +1,45 @@
+package dfa
+
+import (
+	"log"
+	"strings"
+	"testing"
+	"time"
+)
+
+var d *DFA = &DFA{}
+
+func copyMap(m map[string]int) (m2 map[string]int) {
+	m2 = make(map[string]int)
+	for k, v := range m {
+		m2[k] = v
+	}
+	return m2
+}
+
+func TestAnaly(t *testing.T) {
+	d.AddWord("办公", "办+楼", "河+省", "完+你们8")
+	log.Println(strings.Split("河+南+", "+")[2])
+	t1 := time.Now()
+	log.Println(d.Analy("这胡省锦河涛写给江泽民的信我们你们于办公楼上你完就是啊。"), "=====")
+	log.Println(time.Now().Sub(t1).Seconds())
+	d.Clear()
+	//log.Println(d.Analy("这是胡锦涛写给江泽民的信啊。"))
+
+}
+
+func Test_Label(t *testing.T) {
+	log.Println("000----")
+
+	for _, v := range []int{1, 2, 3, 4, 5} {
+		log.Println(v)
+	L1:
+		for _, vv := range []string{"a", "b", "c", "d"} {
+			log.Println(vv)
+			if vv == "add" {
+				break L1
+			}
+		}
+	}
+	log.Println("111----")
+}

jyinfomatch/src/qyfw/handler.go

@@ -7,39 +7,31 @@ import (
 	"path/filepath"
 	"qfw/util"
 	"strings"
-	"sync"
 	"time"
 )
 
 var (
-	AllJobs           = map[string]*Job{}
-	LoadLuaScripePool = make(chan bool, 1)
-	LoadLuaScripeWait = &sync.WaitGroup{}
+	AllJobs = map[string]*Job{}
 )
 
 //加载全部脚本
 func LoadAllLuaScript() {
 	filepath.Walk("./luascript", func(path string, info os.FileInfo, err error) error {
-		LoadLuaScripePool <- true
-		LoadLuaScripeWait.Add(1)
-		go func() {
-			defer func() {
-				<-LoadLuaScripePool
-				LoadLuaScripeWait.Done()
-			}()
-			if info.IsDir() || !strings.HasSuffix(info.Name(), ".lua") {
-				return
-			}
-			bs, _ := ioutil.ReadFile(path)
-			sp := NewLuaScript(strings.TrimRight(info.Name(), ".lua"), string(bs))
-			if sp != nil {
-				AllJobs[sp.Name] = sp
-				log.Println("成功加载脚本", info.Name())
-			}
-		}()
+		if info.IsDir() || !strings.HasSuffix(info.Name(), ".lua") {
+			return nil
+		}
+		bs, err := ioutil.ReadFile(path)
+		if err != nil {
+			log.Println(err)
+			return err
+		}
+		sp := NewLuaScript(strings.TrimRight(info.Name(), ".lua"), string(bs))
+		if sp != nil {
+			AllJobs[sp.Name] = sp
+			log.Println("成功加载脚本", info.Name())
+		}
 		return nil
 	})
-	LoadLuaScripeWait.Wait()
 	go runJob()
 }
 

jyinfomatch/src/qyfw/job.go

@@ -8,6 +8,7 @@ import (
 	qutil "qfw/util"
 	"qfw/util/mongodb"
 	"sync"
+	"time"
 	"util"
 
 	"github.com/yuin/gopher-lua"
@@ -107,6 +108,7 @@ func (j *Job) Save(result *map[string]interface{}, userId string, flag bool) {
 	j.Lock.Lock()
 	defer j.Lock.Unlock()
 	if result != nil {
+		(*result)["createtime"] = time.Now().Unix()
 		(*result)["userid"] = userId
 		(*result)["id"] = (*result)["_id"]
 		delete(*result, "_id")

jyinfomatch/src/qyfw/script.go

@@ -7,7 +7,9 @@ LUA中公共的方法需要抽出来，主脚本文件加载LUA公共文件
 package qyfw
 
 import (
-	"qfw/util"
+	"dfa"
+	qutil "qfw/util"
+	"util"
 
 	lujson "github.com/yuin/gopher-json"
 	"github.com/yuin/gopher-lua"
@@ -20,7 +22,7 @@ type Script struct {
 
 //加载文件
 func (s *Script) LoadScript(name, script_file string, isValidate bool) bool {
-	defer util.Catch()
+	defer qutil.Catch()
 	options := lua.Options{
 		RegistrySize:        256 * 20,
 		CallStackSize:       256,
@@ -35,6 +37,15 @@ func (s *Script) LoadScript(name, script_file string, isValidate bool) bool {
 	if isValidate {
 		return true
 	}
+	s.L.SetGlobal("indexOf", s.L.NewFunction(func(S *lua.LState) int {
+		keywords := util.TableToArray(S.ToTable(-1))
+		text := S.ToString(-2)
+		var d *dfa.DFA = &dfa.DFA{}
+		d.AddWord(keywords...)
+		ok := len(d.Analy(text)) > 0
+		S.Push(lua.LBool(ok))
+		return 1
+	}))
 	return true
 }
 

jyinfomatch/src/util/util.go

@@ -24,3 +24,12 @@ func TableToMap(tab *lua.LTable) *map[string]interface{} {
 	})
 	return &tmp
 }
+func TableToArray(tab *lua.LTable) []string {
+	array := []string{}
+	tab.ForEach(func(k, v lua.LValue) {
+		if value, ok := v.(lua.LString); ok {
+			array = append(array, string(value))
+		}
+	})
+	return array
+}