Merge branch 'master' of http://192.168.3.207:10080/qmx/jyqyfw

jyinfomatch/src/config.json

@@ -0,0 +1,15 @@
+{
+    "StartTime": "",
+    "lastid": "",
+    "durationMinutes": 10,
+    "mgoSize": 50,
+    "mgoAddr": "192.168.3.18:27080",
+	"mongodbName": "jyqyfw",
+	"collection": "usermail",
+    "elasticPoolSize": 10,
+    "elasticsearch": "http://192.168.3.18:9800",
+    "maxSearch": 10000,
+	"saveSize": 100,
+	"poolSize": 100,
+	"fields":["_id","title","detail","projectscope","publishtime","toptype","subtype","type","area","href","projectname","projectcode","s_winner","buyer","budget","bidamount","bidopentime","s_subscopeclass","bidstatus","agency","projectinfo"]
+}

jyinfomatch/src/dfa/interestanalysis.go

@@ -0,0 +1,142 @@
+/**
+ *兴趣分析
+ *
+ */
+package dfa
+
+import (
+	"log"
+	"strings"
+)
+
+//DFA实现
+type DFA struct {
+	link        map[string]interface{} //存放or
+	linkAnd     map[string]int         //存放and
+	linkAndWord map[string]interface{} //存放and中的拆分词
+
+}
+
+//添加词组，用于初始化,该方法是可以调用多次的
+func (d *DFA) AddWord(words ...string) {
+	if d.link == nil {
+		d.link = make(map[string]interface{})
+		d.linkAnd = make(map[string]int)
+		d.linkAndWord = make(map[string]interface{})
+	}
+	var nowMap *map[string]interface{}
+	for _, key := range words {
+		keys := strings.Split(key, "+")
+		lenkeys := len(keys)
+		if lenkeys > 1 {
+			d.linkAnd[key] = lenkeys
+			for k := 0; k < lenkeys; k++ {
+				minKey := keys[k]
+				nowMap = &d.linkAndWord
+				for i := 0; i < len(minKey); i++ {
+					kc := minKey[i : i+1]
+					if v, ok := (*nowMap)[kc]; ok {
+						nowMap, _ = v.(*map[string]interface{})
+					} else {
+						newMap := map[string]interface{}{}
+						newMap["YN"] = "N"
+						(*nowMap)[kc] = &newMap
+						nowMap = &newMap
+					}
+					if i == len(minKey)-1 {
+						(*nowMap)["YN"] = "Y"
+						if (*nowMap)["key"] == nil {
+							(*nowMap)["key"] = make(map[string]int)
+						}
+						(*nowMap)["key"].(map[string]int)[key] = k
+					}
+				}
+			}
+		} else {
+			nowMap = &d.link
+			for i := 0; i < len(key); i++ {
+				kc := key[i : i+1]
+				if v, ok := (*nowMap)[kc]; ok {
+					nowMap, _ = v.(*map[string]interface{})
+				} else {
+					newMap := map[string]interface{}{}
+					newMap["YN"] = "N"
+					(*nowMap)[kc] = &newMap
+					nowMap = &newMap
+				}
+
+				if i == len(key)-1 {
+					(*nowMap)["YN"] = "Y"
+				}
+			}
+		}
+	}
+}
+func (d *DFA) Clear() {
+	d.link = nil
+}
+
+//从给定的内容中找出匹配上的关键词
+func (d *DFA) Analy(src string) []string {
+	if d.link == nil {
+		log.Println("请先添加词组")
+		return []string{}
+	}
+	keywords := []string{}
+	tempMap := make(map[string][]bool)
+	for i := 0; i < len(src); i++ {
+		nowMap := &d.link
+		length := 0 // 匹配标识数默认为0
+		//flag := false // 敏感词结束标识位：用于敏感词只有1位的情况
+		for j := i; j < len(src); j++ {
+			word := src[j : j+1]
+			nowMap, _ = (*nowMap)[word].(*map[string]interface{})
+			if nowMap != nil {
+				length = length + 1
+				tag, _ := (*nowMap)["YN"].(string)
+				if "Y" == tag {
+					//flag = true
+					keywords = append(keywords, src[i:i+length])
+				}
+			} else {
+				break
+			}
+		}
+		nowMap = &d.linkAndWord
+		length = 0
+		for j := i; j < len(src); j++ {
+			word := src[j : j+1]
+			nowMap, _ = (*nowMap)[word].(*map[string]interface{})
+			if nowMap != nil {
+				length = length + 1
+				tag, _ := (*nowMap)["YN"].(string)
+				if "Y" == tag {
+					mkeys := (*nowMap)["key"].(map[string]int)
+					for k, v := range mkeys {
+						tempBool := tempMap[k]
+						if tempBool == nil {
+							tempBool = make([]bool, d.linkAnd[k])
+							tempMap[k] = tempBool
+						}
+						tempBool[v] = true
+					}
+				}
+			} else {
+				break
+			}
+		}
+	}
+	for k, v := range tempMap {
+		ball := true
+		for _, m := range v {
+			if !m {
+				ball = false
+				break
+			}
+		}
+		if ball {
+			keywords = append(keywords, k)
+		}
+	}
+	return keywords
+}

jyinfomatch/src/dfa/interestanalysis_test.go

@@ -0,0 +1,45 @@
+package dfa
+
+import (
+	"log"
+	"strings"
+	"testing"
+	"time"
+)
+
+var d *DFA = &DFA{}
+
+func copyMap(m map[string]int) (m2 map[string]int) {
+	m2 = make(map[string]int)
+	for k, v := range m {
+		m2[k] = v
+	}
+	return m2
+}
+
+func TestAnaly(t *testing.T) {
+	d.AddWord("办公", "办+楼", "河+省", "完+你们8")
+	log.Println(strings.Split("河+南+", "+")[2])
+	t1 := time.Now()
+	log.Println(d.Analy("这胡省锦河涛写给江泽民的信我们你们于办公楼上你完就是啊。"), "=====")
+	log.Println(time.Now().Sub(t1).Seconds())
+	d.Clear()
+	//log.Println(d.Analy("这是胡锦涛写给江泽民的信啊。"))
+
+}
+
+func Test_Label(t *testing.T) {
+	log.Println("000----")
+
+	for _, v := range []int{1, 2, 3, 4, 5} {
+		log.Println(v)
+	L1:
+		for _, vv := range []string{"a", "b", "c", "d"} {
+			log.Println(vv)
+			if vv == "add" {
+				break L1
+			}
+		}
+	}
+	log.Println("111----")
+}

jyinfomatch/src/luascript/dev.lua

@@ -0,0 +1,102 @@
+--用户唯一标识
+appid="jyMi1XQgMABQNcSkBMIhBq"
+--匹配词
+keys1={
+	{"政务云","不动产登记","涉密网","机要网","质量监督管理局","政法网","电子政务","警务云"},
+	{"三通两平台","云学堂","云课堂","云教室","薄改","改薄","校园网","教育云"},
+	{"卫生云","卫生信息平台","医疗卫生机构管理信息系统","医疗机构管理信息系统","工业4.0","智能制造","全民健康","中医馆","MES","HIS","HANA","高性能计算"},
+	{"交通一卡通","两网融合","智慧交通","智慧高速","智慧公交"}
+}
+keys1_1={"政府","教育","企业","公共事业"}
+keys2={"交换机","路由器","防火墙","网络安全","负载均衡","服务器","存储","小机","小型机","无线网","大数据","云计算","云平台","等保","等级保护","物联网"}
+--排除词
+notkey1={"监理","设计","施工","装修","维修","维护","维保","运维","打印","扫描","投影","数据整合","缆","土建","空调","电脑"}
+notkey2={"软件","办公","服务"}
+notkey2_2={{"数据中心","平台","网络"},{"办公网"},{"服务器","硬件集成","云平台建设","网络"}}
+--脚本主入口方法
+function filterValidate(data)
+	--匹配上的父节点，对应的子节点，是否成功
+	local y_p,y_k,y_ok = "","",false
+	--标题处理
+	local title = data["title"]
+	--print(title)
+	if title ~= nil and title ~= "" then
+		--标题匹配
+		y_p,y_k,y_ok = match(title)
+		--print("title包含--",y_p,y_k,y_ok)
+		--如果标题匹配上，进行标题排除
+		if y_ok then
+			--排除匹配上的词，对应的保留词，是否成功
+			local n_p,n_k,n_ok = exclude(title)
+			if n_p ~= "" and n_k ~= "" then
+				--print("title排除--","排除",n_p,"，保留",n_k,"，",n_ok)
+			else
+				--print("title排除--",n_p,n_k,n_ok)
+			end
+			if n_ok then
+				return nil
+			end
+		end
+	end
+	--正文处理
+	local detail = data["detail"]
+	--print(detail)
+	if detail ~= nil and detail ~= "" then
+		--如果标题没有匹配上，匹配正文
+		if not y_ok then
+			--排除匹配上的词，对应的保留词，是否成功
+			y_p,y_k,y_ok = match(detail)
+			--print("detail包含--",y_p,y_k,y_ok)
+		end
+		--匹配上标题或者正文，都要进行正文排除
+		--排除匹配上的词，对应的保留词，是否成功
+		if y_ok then
+			local n_p,n_k,n_ok = exclude(detail)
+			if n_p ~= "" and n_k ~= "" then
+				--print("detail排除--","排除",n_p,"，保留",n_k,"，",n_ok)
+			else
+				--print("detail排除--",n_p,n_k,n_ok)
+			end
+			if n_ok then
+				return nil
+			end
+		end
+	end
+	--没有匹配上
+	if not y_ok then
+		return nil
+	end
+	return data
+end
+--包含
+function match(value)
+	for k,keys in pairs(keys1) do
+		local ok,matchkeys = contain(value,keys)
+		if ok then
+			return keys1_1[k],matchkeys,true
+		end
+	end
+	local ok,matchkeys = contain(value,keys2)
+	if ok then
+		return "",matchkeys,true
+	end
+	return "","",false
+end
+--排除
+function exclude(value)
+	local ok,matchkeys = contain(value,notkey1)
+	if ok then
+		return matchkeys,"",true
+	end
+	for k,key in pairs(notkey2) do
+		local ok_1,matchkeys_1 = contain(value,{key})
+		if ok_1 then
+			local ok_2,matchkeys_2 = contain(value,notkey2_2[k])
+			if ok_2 then
+				return key,matchkeys_2,false
+			end
+			return key,"",true
+		end
+	end
+	return "","",false
+end

jyinfomatch/src/luascript/dev.lua.bak

@@ -0,0 +1,105 @@
+--用户唯一标识
+appid="jyFhJXQgMAAwZfQUNKDzx6"
+--匹配词
+keys1={
+	{"政务云","不动产登记","涉密网","机要网","质量监督管理局","政法网","电子政务","警务云"},
+	{"三通两平台","云学堂","云课堂","云教室","薄改","改薄","校园网","教育云"},
+	{"卫生云","卫生信息平台","医疗卫生机构管理信息系统","医疗机构管理信息系统","工业4.0","智能制造","全民健康","中医馆","MES","HIS","HANA","高性能计算"},
+	{"交通一卡通","两网融合","智慧交通","智慧高速","智慧公交"}
+}
+keys1_1={"政府","教育","企业","公共事业"}
+keys2={"交换机","路由器","防火墙","网络安全","负载均衡","服务器","存储","小机","小型机","无线网","大数据","云计算","云平台","等保","等级保护","物联网"}
+--排除词
+notkey1={"监理","设计","施工","装修","维修","维护","维保","运维","打印","扫描","投影","数据整合","缆","土建","空调","电脑"}
+notkey2={"软件","办公","服务"}
+notkey2_2={{"数据中心","平台","网络"},{"办公网"},{"服务器","硬件集成","云平台建设","网络"}}
+--脚本主入口方法
+function filterValidate(data)
+	--匹配上的父节点，对应的子节点，是否成功
+	local y_p,y_k,y_ok = "","",false
+	--标题处理
+	local title = data["title"]
+	--print(title)
+	if title ~= nil and title ~= "" then
+		--标题匹配
+		y_p,y_k,y_ok = match(title)
+		--print("title包含--",y_p,y_k,y_ok)
+		--如果标题匹配上，进行标题排除
+		if y_ok then
+			--排除匹配上的词，对应的保留词，是否成功
+			local n_p,n_k,n_ok = exclude(title)
+			if n_p ~= "" and n_k ~= "" then
+				--print("title排除--","排除",n_p,"，保留",n_k,"，",n_ok)
+			else
+				--print("title排除--",n_p,n_k,n_ok)
+			end
+			if n_ok then
+				return nil
+			end
+		end
+	end
+	--正文处理
+	local detail = data["detail"]
+	--print(detail)
+	if detail ~= nil and detail ~= "" then
+		--如果标题没有匹配上，匹配正文
+		if not y_ok then
+			--排除匹配上的词，对应的保留词，是否成功
+			y_p,y_k,y_ok = match(detail)
+			--print("detail包含--",y_p,y_k,y_ok)
+		end
+		--匹配上标题或者正文，都要进行正文排除
+		--排除匹配上的词，对应的保留词，是否成功
+		if y_ok then
+			local n_p,n_k,n_ok = exclude(detail)
+			if n_p ~= "" and n_k ~= "" then
+				--print("detail排除--","排除",n_p,"，保留",n_k,"，",n_ok)
+			else
+				--print("detail排除--",n_p,n_k,n_ok)
+			end
+			if n_ok then
+				return nil
+			end
+		end
+	end
+	--没有匹配上
+	if not y_ok then
+		return nil
+	end
+	return data
+end
+--包含
+function match(value)
+	for k,keys in pairs(keys1) do
+		for i,key in pairs(keys) do
+			if string.find(value,key) ~= nil then
+				return keys1_1[k],key,true
+			end
+		end
+	end
+	for k,key in pairs(keys2) do
+		if string.find(value,key) ~= nil then
+			return "",key,true
+		end
+	end
+	return "","",false
+end
+--排除
+function exclude(value)
+	for k,key in pairs(notkey1) do
+		if string.find(value,key) ~= nil then
+			return key,"",true
+		end
+	end
+	for k,key in pairs(notkey2) do
+		if string.find(value,key) ~= nil then
+			for i,n in pairs(notkey2_2[k]) do
+				if string.find(value,n) ~= nil then
+					return key,n,false
+				end
+			end
+			return key,"",true
+		end
+	end
+	return "","",false
+end

jyinfomatch/src/main.go

@@ -1,5 +1,30 @@
 package main
 
-func main() {
+import (
+	"log"
+	"qfw/util"
+	"qfw/util/elastic"
+	"qfw/util/mongodb"
+	"qyfw"
+	"strings"
+)
+
+//初始化
+func init() {
+	util.ReadConfig(&qyfw.SysConfig)
+	qyfw.MaxSearch = util.IntAllDef(qyfw.SysConfig["maxSearch"], 10000)
+	mongodb.InitMongodbPool(util.IntAll(qyfw.SysConfig["mgoSize"]), qyfw.SysConfig["mgoAddr"].(string), qyfw.SysConfig["mongodbName"].(string))
+	elastic.InitElasticSize(qyfw.SysConfig["elasticsearch"].(string), util.IntAllDef(qyfw.SysConfig["elasticPoolSize"], 20))
+	qyfw.Collection = qyfw.SysConfig["collection"].(string)
+	qyfw.SaveSize = util.IntAllDef(qyfw.SysConfig["saveSize"], 200)
+	qyfw.PoolSize = util.IntAllDef(qyfw.SysConfig["poolSize"], 100)
+	qyfw.ShowFields = `"` + strings.Join(util.ObjArrToStringArr(qyfw.SysConfig["fields"].([]interface{})), `","`) + `"`
+}
 
+//主应用，定时任务
+func main() {
+	go qyfw.LoadAllLuaScript()
+	log.Println("程序启动。。。")
+	flag := make(chan bool)
+	<-flag
 }

jyinfomatch/src/mainTest.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+	qutil "qfw/util"
+	"qfw/util/mongodb"
+	"qyfw"
+)
+
+func main_1() {
+	modle := flag.Int("m", 0, "模式")
+	id := flag.String("id", "", "信息id")
+	collection := flag.String("c", "", "表名")
+	flag.Parse()
+	if *id == "" {
+		log.Println("请输入id")
+		os.Exit(0)
+		return
+	}
+	if *collection == "" {
+		log.Println("请输入表名")
+		os.Exit(0)
+		return
+	}
+	if *modle == 1 {
+		*id = qutil.CommonDecodeArticle("content", *id)[0]
+		log.Println("解密后id是", *id)
+	}
+	mongodb.InitMongodbPool(1, "192.168.3.14:27080", "jyqyfw")
+	list := []map[string]interface{}{}
+	log.Println(*id, *collection)
+	data := mongodb.FindOne(*collection, map[string]interface{}{
+		"id": *id,
+	})
+	if data == nil || len(*data) == 0 {
+		log.Println("没有找到数据！")
+		return
+	}
+	list = append(list, *data)
+	qyfw.IsSave = false
+	job := &qyfw.Job{}
+	job.Results = &[]map[string]interface{}{}
+	job.Name = "dev.lua"
+	job.EachListPool = make(chan bool, 10)
+	luafile, err := ioutil.ReadFile("luascript/dev.lua")
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	job.ScriptFile = string(luafile)
+	job.Start(&list)
+}

jyinfomatch/src/qyfw/config.go

@@ -0,0 +1,7 @@
+//
+package qyfw
+
+//配置
+var (
+	SysConfig map[string]interface{}
+)

jyinfomatch/src/qyfw/handler.go

@@ -0,0 +1,66 @@
+package qyfw
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"qfw/util"
+	"strings"
+	"time"
+)
+
+var (
+	AllJobs = map[string]*Job{}
+)
+
+//加载全部脚本
+func LoadAllLuaScript() {
+	filepath.Walk("./luascript", func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() || !strings.HasSuffix(info.Name(), ".lua") {
+			return nil
+		}
+		bs, err := ioutil.ReadFile(path)
+		if err != nil {
+			log.Println(err)
+			return err
+		}
+		sp := NewLuaScript(info.Name(), string(bs))
+		if sp != nil {
+			AllJobs[sp.Name] = sp
+			log.Println("成功加载脚本", info.Name())
+		}
+		return nil
+	})
+	go runJob()
+}
+
+//开始定时任务
+func runJob() {
+	util.Try(func() {
+		LoadDatasByEs(&SysConfig)
+		util.WriteSysConfig(SysConfig)
+	}, func(e interface{}) {})
+	time.AfterFunc(time.Duration(util.IntAll(SysConfig["durationMinutes"]))*time.Minute, runJob)
+}
+
+//初始化脚本
+func NewLuaScript(name, luafile string) *Job {
+	defer util.Catch()
+	job := &Job{}
+	job.Results = &[]map[string]interface{}{}
+	job.Name = name
+	job.ScriptFile = luafile
+	job.EachListPool = make(chan bool, PoolSize)
+	//只是单纯的验证一下，lua脚本是否有问题
+	script := &Script{}
+	if !script.LoadScript(name, luafile, true) {
+		return nil
+	}
+	job.Appid = script.GetVar("appid")
+	if job.Appid == "" {
+		log.Println("error：从脚本", name, "中获取到Appid为空！")
+		return nil
+	}
+	return job
+}

jyinfomatch/src/qyfw/job.go

@@ -0,0 +1,126 @@
+/**
+爬虫，脚本接口，需要扩展
+*/
+package qyfw
+
+import (
+	"log"
+	qutil "qfw/util"
+	"qfw/util/mongodb"
+	"sync"
+	"time"
+	"util"
+
+	"github.com/yuin/gopher-lua"
+)
+
+var (
+	Collection = "usermail"
+	SaveSize   = 200
+	IsSave     = true
+	PoolSize   = 100
+)
+
+type Job struct {
+	Name         string                    //脚本名称
+	Appid        string                    //用户唯一标识
+	Results      *[]map[string]interface{} //最终要存库的数据
+	Lock         sync.Mutex
+	WaitGroup    sync.WaitGroup
+	ScriptFile   string
+	EachListPool chan bool
+}
+
+//任务
+func (j *Job) Start(list *[]map[string]interface{}) {
+	count := 0
+	for _, v := range *list {
+		j.EachListPool <- true
+		j.WaitGroup.Add(1)
+		go func(info map[string]interface{}) {
+			defer func() {
+				<-j.EachListPool
+				j.WaitGroup.Done()
+			}()
+			script := &Script{}
+			if !script.LoadScript(j.Name, j.ScriptFile, false) {
+				return
+			}
+			//调用lua，匹配
+			result := j.ExecJob(script, &info)
+			//保存
+			if result != nil && len(*result) > 0 && IsSave {
+				j.Save(result, j.Appid, false)
+			}
+		}(v)
+		if count%200 == 0 {
+			log.Println(j.Name, "index：", count)
+		}
+		count++
+	}
+	j.WaitGroup.Wait()
+	j.Save(nil, j.Appid, true)
+	log.Println("脚本", j.Name, "执行完毕！")
+}
+
+func (j *Job) ExecJob(script *Script, info *map[string]interface{}) *map[string]interface{} {
+	defer qutil.Catch()
+	tab := script.L.NewTable()
+	for k, v := range *info {
+		if val, ok := v.(string); ok {
+			tab.RawSet(lua.LString(k), lua.LString(val))
+		} else if val, ok := v.(int64); ok {
+			tab.RawSet(lua.LString(k), lua.LNumber(val))
+		} else if val, ok := v.(int32); ok {
+			tab.RawSet(lua.LString(k), lua.LNumber(val))
+		} else if val, ok := v.(float64); ok {
+			tab.RawSet(lua.LString(k), lua.LNumber(val))
+		} else if val, ok := v.(float32); ok {
+			tab.RawSet(lua.LString(k), lua.LNumber(val))
+		} else if val, ok := v.(bool); ok {
+			tab.RawSet(lua.LString(k), lua.LBool(val))
+		}
+	}
+	err := script.L.CallByParam(lua.P{
+		Fn:      script.L.GetGlobal("filterValidate"),
+		NRet:    1,
+		Protect: true,
+	}, tab)
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	lv := script.L.Get(-1)
+	script.L.Pop(1)
+	if tbl, ok := lv.(*lua.LTable); ok {
+		return util.TableToMap(tbl)
+	} else {
+		return nil
+	}
+}
+
+//保存到mongodb
+func (j *Job) Save(result *map[string]interface{}, appid string, flag bool) {
+	j.Lock.Lock()
+	defer j.Lock.Unlock()
+	if result != nil {
+		(*result)["createtime"] = time.Now().Unix()
+		(*result)["appid"] = appid
+		(*result)["id"] = (*result)["_id"]
+		delete(*result, "_id")
+		*j.Results = append(*j.Results, *result)
+	}
+	length := len(*j.Results)
+	if length == 0 {
+		return
+	}
+	if length >= SaveSize || flag {
+		thisSaveSize := SaveSize
+		if flag {
+			thisSaveSize = len(*j.Results)
+		}
+		log.Println(appid, "save", thisSaveSize)
+		mongodb.SaveBulk(Collection, *j.Results...)
+		j.Results = &[]map[string]interface{}{}
+	}
+}

jyinfomatch/src/qyfw/loadDatas.go

@@ -0,0 +1,136 @@
+package qyfw
+
+import (
+	"fmt"
+	"log"
+	"qfw/util"
+	"qfw/util/elastic"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	SortQuery   = `{"publishtime":"desc"}`
+	DB          = "bidding"
+	IDRange     = `{"range":{"id":{"gt":"%s","lte":"%s"}}},{"range":{"publishtime":{"gt": %d}}}`
+	TimeRange   = `{"range":{"comeintime":{"gte":%d,"lte":%d}}}`
+	MaxId       = `{"query":{"filtered":{"filter":{"bool":{"must":{"range":{"id":{"gt":"%s"}}}}}}},"_source":["_id","comeintime"],"sort":{"id":"desc"},"from":0,"size":1}`
+	FilterQuery = `{"query":{"filtered":{"filter":{"bool":{"must":[%s]}}}}}`
+)
+
+var (
+	ShowFields      = ``
+	eachLuaPool     = make(chan bool, 100)
+	searchpool      = make(chan bool, 8)
+	eachpool        = make(chan bool, 100)
+	searchWaitGroup = &sync.WaitGroup{}
+	MaxSearch       = 10000 //缓存中总共加载这么多条
+)
+
+func LoadDatasByEs(Config *map[string]interface{}) bool {
+	defer util.Catch()
+	st, _ := time.ParseInLocation(util.Date_Full_Layout, (*Config)["StartTime"].(string), time.Local)
+	lastTime := st.Unix()
+	_id := util.ObjToString((*Config)["lastid"])
+	log.Println("开始执行定时任务-id-lasttime:", _id, lastTime)
+	//获取本次查询的最大id
+	idQuery := ""
+	if _id == "" {
+		idQuery = strings.Replace(fmt.Sprintf(MaxId, _id), `"gt"`, `"gte"`, -1)
+	} else {
+		idQuery = fmt.Sprintf(MaxId, _id)
+	}
+	resId := elastic.Get(DB, DB, idQuery)
+	lastid := ""
+	var comeintime interface{}
+	if resId != nil && *resId != nil && len(*resId) == 1 {
+		lastid = util.ObjToString((*resId)[0]["_id"])
+		comeintime = (*resId)[0]["comeintime"]
+	} else {
+		log.Println("获取本次查询的最大id的时候，未查找到数据...", idQuery)
+		return false
+	}
+	ok, list := initBiddingCache(_id, lastid, lastTime, 0, 0)
+	if !ok {
+		log.Println("加载数据到内存中的时候，未查找到数据...")
+		return false
+	}
+	//
+	go runLuaScript(&list)
+	log.Println("定时任务结束-comeintime-lastid", comeintime, lastid)
+	(*Config)["StartTime"] = util.FormatDateWithObj(&comeintime, util.Date_Full_Layout)
+	(*Config)["lastid"] = lastid
+	return true
+}
+
+//加载数据到内存中
+func initBiddingCache(_id, lastid string, lastTime int64, startTime, endTime int64) (bool, []map[string]interface{}) {
+	c_query := fmt.Sprintf(FilterQuery, fmt.Sprintf(IDRange, _id, lastid, lastTime-7*86400))
+	log.Println("es query：", c_query)
+	//testquery := `{"terms":{"_id":["596f21935d11e1c7455ddc7f"]}}`
+	//c_query = fmt.Sprintf(FilterQuery, testquery)
+	count := int(elastic.Count(DB, DB, c_query))
+	log.Println("本次共查到数据", count, "条")
+	var res []map[string]interface{}
+	if count == 0 {
+		return false, res
+	}
+	if count > MaxSearch {
+		count = MaxSearch
+		log.Println("目前数据多于", MaxSearch, "，只加载了", MaxSearch, "条！")
+	}
+	onceSize := 400 //ES一次查询这么多条
+	if onceSize > count {
+		onceSize = count
+	}
+	totalPage := int((count + onceSize - 1) / onceSize)
+	log.Println("数据一共", totalPage, "页！")
+	//如果res长度和cout相差5条，重试
+	for t := 1; t <= 3; t++ {
+		res = []map[string]interface{}{}
+		for i := 0; i < totalPage; i++ {
+			searchpool <- true
+			searchWaitGroup.Add(1)
+			go func(start int) {
+				defer func() {
+					searchWaitGroup.Done()
+					<-searchpool
+				}()
+				size := onceSize
+				if start == totalPage-1 && count%onceSize != 0 {
+					size = count % onceSize
+				}
+				r := elastic.GetAllByNgram(DB, DB, c_query, "", SortQuery, ShowFields, start*onceSize, size, 0, false)
+				res = append(res, *r...)
+				log.Println("第", start+1, "页数据加载完成！")
+			}(i)
+		}
+		searchWaitGroup.Wait()
+		if len(res) >= count-5 {
+			break
+		}
+		log.Println("第", t, "次加载数据完成，数据总数", len(res), "，由于数据量不够，重新加载！")
+	}
+	resLenght := len(res)
+	if resLenght == 0 {
+		return false, res
+	}
+	log.Println(resLenght, "条数据已经加载完成！")
+	return true, res
+}
+
+//数据加载完，执行脚本
+func runLuaScript(list *[]map[string]interface{}) bool {
+	for _, j := range AllJobs {
+		eachLuaPool <- true
+		go func(job *Job) {
+			defer func() {
+				<-eachLuaPool
+			}()
+			log.Println("执行脚本", job.Name)
+			job.Start(list)
+		}(j)
+	}
+	return true
+}

jyinfomatch/src/qyfw/script.go

@@ -0,0 +1,57 @@
+/**
+脚本加载+调用 封装，
+前期走文件系统加载
+后期走数据库配置，
+LUA中公共的方法需要抽出来，主脚本文件加载LUA公共文件
+*/
+package qyfw
+
+import (
+	"dfa"
+	qutil "qfw/util"
+	"strings"
+	"util"
+
+	lujson "github.com/yuin/gopher-json"
+	"github.com/yuin/gopher-lua"
+)
+
+//脚本
+type Script struct {
+	L *lua.LState
+}
+
+//加载文件
+func (s *Script) LoadScript(name, script_file string, isValidate bool) bool {
+	defer qutil.Catch()
+	options := lua.Options{
+		RegistrySize:        256 * 20,
+		CallStackSize:       256,
+		IncludeGoStackTrace: false,
+	}
+	s.L = lua.NewState(options)
+	s.L.PreloadModule("json", lujson.Loader)
+	if err := s.L.DoString(script_file); err != nil {
+		panic("加载" + name + "脚本错误：" + err.Error())
+		return false
+	}
+	if isValidate {
+		return true
+	}
+	s.L.SetGlobal("contain", s.L.NewFunction(func(S *lua.LState) int {
+		keywords := util.TableToArray(S.ToTable(-1))
+		text := S.ToString(-2)
+		var d *dfa.DFA = &dfa.DFA{}
+		d.AddWord(keywords...)
+		result := d.Analy(text)
+		S.Push(lua.LBool(len(result) > 0))
+		S.Push(lua.LString(strings.Join(util.ArrayDuplicate(result), ",")))
+		return 2
+	}))
+	return true
+}
+
+//取得变量
+func (s *Script) GetVar(key string) string {
+	return s.L.GetGlobal(key).String()
+}

jyinfomatch/src/util/util.go

@@ -0,0 +1,52 @@
+package util
+
+import (
+	"fmt"
+
+	"github.com/yuin/gopher-lua"
+)
+
+//table转map
+func TableToMap(tab *lua.LTable) *map[string]interface{} {
+	tmp := make(map[string]interface{})
+	tab.ForEach(func(k, v lua.LValue) {
+		key := fmt.Sprint(k)
+		if val, ok := v.(*lua.LTable); ok {
+			tmp[key] = TableToMap(val)
+		} else {
+			if val, ok := v.(lua.LString); ok {
+				tmp[key] = string(val)
+			} else if val, ok := v.(lua.LNumber); ok {
+				tmp[key] = int64(val)
+			} else {
+				tmp[key] = fmt.Sprint(v)
+			}
+		}
+	})
+	return &tmp
+}
+
+//table转数组
+func TableToArray(tab *lua.LTable) []string {
+	array := []string{}
+	tab.ForEach(func(k, v lua.LValue) {
+		if value, ok := v.(lua.LString); ok {
+			array = append(array, string(value))
+		}
+	})
+	return array
+}
+
+//数组去重
+func ArrayDuplicate(array []string) []string {
+	temp := []string{}
+	m := map[string]bool{}
+	for _, v := range array {
+		if m[v] {
+			continue
+		}
+		m[v] = true
+		temp = append(temp, v)
+	}
+	return temp
+}