xiugai

jyinfomatch/src/config.json

@@ -9,7 +9,7 @@
     "elasticPoolSize": 10,
     "elasticsearch": "http://192.168.3.14:9800",
 	"redisServers":"filter=192.168.3.14:2379",
-    "maxSearch": 10000,
+    "maxSearch": 20000,
 	"saveSize": 100,
 	"poolSize": 100,
 	"fields":["_id","title","detail","projectscope","publishtime","toptype","subtype","type","area","href","projectname","projectcode","s_winner","buyer","budget","bidamount","bidopentime","s_subscopeclass","bidstatus","agency","projectinfo","buyerperson","buyertel","city"]

jyinfomatch/src/luascript/jyMi1XQgMABQNcSkBMIhBq_2.lua

@@ -3,7 +3,7 @@ appid="jyMi1XQgMABQNcSkBMIhBq"
 --打分
 matchscore=5
 --标签
-department="软件"
+department="硬件"
 --匹配词
 title_keys={"政务内网","政务外网","党务网","政务网","机房改造","安全审计","卫生信息化","医疗信息平台","教育资源公共服务平台","智慧城市","智慧医疗","智慧教育"}
 --脚本主入口方法

jyinfomatch/src/mainTest.go

@@ -49,5 +49,5 @@ func main_d() {
 		return
 	}
 	job.ScriptFile = string(luafile)
-	job.Start(&list)
+	job.Start(nil, &list)
 }

jyinfomatch/src/qyfw/handler.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	AllJobs = map[string]*Job{}
+	AllJobs = map[string]*MyJobs{}
 )
 
 //加载全部脚本
@@ -27,7 +27,16 @@ func LoadAllLuaScript() {
 		}
 		sp := NewLuaScript(info.Name(), string(bs))
 		if sp != nil {
-			AllJobs[sp.Name] = sp
+			myJobs := AllJobs[sp.Appid]
+			if myJobs == nil {
+				myJobs = &MyJobs{
+					Appid:         sp.Appid,
+					Jobs:          []*Job{},
+					RedisFiveData: map[string]*[]*map[string]interface{}{},
+				}
+				AllJobs[sp.Appid] = myJobs
+			}
+			myJobs.Jobs = append(myJobs.Jobs, sp)
 			log.Println("成功加载脚本", info.Name())
 		}
 		return nil
@@ -62,8 +71,7 @@ func NewLuaScript(name, luafile string) *Job {
 	job.Results = &[]map[string]interface{}{}
 	job.ScriptFile = luafile
 	job.EachListPool = make(chan bool, PoolSize)
-	job.MatchScore = script.GetFloatVar("matchscore")
+	job.MatchScore = script.GetIntVar("matchscore")
 	job.Department = script.GetStringVar("department")
-	job.RedisFiveData = map[string]*[]*map[string]interface{}{}
 	return job
 }

jyinfomatch/src/qyfw/job.go

@@ -26,22 +26,50 @@ var (
 	PoolSize   = 100
 )
 
-type Job struct {
-	Name          string                    //脚本名称
-	Appid         string                    //用户唯一标识
-	MatchScore    float64                   //打分
-	Department    string                    //标签
-	Results       *[]map[string]interface{} //最终要存库的数据
+type MyJobs struct {
+	Appid         string
 	Lock          sync.Mutex
 	WaitGroup     sync.WaitGroup
-	ScriptFile    string
-	EachListPool  chan bool
+	Jobs          []*Job
 	RedisFiveData map[string]*[]*map[string]interface{}
-	FilterCount   int
+}
+
+//更新redis
+func (j *MyJobs) UpdateRedis() {
+	log.Println(j.Appid, "更新redis。。。")
+	if len(j.RedisFiveData) == 0 {
+		return
+	}
+	now := time.Now()
+	for k, v := range j.RedisFiveData {
+		if v == nil || len(*v) == 0 {
+			continue
+		}
+		createtime, _ := time.ParseInLocation(qutil.Date_yyyyMMdd, strings.Split(k, "_")[3], time.Local)
+		timeout := createtime.AddDate(0, 0, 5).Sub(now).Seconds()
+		if timeout > 0 {
+			redis.Put("filter", k, v, qutil.IntAll(timeout))
+		}
+	}
+	j.RedisFiveData = map[string]*[]*map[string]interface{}{}
+	log.Println(j.Appid, "更新redis over!")
+}
+
+type Job struct {
+	Name         string                    //脚本名称
+	Appid        string                    //用户唯一标识
+	MatchScore   int                       //打分
+	Department   string                    //标签
+	Results      *[]map[string]interface{} //最终要存库的数据
+	Lock         sync.Mutex
+	WaitGroup    sync.WaitGroup
+	ScriptFile   string
+	EachListPool chan bool
+	FilterCount  int
 }
 
 //任务
-func (j *Job) Start(list *[]map[string]interface{}) {
+func (j *Job) Start(myJobs *MyJobs, list *[]map[string]interface{}) {
 	defer qutil.Catch()
 	count := 0
 	for _, v := range *list {
@@ -60,7 +88,7 @@ func (j *Job) Start(list *[]map[string]interface{}) {
 			result := j.ExecJob(script, &info)
 			//保存
 			if result != nil && len(*result) > 0 && IsSave {
-				j.Save(result, false)
+				j.Save(myJobs, result, false)
 			}
 		}(v)
 		if count%200 == 0 {
@@ -69,8 +97,7 @@ func (j *Job) Start(list *[]map[string]interface{}) {
 		count++
 	}
 	j.WaitGroup.Wait()
-	j.Save(nil, true)
-	j.UpdateRedis()
+	j.Save(myJobs, nil, true)
 	log.Println("脚本", j.Name, "执行完毕！")
 }
 
@@ -111,7 +138,7 @@ func (j *Job) ExecJob(script *Script, info *map[string]interface{}) *map[string]
 }
 
 //保存到mongodb
-func (j *Job) Save(result *map[string]interface{}, flag bool) {
+func (j *Job) Save(myJobs *MyJobs, result *map[string]interface{}, flag bool) {
 	j.Lock.Lock()
 	defer j.Lock.Unlock()
 	if result != nil {
@@ -120,8 +147,9 @@ func (j *Job) Save(result *map[string]interface{}, flag bool) {
 		(*result)["appid"] = j.Appid
 		(*result)["department"] = j.Department
 		(*result)["matchscore"] = j.MatchScore
+		(*result)["remark"] = ""
 		delete(*result, "_id")
-		if j.Filter(*result) {
+		if j.Filter(myJobs, *result) {
 			j.FilterCount++
 		} else {
 			*j.Results = append(*j.Results, *result)
@@ -144,7 +172,9 @@ func (j *Job) Save(result *map[string]interface{}, flag bool) {
 }
 
 //过滤
-func (j *Job) Filter(result map[string]interface{}) bool {
+func (j *Job) Filter(myJobs *MyJobs, result map[string]interface{}) bool {
+	myJobs.Lock.Lock()
+	defer myJobs.Lock.Unlock()
 	area := qutil.ObjToString(result["area"])
 	if area == "A" {
 		area = "全国"
@@ -169,10 +199,10 @@ L:
 			keys = append(keys, fmt.Sprintf("%s_%s_%s_%s_%s", j.Appid, j.Department, toptype, v, "全国"))
 		}
 		for _, key := range keys {
-			fiveData := j.RedisFiveData[key]
+			fiveData := myJobs.RedisFiveData[key]
 			if fiveData == nil {
 				fiveData = &[]*map[string]interface{}{}
-				j.RedisFiveData[key] = fiveData
+				myJobs.RedisFiveData[key] = fiveData
 				redisDatas, _ := redis.Get("filter", key).([]interface{})
 				for _, rsd := range redisDatas {
 					var rddm map[string]interface{}
@@ -245,31 +275,12 @@ L:
 			"city":        city,
 			"publishtime": publishtime,
 		}
-		array := j.RedisFiveData[redisKey]
+		array := myJobs.RedisFiveData[redisKey]
 		if array == nil {
 			array = &[]*map[string]interface{}{}
-			j.RedisFiveData[redisKey] = array
+			myJobs.RedisFiveData[redisKey] = array
 		}
 		*array = append(*array, &data)
 	}
 	return false
 }
-
-//更新redis
-func (j *Job) UpdateRedis() {
-	if j.RedisFiveData == nil || len(j.RedisFiveData) == 0 {
-		return
-	}
-	now := time.Now()
-	for k, v := range j.RedisFiveData {
-		if v == nil || len(*v) == 0 {
-			continue
-		}
-		createtime, _ := time.ParseInLocation(qutil.Date_yyyyMMdd, strings.Split(k, "_")[3], time.Local)
-		timeout := createtime.AddDate(0, 0, 5).Sub(now).Seconds()
-		if timeout > 0 {
-			redis.Put("filter", k, v, qutil.IntAll(timeout))
-		}
-	}
-	j.RedisFiveData = map[string]*[]*map[string]interface{}{}
-}

jyinfomatch/src/qyfw/loadDatas.go

@@ -21,7 +21,8 @@ const (
 
 var (
 	ShowFields      = ``
-	eachLuaPool     = make(chan bool, 100)
+	eachAllLuaPool  = make(chan bool, 50)
+	eachMyLuaPool   = make(chan bool, 10)
 	searchpool      = make(chan bool, 8)
 	eachpool        = make(chan bool, 100)
 	searchWaitGroup = &sync.WaitGroup{}
@@ -123,15 +124,27 @@ func initBiddingCache(_id, lastid string, lastTime int64, startTime, endTime int
 
 //数据加载完，执行脚本
 func runLuaScript(list *[]map[string]interface{}) bool {
-	for _, j := range AllJobs {
-		eachLuaPool <- true
-		go func(job *Job) {
+	for _, my := range AllJobs {
+		eachAllLuaPool <- true
+		go func(myJobs *MyJobs) {
 			defer func() {
-				<-eachLuaPool
+				<-eachAllLuaPool
 			}()
-			log.Println("执行脚本", job.Name)
-			job.Start(list)
-		}(j)
+			for _, j := range myJobs.Jobs {
+				eachMyLuaPool <- true
+				myJobs.WaitGroup.Add(1)
+				go func(job *Job) {
+					defer func() {
+						<-eachMyLuaPool
+						myJobs.WaitGroup.Done()
+					}()
+					log.Println("执行脚本", job.Name)
+					job.Start(myJobs, list)
+				}(j)
+			}
+			myJobs.WaitGroup.Wait()
+			myJobs.UpdateRedis()
+		}(my)
 	}
 	return true
 }