清理lua测试功能

src/jy/admin/rule.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	. "jy/mongodbutil"
 	"jy/util"
+	"log"
 	qu "qfw/util"
 	"time"
 

src/jy/admin/rulecheck.go

@@ -7,6 +7,7 @@ import (
 	. "jy/mongodbutil"
 	ju "jy/util"
 	qu "qfw/util"
+	"qfw/util/elastic"
 	"regexp"
 	"strings"
 
@@ -163,12 +164,14 @@ func init() {
 				"pid": pid, "vid": vid},
 		)
 	})
-	Admin.POST("/check/lua/clearule", func(c *gin.Context) {
+	Admin.POST("/check/lua/checkclearlua", func(c *gin.Context) {
 		luascript, _ := c.GetPostForm("s_luascript")
-		infoid, _ := c.GetPostForm("infoid")
 		code, _ := c.GetPostForm("s_code")
 		name, _ := c.GetPostForm("s_name")
-		extinfo := checkClearScript(code, name, infoid, luascript)
+		rid, _ := c.GetPostForm("rid")
+		projectset, _ := c.GetPostForm("projectset")
+		resultcoll, _ := c.GetPostForm("resultcoll")
+		extinfo := checkClearScript(code, name, resultcoll, rid, projectset, luascript)
 		c.JSON(200, gin.H{"rep": extinfo})
 	})
 }
@@ -331,12 +334,16 @@ func checkCoreScript(code, name, infoid, script string) interface{} {
 }
 
 //lua清理脚本测试
-func checkClearScript(code, name, infoid, script string) interface{} {
-	//	doc, _ := Mgo.FindById("bidding", infoid, extract.Fields)
-	//	delete(*j.Data, "contenthtml")
-	//	lua := ju.LuaScript{Code: code, Name: name, Doc: *j.Data, Script: script}
-	//	lua.Block = j.Block
-	//	rep := lua.RunScript("core")
-	//	return rep
-	return nil
+func checkClearScript(code, name, resultcoll, _id, projectset, script string) interface{} {
+	doc, _ := Mgo.FindById(resultcoll, _id, extract.Fields2)
+	data := elastic.Get(qu.ObjToString(ju.Config["mergetablealias"]), qu.ObjToString(ju.Config["mergetable"]), `{"query":{"term":{"list.infoid":"`+_id+`"}}}`)
+	kvMap := make(map[string][]map[string]interface{})
+	if len(*data) > 0 {
+		list := (*data)[0]["list"].([]interface{})
+		listArr := qu.ObjArrToMapArr(list)
+		kvMap["list"] = listArr
+	}
+	lua := ju.LuaScript{Code: code, Name: name, Doc: *doc, Script: script, KvMap: kvMap}
+	clearResult := lua.ClearRunScript() //清理后结果
+	return clearResult
 }

src/jy/extract/clearesult.go

@@ -13,7 +13,7 @@ var CltLogs []map[string]interface{} //清理日志
 
 func (c *ClearTask) ClearProcess(doc *map[string]interface{}) {
 	qu.Try(func() {
-		//data := elastic.Get(qu.ObjToString(ju.Config["mergetablealias"]), qu.ObjToString(ju.Config["mergetable"]), `{"query":{"term":{"list.infoid":"59925222e138231f45d7bb4f"}}}`)
+		//log.Println("==============", elastic.Get(qu.ObjToString(ju.Config["mergetablealias"]), qu.ObjToString(ju.Config["mergetable"]), `{"query":{"term":{"list.infoid":"596c1b6a5d11e1c74548725f"}}}`))
 		_id := qu.BsonIdToSId((*doc)["_id"])
 		data := elastic.Get(qu.ObjToString(ju.Config["mergetablealias"]), qu.ObjToString(ju.Config["mergetable"]), `{"query":{"term":{"list.infoid":"`+qu.BsonIdToSId((*doc)["_id"])+`"}}}`)
 		kvMap := make(map[string][]map[string]interface{})
@@ -45,8 +45,6 @@ func (c *ClearTask) ClearProcess(doc *map[string]interface{}) {
 				}
 			}
 			tmpmaparr[field] = tmparr
-			log.Println(field, "====", len((*doc)[field]), (*doc)[field])
-			log.Println("tmpmaparr=====", len(tmpmaparr), tmpmaparr)
 			set[field] = (*doc)[field]
 		}
 		tmpmaparr["resultid"] = _id
@@ -57,16 +55,14 @@ func (c *ClearTask) ClearProcess(doc *map[string]interface{}) {
 		//封装更新信息
 		updatearr := []map[string]interface{}{
 			map[string]interface{}{
-				"_id": _id,
+				"_id": qu.StringTOBsonId(_id),
 			},
 			map[string]interface{}{
 				"$set": set,
 			},
 		}
-		log.Println("updatearr====", updatearr)
 		lock.Lock()
 		c.UpdateResult = append(c.UpdateResult, updatearr)
-		log.Println("len(updatearr)=====", len(c.UpdateResult))
 		lock.Unlock()
 	}, func(err interface{}) {
 		log.Println((*doc)["_id"], err)
@@ -82,17 +78,19 @@ func SaveCltLog() {
 	tmpLogs = CltLogs
 	CltLogs = []map[string]interface{}{}
 	lock.Unlock()
-	if len(tmpLogs) < saveLimit {
-		db.Mgo.SaveBulk("clearlogs", tmpLogs...)
-	} else {
-		for {
-			if len(tmpLogs) > saveLimit {
-				tmp := tmpLogs[:saveLimit]
-				db.Mgo.SaveBulk("clearlogs", tmp...)
-				tmpLogs = tmpLogs[saveLimit:]
-			} else {
-				db.Mgo.SaveBulk("clearlogs", tmpLogs...)
-				break
+	if len(tmpLogs) > 0 {
+		if len(tmpLogs) < saveLimit {
+			db.Mgo.SaveBulk("clearlogs", tmpLogs...)
+		} else {
+			for {
+				if len(tmpLogs) > saveLimit {
+					tmp := tmpLogs[:saveLimit]
+					db.Mgo.SaveBulk("clearlogs", tmp...)
+					tmpLogs = tmpLogs[saveLimit:]
+				} else {
+					db.Mgo.SaveBulk("clearlogs", tmpLogs...)
+					break
+				}
 			}
 		}
 	}
@@ -105,9 +103,9 @@ func (c *ClearTask) UpdateResultVal() {
 	c.ClearChannel = make(chan bool, 5)
 	c.UpdateResult = [][]map[string]interface{}{}
 	for {
-		if len(c.UpdateResult) > 500 {
+		if len(c.UpdateResult) > 2 {
 			c.ClearChannel <- true
-			arr := c.UpdateResult[:500]
+			arr := c.UpdateResult[:2]
 			go func(tmp *[][]map[string]interface{}) {
 				qu.Try(func() {
 					db.Mgo.UpdateBulk(c.ClearTaskInfo.FromColl, *tmp...)
@@ -118,7 +116,7 @@ func (c *ClearTask) UpdateResultVal() {
 					<-c.ClearChannel
 				})
 			}(&arr)
-			c.UpdateResult = c.UpdateResult[500:]
+			c.UpdateResult = c.UpdateResult[2:]
 		} else {
 			c.ClearChannel <- true
 			arr := c.UpdateResult

src/jy/extract/clearudp.go

@@ -8,16 +8,14 @@ import (
 	mu "mfw/util"
 	"net"
 	qu "qfw/util"
-	"qfw/util/elastic"
 
 	"gopkg.in/mgo.v2/bson"
 )
 
 var ClearUdpclient mu.UdpClient //清理udp对象
+
 //udp通知清理
 func ClearUdp() {
-	//初始化elastic连接
-	elastic.InitElasticSize(qu.ObjToString(ju.Config["elasticsearch"]), qu.IntAllDef(ju.Config["elasticPoolSize"], 30))
 	//nextNodes = qu.ObjArrToMapArr(ju.Config["nextNode"].([]interface{}))
 	ClearUdpclient = mu.UdpClient{Local: ":" + qu.ObjToString(ju.Config["clearudpport"]), BufSize: 1024}
 	ClearUdpclient.Listen(clearProcessUdpMsg)
@@ -72,7 +70,6 @@ func ClearByUdp(sid, eid string) {
 	if count < PageSize {
 		limit = count
 	}
-	log.Println("pageNum---", pageNum)
 	//一次查询5000条数据进行清理
 	for i := 0; i < pageNum; i++ {
 		query = bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sid)}}

src/main.go

@@ -11,13 +11,18 @@ import (
 	"jy/util"
 	"log"
 	qu "qfw/util"
+	"qfw/util/elastic"
 	redis "qfw/util/redis"
 )
 
 func init() {
 	qu.ReadConfig(&util.Config)
+	//初始化mongo连接
 	util.InitMgoPool()
+	//初始化redis
 	redis.InitRedisBySize(qu.ObjToString(util.Config["redis"]), 50, 30, 240)
+	//初始化elastic连接
+	elastic.InitElasticSize(qu.ObjToString(util.Config["elasticsearch"]), qu.IntAllDef(util.Config["elasticPoolSize"], 30))
 }
 
 func main() {

src/web/templates/admin/check_luaclearlogic.html

@@ -10,7 +10,7 @@
 		  <li><a href="/admin/versionclear"><i class="fa fa-dashboard"></i>清理版本</a></li>
 		  <li><a href="/admin/clearversion/info?vid={{.vid}}">属性配置</a></li>
 		  <li><a href="/admin/clearulelogic?vid={{.vid}}&pid={{.pid}}">清理逻辑</a></li>
-		 <li class="active"><a href="/admin/check/lua/core?vid={{.vid}}&pid={{.pid}}&sid={{.sid}}&s_code={{.code}}">{{.code}}</a></li>
+		 <li class="active"><a href="/admin/check/lua/clearlogic?vid={{.vid}}&pid={{.pid}}&s_code={{.code}}">{{.code}}</a></li>
 		</ol>
     </section>
   <!-- Main content -->
@@ -18,19 +18,30 @@
     	<form id="dataform" class="form-horizontal">
 			<div class="form-group">
 				<div class="col-sm-12">
-					<select name="infoid" class="form-control">
-	                	<option value="">请选择测试信息</option>
-						{{if .rule.infoid}}
-							{{$ruleid := .rule.infoid}}
-							{{range .infos}}
-								<option value="{{._id}}" {{if eq ._id $ruleid}} selected {{end}}>{{.title}}</option>
-							{{end}}
-						{{else}}
-							{{range .infos}}
-								<option value="{{._id}}">{{.title}}</option>
-							{{end}}
-						{{end}}
-	              	</select>
+					<div class="col-sm-4">
+						<div class="row form-group">
+							<label for="lastname" class="col-sm-3 control-label">结果表：</label>
+							<div class="col-sm-8">
+								<input type="text" name="resultcoll" class="form-control" must="true">
+							</div>
+						</div>
+					</div>
+					<div class="col-sm-4">
+						<div class="row form-group">
+							<label for="lastname" class="col-sm-2 control-label">id：</label>
+							<div class="col-sm-8">
+								<input type="text" name="rid" class="form-control" must="true">
+							</div>
+						</div>
+					</div>
+					<!--<div class="col-sm-4">
+						<div class="row form-group">
+							<label for="lastname" class="col-sm-3 control-label">合并表：</label>
+							<div class="col-sm-8">
+								<input type="text" name="projectset" class="form-control" must="true">
+							</div>
+						</div>
+					</div>-->
 				</div>
 				<label for="code" class="col-sm-1 control-label">代码: {{.rule.s_code}}</label>
 			    <label for="code" class="col-sm-3 control-label">名称: {{.rule.s_name}}</label>
@@ -59,7 +70,7 @@
 com.initMirror(400)
 menuActive("versionclear")
 function check(){
-	$.post("/admin/check/lua/corerule",
+	$.post("/admin/check/lua/checkclearlua",
 		com.serializeArray("dataform"), 
 		function(r) {
 			if(r.rep){				
@@ -69,7 +80,7 @@ function check(){
 	);
 }
 function save(){
-	$.post("/admin/logicore/save",
+	$.post("/admin/clearulelogic/save",
 		com.serializeArray("dataform"),
 		function(r){
 			if(r){