任务测试

src/github.com/dlclark/regexp2

@@ -1 +1 @@
-Subproject commit 902a5ce7a7812e2ba9f73b9d96c09d5136df39cd
+Subproject commit 7632a260cbaf5e7594fc1544a503456ecd0827f1

src/jy/admin/task/task.go

@@ -4,6 +4,7 @@ package task
 
 import (
 	. "jy/admin"
+	. "jy/extract"
 	. "jy/mongodbutil"
 	"time"
 
@@ -52,4 +53,13 @@ func init() {
 		}
 		c.JSON(200, gin.H{"data": list})
 	})
+
+	//任务测试
+	Admin.POST("/task/test", func(c *gin.Context) {
+		startid, _ := c.GetPostForm("startid")
+		num, _ := c.GetPostForm("num")
+		taskid, _ := c.GetPostForm("taskid")
+		b := StartExtractTestTask(taskid, startid, num)
+		c.JSON(200, gin.H{"rep": b})
+	})
 }

src/jy/extract/extract.go

@@ -8,6 +8,7 @@ import (
 	ju "jy/util"
 	"log"
 	qu "qfw/util"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -25,6 +26,49 @@ var (
 	Fields = `{"title":1,"detail":1,"contenthtml":1,"href":1,"spidercode":1,"toptype":1,"area":1,"city":1}`
 )
 
+//启动测试抽取
+func StartExtractTestTask(taskId, startId, num string) bool {
+	defer qu.Catch()
+	ext := &ExtractTask{}
+	ext.Id = taskId
+	ext.IsRun = true
+	ext.InitTaskInfo()
+	ext.TaskInfo.DB = db.MgoFactory(1, 3, 120, ext.TaskInfo.FromDbAddr, ext.TaskInfo.FromDB)
+	ext.InitRulePres()
+	ext.InitRuleBacks()
+	ext.InitRuleCore()
+	ext.InitTag()
+	ext.InitClearFn()
+	return RunExtractTestTask(ext, startId, num)
+}
+
+func IdTrans(startId string) bson.ObjectId {
+	defer qu.Catch()
+	return bson.ObjectIdHex(startId)
+}
+
+//开始测试任务抽取
+func RunExtractTestTask(ext *ExtractTask, startId, num string) bool {
+	n, _ := strconv.Atoi(num)
+	id := IdTrans(startId)
+	if id.Valid() {
+		query := bson.M{"_id": bson.M{"$gt": bson.ObjectIdHex(startId)}}
+		//list, _ := ext.TaskInfo.DB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, -1, -1)
+		list, _ := ext.TaskInfo.DB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, n)
+		for _, v := range *list {
+			if !ext.IsRun {
+				break
+			}
+			j := PreInfo(v)
+			ext.TaskInfo.ProcessPool <- true
+			go ext.ExtractProcess(j)
+		}
+		return true
+	} else {
+		return false
+	}
+}
+
 //启动抽取
 func StartExtractTaskId(taskId string) bool {
 	ext := TaskList[taskId]

src/jy/pretreated/colonkv.go

@@ -2,7 +2,7 @@
 package pretreated
 
 import (
-	"clear"
+	"jy/clear"
 	. "jy/util"
 	qutil "qfw/util"
 	"regexp"

src/jy/pretreated/winnerorder.go

@@ -1,7 +1,7 @@
 package pretreated
 
 import (
-	"clear"
+	//"jy/clear"
 	"jy/util"
 	qutil "qfw/util"
 	"regexp"
@@ -236,7 +236,7 @@ func (wo *WinnerOrderEntity) clear(typ, v string) interface{} {
 	if typ == "中标单位" && regDivision.MatchString(v) {
 		v = findCompanyReg.FindString(v)
 	}
-	return clear.ClearResult(typ, v)
+	return nil //clear.ClearResult(typ, v)
 }
 
 //

src/web/templates/admin/task_list.html

@@ -67,7 +67,7 @@ $(function () {
 					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
 					'<a class="btn btn-sm btn-success opr" opr="start">启动</a>　'+
 					'<a class="btn btn-sm btn-info opr" opr="stop">停止</a>　'+
-					'<a class="btn btn-sm btn-warning opr" opr="test">测试</a>'+
+					'<a class="btn btn-sm btn-warning opr" opr="test" taskid="'+val+'">测试</a>'+
 					'</div>';
 				return  tmp
 			}}
@@ -75,18 +75,18 @@ $(function () {
 	});
 	ttable.on('init.dt', function () {
 		$(".opr").click(function(){
-			var n=$(this).attr("opr")
-			var htmlObj={},obj
-			var _tit=""
+			var n=$(this).attr("opr");
+			var taskid=$(this).attr("taskid");
+			var htmlObj={},obj,tag=[],bts=[];
+			var _tit="";
 			switch(n){
 			case "edit":			
-				obj=ttable.row($(this).closest("tr")).data()	
-				_tit="编辑-"+obj.s_taskname
+				obj=ttable.row($(this).closest("tr")).data();	
+			case "test":
+				obj=ttable.row($(this).closest("tr")).data();
 			case "new":
-				_tit="新增抽取任务"
-				htmlObj={
-					title:_tit,
-					tag:[
+				/*表单*/
+				addtask=[
 					{label:"任务名称",s_label:"s_taskname",placeholder:"剑鱼抽取",must:true},
 					{label:"数据库连接",s_label:"s_mgoaddr",must:true},
 					{label:"数据库",s_label:"s_mgodb",must:true},
@@ -106,35 +106,98 @@ $(function () {
 					}()},
 					{label:"是否统计",s_label:"i_count",type:"tpl_list_local",must:true,list:[{"s_name":"是","_id":1},{"s_name":"否","_id":0}],default:0},
 					{label:"使用版本",s_label:"s_version",type:"tpl_list_local",must:true,url:"/admin/task/getversion"},
-					{s_label:"_id",type:"tpl_hidden"}
-					],
-					bts:[
-						{label:"保存",class:"btn-primary",
-							fun:function(){
-								var obj={}
-								var bcon=true
-								$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
-									var val=$(el).val();
-									if(el.id!="_id"&&$(el).attr("must")&&!val){
-										bcon=false
-										return false
+					//{s_label:"_id",type:"tpl_hidden"}
+					{label:"起始id",s_label:"s_extlastid"}
+				];
+				testtask=[
+					{label:"起始id",s_label:"s_startid",must:true},
+					{label:"数据数量",s_label:"s_datanum",placeholder:"5",must:true}
+				];
+				/*按钮*/
+				//新增保存按钮
+				addtaskbtn=[
+					{label:"保存",class:"btn-primary",
+						fun:function(){
+							var obj={}
+							var bcon=true
+							$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+								var val=$(el).val();
+								if(el.id!="_id"&&$(el).attr("must")&&!val){
+									bcon=false
+									return false
+								}
+								obj[el.id]=$(el).val()
+							})
+							if (bcon){								
+								$.post("/admin/task/save",obj,function(data){
+									if(data&&data.rep){
+										window.location.href="/admin/task"								
+									}else{
+										alert(data.msg)
 									}
-									obj[el.id]=$(el).val()
-								})
-								if (bcon){								
-									$.post("/admin/task/save",obj,function(data){
-										if(data&&data.rep){
-											window.location.href="/admin/task"								
-										}else{
-											alert(data.msg)
-										}
-									},'json')
-								}else{
-									alert("红色标签的表单不能为空！")
+								},'json')
+							}else{
+								alert("红色标签的表单不能为空！")
+							}
+						}
+					}
+				];
+				//测试启动按钮
+				testtaskbtn=[
+					{label:"Run",class:"btn-danger",
+						fun:function(){
+							var reg = /^[0-9]*$/;
+							var id = $("#s_startid").val();
+							var num = $("#s_datanum").val();
+							var bcon = true;
+							if(id == ""){
+								bcon = false;
+								alert("红色标签的表单不能为空！");
+								return
+							}
+							if(num != ""){//验证数字
+								if(!reg.test(num)){
+									bcon = false;
+									alert("数据数量填写错误！");
+									return
 								}
+							}else{
+								num = "5";
+							}
+							if(bcon){								
+								$.post("/admin/task/test",{"startid":id,"num":num,"taskid":taskid},function(data){
+									if(data&&data.rep){
+										$("#myModal").modal("hide");
+										alert("测试完成!");								
+									}else{
+										alert("起始id输入错误!");	
+									}
+								},'json')
+								
+							}else{
+								alert("红色标签的表单不能为空！")
 							}
 						}
-					]
+					}
+				];
+				
+				if(n == "new"){
+					_tit="新增抽取任务";
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}else if(n == "test"){
+					_tit="测试任务";
+					tag = com.pushArry(tag,testtask);
+					bts = com.pushArry(bts,testtaskbtn);
+				}else if(n == "edit"){
+					_tit="编辑-"+obj.s_taskname;
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					bts:bts
 				}
 			OpenDialog(htmlObj,obj)
 			break;