Merge branch 'dev3.3' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.3

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
-  </component>
-</project>

dataprocess/src/front/front.go

@@ -43,6 +43,12 @@ type Front struct {
 	roleDel        xweb.Mapper `xweb:"/front/role/edit/del"`    //权限编辑删除
 	roleSecondEdit xweb.Mapper `xweb:"/front/role/second/edit"` //二级权限编辑
 
+	logicManager	xweb.Mapper `xweb:"/front/logic"`			//逻辑管理
+	logicPre		xweb.Mapper	`xweb:"/front/logic/pre"`		//预处理逻辑
+	logicMatch		xweb.Mapper	`xweb:"/front/logic/match"`		//匹配逻辑
+	logicClean		xweb.Mapper	`xweb:"/front/logic/clean"`		//清洗
+	logicPreSav		xweb.Mapper	`xweb:"/front/logic/pre/save"`	//预处理逻辑保存
+
 }
 
 func (f *Front) Login() {

dataprocess/src/front/logic.go

@@ -0,0 +1,74 @@
+package front
+
+import (
+	. "../util"
+	qu "qfw/util"
+	"time"
+)
+
+func (f *Front) LogicManager() {
+	defer qu.Catch()
+	f.Render("com/logic_list.html")
+}
+
+func (f *Front) LogicPre() {
+	defer qu.Catch()
+	if f.Method() == "POST" {
+		data, _ := Mgo.Find("logic", nil, nil, nil, false, -1, -1)
+		f.ServeJson(map[string]interface{}{
+			"data": data,
+		})
+	} else {
+		f.Render("com/logic_pre.html")
+	}
+}
+
+func (f *Front) LogicMatch() {
+	defer qu.Catch()
+	if f.Method() == "POST" {
+		data, _ := Mgo.Find("logic", nil, nil, nil, false, -1, -1)
+		f.ServeJson(map[string]interface{}{
+			"data": data,
+		})
+	} else {
+		f.Render("com/logic_match.html")
+	}
+}
+
+func (f *Front) LogicClean() {
+	defer qu.Catch()
+	if f.Method() == "POST" {
+		data, _ := Mgo.Find("logic", nil, nil, nil, false, -1, -1)
+		f.ServeJson(map[string]interface{}{
+			"data": data,
+		})
+	} else {
+		f.Render("com/logic_clean.html")
+	}
+}
+
+func (f *Front) LogicPreSav() {
+	defer qu.Catch()
+	_id := f.GetString("_id")
+	data := GetPostForm(f.Request)
+	b := false
+	if _id == "" {
+		data["delete"] = false
+		data["l_comeintime"] = time.Now().Unix()
+		data["i_type"] = 0
+		//session := f.GetSession("user").(map[string]interface{})
+		//data["s_username"] = session["name"].(string)
+		//data["s_userid"] = session["id"].(string)
+		//data["i_runstate"] = 0
+		b = Mgo.Save("logic", data) != ""
+	} else {
+		data["l_lasttime"] = time.Now().Unix()
+		b = Mgo.Update("logic", `{"_id":"`+_id+`"}`, map[string]interface{}{
+			"$set": data,
+		}, false, false)
+	}
+	f.ServeJson(map[string]interface{}{
+		"rep": b,
+	})
+}
+

dataprocess/src/task/task.go

@@ -18,6 +18,13 @@ type Task struct {
 	taskDel   xweb.Mapper `xweb:"/task/del"`   //删除任务
 	taskStope xweb.Mapper `xweb:"/task/start"` //启动任务
 	taskStop  xweb.Mapper `xweb:"/task/stop"`  //停止任务
+	//逻辑配置
+	taskPreProcessLogic xweb.Mapper `xweb:"/task/logic/preProcess"` //预处理逻辑
+	taskPreProcessSave  xweb.Mapper `xweb:"/task/preprocess/save"`  //预处理逻辑保存
+	askPreProcessDel    xweb.Mapper `xweb:"/task/preprocess/del"`   //预处理逻辑删除
+
+	taskMatchLogic xweb.Mapper `xweb:"/task/logic/match"` //匹配逻辑
+	taskfieldLogic xweb.Mapper `xweb:"/task/logic/field"` //字段逻辑
 }
 
 func (t *Task) TaskList() {
@@ -81,3 +88,45 @@ func (t *Task) TaskDel() {
 	})
 
 }
+
+func (t *Task) TaskPreProcessLogic() {
+	defer qu.Catch()
+	taskid := t.GetString("taskid")
+	if t.Method() == "POST" {
+		data, _ := Mgo.Find("task", `{"delete":false}`, nil, nil, false, -1, -1)
+		t.ServeJson(map[string]interface{}{
+			"data": data,
+		})
+	} else {
+		t.T["taskid"] = taskid
+		t.Render("com/task_preprocess_list.html", &t.T)
+	}
+}
+
+func (t *Task) TaskMatchLogic() {
+	defer qu.Catch()
+	taskid := t.GetString("taskid")
+	if t.Method() == "POST" {
+		data, _ := Mgo.Find("task", `{"delete":false}`, nil, nil, false, -1, -1)
+		t.ServeJson(map[string]interface{}{
+			"data": data,
+		})
+	} else {
+		t.T["taskid"] = taskid
+		t.Render("com/task_match_list.html", &t.T)
+	}
+}
+
+func (t *Task) TaskfieldLogic() {
+	defer qu.Catch()
+	taskid := t.GetString("taskid")
+	if t.Method() == "POST" {
+		data, _ := Mgo.Find("task", `{"delete":false}`, nil, nil, false, -1, -1)
+		t.ServeJson(map[string]interface{}{
+			"data": data,
+		})
+	} else {
+		t.T["taskid"] = taskid
+		t.Render("com/task_field_list.html", &t.T)
+	}
+}

dataprocess/src/web/templates/com/logic_clean.html

@@ -0,0 +1,252 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<div class="content-wrapper" id="showbtn">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增逻辑</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  	<li><a href="/front/logic"><i class="fa fa-dashboard"></i> 逻辑管理</a></li>
+			<li><a href="/front/logic/clean"><i class="fa fa-dashboard"></i> 清洗逻辑</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+		                <th>逻辑名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>任务逻辑</th> <th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+</div>
+{{include "com/dialog.html"}}
+{{include "com/footer.html"}}
+<script>
+menuActive("logic/list")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : false,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/logic/list",
+			"type": "post",
+			"data":{}
+		 },
+		"language": {
+            "url": "/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+            { "data": "s_taskname"},
+			{ "data": "l_comeintime",render:function(val){
+				var dt=new Date()
+				dt.setTime(parseInt(val)*1000)
+				return dt.format("yyyy-MM-dd hh:mm:ss")
+			}},
+			{ "data": "s_username"},
+			{ "data": "s_descript","width":"25%"},
+			{ "data": "i_runstate",render:function(val){
+					if(val==1){
+						return "<i class='fa fa-fw fa-circle text-green'></i>运行中"
+					}else{
+						return "<i class='fa fa-fw fa-circle text-danger'></i>未启动"
+					}
+				}},
+			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+					'<a class="btn btn-sm btn-success opr" opr="start" onclick="start(\''+row._id+'\')">启动</a>　'+
+					'<a class="btn btn-sm btn-info opr" opr="stop" onclick="stop(\''+row._id+'\')">停止</a>　'+
+					'<a class="btn btn-sm btn-danger opr" opr="del" onclick="del(\''+val+'\')">删除</a>'+
+					'</div>';
+				return  tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$("#showbtn").on('click','a.opr',function(){
+			var n=$(this).attr("opr");
+			var taskid=$(this).attr("taskid");
+			var htmlObj={},obj,tag=[],bts=[];
+			var _tit="";
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();	
+			case "new":
+				/*表单*/
+				addtask=[
+					{label:"任务名称",s_label:"s_taskname",placeholder:"数据清洗",must:true},
+					{label:"源库连接",s_label:"s_mgoaddr",must:true},
+					{label:"源数据库",s_label:"s_mgodb",must:true},
+					{label:"源表",s_label:"s_mgocoll",must:true},
+					{label:"保存表",s_label:"s_mgosavecoll",placeholder:"127.0.0.1:27080/dataprocess/bidding(数据库地址/数据库/表)",must:true},
+					{label:"描述",s_label:"s_descript",type:"tpl_text"},
+					/*
+					{label:"是否追踪",s_label:"i_track",type:"tpl_list_local",must:true,list:[{"s_name":"是","_id":1},{"s_name":"否","_id":0}],default:0,fun:function(){
+						var to=$("#i_track")					
+						to.val($(this).attr("_id"))
+						$("#s_show",to.closest("div")).val($(this).text())
+						//追踪表样式
+						$("#s_trackcoll").attr("must",$(this).attr("_id")==1)
+						$("#s_trackcoll").closest("div.row").find("label").css("color",$(this).attr("_id")==1?"red":"")
+					}},
+					{label:"追踪记录表",s_label:"s_trackcoll",must:function(){
+						return  obj&&obj.i_track
+					}()},
+					{label:"是否统计",s_label:"i_count",type:"tpl_list_local",must:true,list:[{"s_name":"是","_id":1},{"s_name":"否","_id":0}],default:0},
+					*/
+					//{label:"使用版本",s_label:"s_version",type:"tpl_list_local",must:true,url:"/admin/task/getversion"},
+					{label:"并发数量",s_label:"i_process",placeholder:"5",must:true},
+					{label:"预处理逻辑",s_label:"s_preprocesslogic",type:"tpl_list_local",url:""},
+					{label:"匹配逻辑",s_label:"s_class",type:"tpl_list_ajax",url:"",fun:function(){
+						var ids=$("#s_class").data("ids")
+						var parentDiv=$("#s_class").closest("div")
+						ids=ids||{}
+						var tid=$(this).attr("_id")
+						if(!ids[tid]){
+							ids[tid]=true
+							$("#s_class").data("ids",ids)
+							var tpl1=$('<div class="alert alert-dismissible alert-success" style="min-width:50px;max-width:250px;font-size:10px;padding:3px;margin:5px;display:inline-block"><button type="button" class="close" data-dismiss="alert" style="right:0px;">&times;</button><span></span></div>')
+							tpl1.find("span").text($(this).text())							
+							tpl1.attr("tid",tid)
+							tpl1.find("button").click(function(){
+								var ttid=$(this).closest(".alert").attr("tid");
+								var iids=$("#s_class").data("ids")
+								delete iids[ttid]
+								$("#s_class").val(function(){
+									var strid=[]
+									for(var k in iids){
+										strid.push(k)
+									}
+									return strid.join(",")
+								}())
+							})
+							$("#s_show",parentDiv).append(tpl1)
+							$("#s_class").val(function(){
+								var strid=[]
+								for(var k in ids){
+									strid.push(k)
+								}
+								return strid.join(",")
+							}())
+						}
+					}},
+					//{label:"匹配逻辑",s_label:"s_matchlogic",type:"tpl_list_local",url:""},
+					{label:"字段",s_label:"s_field",placeholder:"title"},
+					{label:"起始id",s_label:"s_extlastid",must:true},
+					{s_label:"_id",type:"tpl_hidden"},
+				];
+				/*testtask=[
+					{label:"起始id",s_label:"s_startid",must:true},
+					{label:"数据数量",s_label:"s_datanum",placeholder:"5",must:true}
+				];*/
+				/*按钮*/
+				//新增保存按钮
+				addtaskbtn=[
+					{label:"保存",class:"btn-primary",
+						fun:function(){
+							var obj={}
+							var bcon=true
+							$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+								var val=$(el).val();
+								if(el.id!="_id"&&$(el).attr("must")&&!val){
+									bcon=false
+									return false
+								}
+								obj[el.id]=$(el).val()
+							})
+							if (bcon){								
+								$.post("/task/save",obj,function(data){
+									if(data&&data.rep){
+										//window.location.href="/task/list"
+										$('#myModal').modal('hide');
+										ttable.ajax.reload();
+									}else{
+										alert(data.msg)
+									}
+								},'json')
+							}else{
+								alert("红色标签的表单不能为空！")
+							}
+						}
+					}
+				];
+				if(n == "new"){
+					_tit="新增抽取任务";
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}else if(n == "edit"){
+					_tit="编辑-"+obj.s_taskname;
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+	
+})
+function start(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/start",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				//window.location.reload();
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+function stop(_id){
+	showConfirm("确定停止？", function() {
+		$.ajax({
+			url:"/task/stop",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				//window.location.reload()
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+function del(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+</script>

dataprocess/src/web/templates/com/logic_list.html

@@ -7,10 +7,10 @@
 <div class="content-wrapper" id="showbtn">
 	<section class="content-header">
 		<h1>
-			<small><a class="btn btn-primary opr" opr="new">新增任务</a></small>
+			<small><a class="btn btn-primary opr" opr="new">新增逻辑</a></small>
 		</h1>
 		<ol class="breadcrumb">
-		  <li><a href="/admin/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>		  
+		  	<li><a href="/front/logic"><i class="fa fa-dashboard"></i> 逻辑管理</a></li>
 		</ol>
     </section>
   <!-- Main content -->
@@ -22,7 +22,7 @@
 		            <table id="dataTable" class="table table-bordered table-hover">
 		              <thead>
 		              <tr>
-		                <th>任务名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>运行状态</th> <th>操作</th>
+		                <th>逻辑名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>任务逻辑</th> <th>操作</th>
 		              </tr>
 		              </thead>
 		            </table>
@@ -37,7 +37,7 @@
 {{include "com/dialog.html"}}
 {{include "com/footer.html"}}
 <script>
-menuActive("list")
+menuActive("logic/list")
 $(function () {
 	ttable=$('#dataTable').DataTable({
 		"paging"      : false,
@@ -62,13 +62,14 @@ $(function () {
 				return dt.format("yyyy-MM-dd hh:mm:ss")
 			}},
 			{ "data": "s_username"},
-			{ "data": "s_descript","width":"25%"},
-			{ "data": "i_runstate",render:function(val){
-				if(val==1){
-					return "<i class='fa fa-fw fa-circle text-green'></i>运行中"
-				}else{
-					return "<i class='fa fa-fw fa-circle text-danger'></i>未启动"
-				}
+			{ "data": "s_descript","width":"23%"},
+			{ "data": "_id","width":"21%",render:function(row){
+				add_logic = '<div>'+
+						'<a class="btn btn-sm btn-primary" href="/front/logic/pre">预处理</a>　'+
+						'<a class="btn btn-sm btn-success" href="/front/logic/match">匹配</a>　'+
+						'<a class="btn btn-sm btn-info" href="/front/logic/clean">清洗</a>　'+
+						'</div>'
+				return add_logic
 			}},
 			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
 				tmp = '<div>'+

dataprocess/src/web/templates/com/logic_match.html

@@ -0,0 +1,252 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<div class="content-wrapper" id="showbtn">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增逻辑</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  	<li><a href="/front/logic"><i class="fa fa-dashboard"></i> 逻辑管理</a></li>
+			<li><a href="/front/logic/pre"><i class="fa fa-dashboard"></i> 预处理逻辑</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+		                <th>逻辑名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>任务逻辑</th> <th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+</div>
+{{include "com/dialog.html"}}
+{{include "com/footer.html"}}
+<script>
+menuActive("logic/list")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : false,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/logic/list",
+			"type": "post",
+			"data":{}
+		 },
+		"language": {
+            "url": "/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+            { "data": "s_taskname"},
+			{ "data": "l_comeintime",render:function(val){
+				var dt=new Date()
+				dt.setTime(parseInt(val)*1000)
+				return dt.format("yyyy-MM-dd hh:mm:ss")
+			}},
+			{ "data": "s_username"},
+			{ "data": "s_descript","width":"25%"},
+			{ "data": "i_runstate",render:function(val){
+					if(val==1){
+						return "<i class='fa fa-fw fa-circle text-green'></i>运行中"
+					}else{
+						return "<i class='fa fa-fw fa-circle text-danger'></i>未启动"
+					}
+				}},
+			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+					'<a class="btn btn-sm btn-success opr" opr="start" onclick="start(\''+row._id+'\')">启动</a>　'+
+					'<a class="btn btn-sm btn-info opr" opr="stop" onclick="stop(\''+row._id+'\')">停止</a>　'+
+					'<a class="btn btn-sm btn-danger opr" opr="del" onclick="del(\''+val+'\')">删除</a>'+
+					'</div>';
+				return  tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$("#showbtn").on('click','a.opr',function(){
+			var n=$(this).attr("opr");
+			var taskid=$(this).attr("taskid");
+			var htmlObj={},obj,tag=[],bts=[];
+			var _tit="";
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();	
+			case "new":
+				/*表单*/
+				addtask=[
+					{label:"任务名称",s_label:"s_taskname",placeholder:"数据清洗",must:true},
+					{label:"源库连接",s_label:"s_mgoaddr",must:true},
+					{label:"源数据库",s_label:"s_mgodb",must:true},
+					{label:"源表",s_label:"s_mgocoll",must:true},
+					{label:"保存表",s_label:"s_mgosavecoll",placeholder:"127.0.0.1:27080/dataprocess/bidding(数据库地址/数据库/表)",must:true},
+					{label:"描述",s_label:"s_descript",type:"tpl_text"},
+					/*
+					{label:"是否追踪",s_label:"i_track",type:"tpl_list_local",must:true,list:[{"s_name":"是","_id":1},{"s_name":"否","_id":0}],default:0,fun:function(){
+						var to=$("#i_track")					
+						to.val($(this).attr("_id"))
+						$("#s_show",to.closest("div")).val($(this).text())
+						//追踪表样式
+						$("#s_trackcoll").attr("must",$(this).attr("_id")==1)
+						$("#s_trackcoll").closest("div.row").find("label").css("color",$(this).attr("_id")==1?"red":"")
+					}},
+					{label:"追踪记录表",s_label:"s_trackcoll",must:function(){
+						return  obj&&obj.i_track
+					}()},
+					{label:"是否统计",s_label:"i_count",type:"tpl_list_local",must:true,list:[{"s_name":"是","_id":1},{"s_name":"否","_id":0}],default:0},
+					*/
+					//{label:"使用版本",s_label:"s_version",type:"tpl_list_local",must:true,url:"/admin/task/getversion"},
+					{label:"并发数量",s_label:"i_process",placeholder:"5",must:true},
+					{label:"预处理逻辑",s_label:"s_preprocesslogic",type:"tpl_list_local",url:""},
+					{label:"匹配逻辑",s_label:"s_class",type:"tpl_list_ajax",url:"",fun:function(){
+						var ids=$("#s_class").data("ids")
+						var parentDiv=$("#s_class").closest("div")
+						ids=ids||{}
+						var tid=$(this).attr("_id")
+						if(!ids[tid]){
+							ids[tid]=true
+							$("#s_class").data("ids",ids)
+							var tpl1=$('<div class="alert alert-dismissible alert-success" style="min-width:50px;max-width:250px;font-size:10px;padding:3px;margin:5px;display:inline-block"><button type="button" class="close" data-dismiss="alert" style="right:0px;">&times;</button><span></span></div>')
+							tpl1.find("span").text($(this).text())							
+							tpl1.attr("tid",tid)
+							tpl1.find("button").click(function(){
+								var ttid=$(this).closest(".alert").attr("tid");
+								var iids=$("#s_class").data("ids")
+								delete iids[ttid]
+								$("#s_class").val(function(){
+									var strid=[]
+									for(var k in iids){
+										strid.push(k)
+									}
+									return strid.join(",")
+								}())
+							})
+							$("#s_show",parentDiv).append(tpl1)
+							$("#s_class").val(function(){
+								var strid=[]
+								for(var k in ids){
+									strid.push(k)
+								}
+								return strid.join(",")
+							}())
+						}
+					}},
+					//{label:"匹配逻辑",s_label:"s_matchlogic",type:"tpl_list_local",url:""},
+					{label:"字段",s_label:"s_field",placeholder:"title"},
+					{label:"起始id",s_label:"s_extlastid",must:true},
+					{s_label:"_id",type:"tpl_hidden"},
+				];
+				/*testtask=[
+					{label:"起始id",s_label:"s_startid",must:true},
+					{label:"数据数量",s_label:"s_datanum",placeholder:"5",must:true}
+				];*/
+				/*按钮*/
+				//新增保存按钮
+				addtaskbtn=[
+					{label:"保存",class:"btn-primary",
+						fun:function(){
+							var obj={}
+							var bcon=true
+							$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+								var val=$(el).val();
+								if(el.id!="_id"&&$(el).attr("must")&&!val){
+									bcon=false
+									return false
+								}
+								obj[el.id]=$(el).val()
+							})
+							if (bcon){
+								$.post("/task/save",obj,function(data){
+									if(data&&data.rep){
+										//window.location.href="/task/list"
+										$('#myModal').modal('hide');
+										ttable.ajax.reload();
+									}else{
+										alert(data.msg)
+									}
+								},'json')
+							}else{
+								alert("红色标签的表单不能为空！")
+							}
+						}
+					}
+				];
+				if(n === "new"){
+					_tit="新增抽取任务";
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}else if(n == "edit"){
+					_tit="编辑-"+obj.s_taskname;
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+	
+})
+function start(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/start",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				//window.location.reload();
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+function stop(_id){
+	showConfirm("确定停止？", function() {
+		$.ajax({
+			url:"/task/stop",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				//window.location.reload()
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+function del(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+</script>

dataprocess/src/web/templates/com/logic_pre.html

@@ -0,0 +1,190 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<div class="content-wrapper" id="showbtn">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增逻辑</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  	<li><a href="/front/logic"><i class="fa fa-dashboard"></i> 逻辑管理</a></li>
+			<li><a href="/front/logic/pre"><i class="fa fa-dashboard"></i> 预处理逻辑</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+		                <th>逻辑名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>任务逻辑</th> <th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+</div>
+{{include "com/dialog.html"}}
+{{include "com/footer.html"}}
+<script>
+menuActive("logic/list")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : false,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/logic/list",
+			"type": "post",
+			"data":{}
+		 },
+		"language": {
+            "url": "/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+            { "data": "s_taskname"},
+			{ "data": "l_comeintime",render:function(val){
+				var dt=new Date()
+				dt.setTime(parseInt(val)*1000)
+				return dt.format("yyyy-MM-dd hh:mm:ss")
+			}},
+			{ "data": "s_username"},
+			{ "data": "s_descript","width":"25%"},
+			{ "data": "i_runstate",render:function(val){
+					if(val==1){
+						return "<i class='fa fa-fw fa-circle text-green'></i>运行中"
+					}else{
+						return "<i class='fa fa-fw fa-circle text-danger'></i>未启动"
+					}
+				}},
+			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+					'<a class="btn btn-sm btn-success opr" opr="start" onclick="start(\''+row._id+'\')">启动</a>　'+
+					'</div>';
+				return  tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$("#showbtn").on('click','a.opr',function(){
+			var n=$(this).attr("opr");
+			var taskid=$(this).attr("taskid");
+			var htmlObj={},obj,tag=[],bts=[];
+			var _tit="";
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();	
+			case "new":
+				/*表单*/
+				addtask=[
+					{label:"逻辑名称",s_label:"s_logicname",placeholder:"数据清洗",must:true},
+					{label:"描述",s_label:"s_descript",type:"tpl_text",must:true},
+					{s_label:"_id",type:"tpl_hidden"},
+				];
+				/*按钮*/
+				//新增保存按钮
+				addtaskbtn=[
+					{label:"保存",class:"btn-primary",
+						fun:function(){
+							var obj={}
+							var bcon=true
+							$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+								var val=$(el).val();
+								if(el.id!="_id"&&$(el).attr("must")&&!val){
+									bcon=false
+									return false
+								}
+								obj[el.id]=$(el).val()
+							})
+							if (bcon){
+								console.log(obj)
+								$.post("/front/logic/pre/save",obj,function(data){
+									if(data&&data.rep){
+										//window.location.href="/task/list"
+										$('#myModal').modal('hide');
+										ttable.ajax.reload();
+									}else{
+										alert(data.msg)
+									}
+								},'json')
+							}else{
+								alert("红色标签的表单不能为空！")
+							}
+						}
+					}
+				];
+				if(n == "new"){
+					_tit="新增抽取任务";
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}else if(n == "edit"){
+					_tit="编辑-"+obj.s_taskname;
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+	
+})
+function start(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/start",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				//window.location.reload();
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+function stop(_id){
+	showConfirm("确定停止？", function() {
+		$.ajax({
+			url:"/task/stop",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				//window.location.reload()
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+function del(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+</script>

dataprocess/src/web/templates/com/task_field_list.html

@@ -0,0 +1,167 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<div class="content-wrapper" id="showbtn">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增字段逻辑</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  	<li><a href="/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>	
+			<li><a class="active" href="/task/logic/field?taskid={{.T.taskid}}">字段逻辑</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+		                <th>逻辑名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+</div>
+{{include "com/dialog.html"}}
+{{include "com/footer.html"}}
+<script>
+menuActive("task/list")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : false,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/task/logic/field",
+			"type": "post",
+			"data":{"taskid":{{.T.taskid}}}
+		 },
+		"language": {
+            "url": "/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+            { "data": "s_taskname"},
+			{ "data": "l_comeintime",render:function(val){
+				var dt=new Date()
+				dt.setTime(parseInt(val)*1000)
+				return dt.format("yyyy-MM-dd hh:mm:ss")
+			}},
+			{ "data": "s_username"},
+			{ "data": "s_descript","width":"25%"},
+			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+					'<a class="btn btn-sm btn-danger opr" opr="del" onclick="del(\''+val+'\')">删除</a>'+
+					'</div>';
+				return  tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$("#showbtn").on('click','a.opr',function(){
+			var n=$(this).attr("opr");
+			var taskid=$(this).attr("taskid");
+			var htmlObj={},obj,tag=[],bts=[];
+			var _tit="";
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();	
+			case "new":
+				/*表单*/
+				addtask=[
+					{label:"任务名称",s_label:"s_taskname",placeholder:"数据清洗",must:true},
+					{label:"源库连接",s_label:"s_mgoaddr",must:true},
+					{label:"源数据库",s_label:"s_mgodb",must:true},
+					{label:"源表",s_label:"s_mgocoll",must:true},
+					{label:"保存表",s_label:"s_mgosavecoll",placeholder:"127.0.0.1:27080/dataprocess/bidding(数据库地址/数据库/表)",must:true},
+					{label:"描述",s_label:"s_descript",type:"tpl_text"},
+					//{label:"使用版本",s_label:"s_version",type:"tpl_list_local",must:true,url:"/admin/task/getversion"},
+					{label:"并发数量",s_label:"i_process",placeholder:"5",must:true},
+					{label:"起始id",s_label:"s_extlastid",must:true},
+					{s_label:"_id",type:"tpl_hidden"},
+				];
+				/*testtask=[
+					{label:"起始id",s_label:"s_startid",must:true},
+					{label:"数据数量",s_label:"s_datanum",placeholder:"5",must:true}
+				];*/
+				/*按钮*/
+				//新增保存按钮
+				addtaskbtn=[
+					{label:"保存",class:"btn-primary",
+						fun:function(){
+							var obj={}
+							var bcon=true
+							$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+								var val=$(el).val();
+								if(el.id!="_id"&&$(el).attr("must")&&!val){
+									bcon=false
+									return false
+								}
+								obj[el.id]=$(el).val()
+							})
+							if (bcon){								
+								$.post("/task/save",obj,function(data){
+									if(data&&data.rep){
+										//window.location.href="/task/list"
+										$('#myModal').modal('hide');
+										ttable.ajax.reload();
+									}else{
+										alert(data.msg)
+									}
+								},'json')
+							}else{
+								alert("红色标签的表单不能为空！")
+							}
+						}
+					}
+				];
+				if(n == "new"){
+					_tit="新增抽取任务";
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}else if(n == "edit"){
+					_tit="编辑-"+obj.s_taskname;
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+	
+})
+function del(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+</script>

dataprocess/src/web/templates/com/task_list.html

@@ -10,7 +10,7 @@
 			<small><a class="btn btn-primary opr" opr="new">新增任务</a></small>
 		</h1>
 		<ol class="breadcrumb">
-		  <li><a href="/admin/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>		  
+		  <li><a href="/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>		  
 		</ol>
     </section>
   <!-- Main content -->
@@ -22,7 +22,7 @@
 		            <table id="dataTable" class="table table-bordered table-hover">
 		              <thead>
 		              <tr>
-		                <th>任务名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>运行状态</th> <th>操作</th>
+		                <th>任务名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>运行状态</th> <th>逻辑配置</th> <th>操作</th> <th>流程图</th>
 		              </tr>
 		              </thead>
 		            </table>
@@ -37,7 +37,7 @@
 {{include "com/dialog.html"}}
 {{include "com/footer.html"}}
 <script>
-menuActive("list")
+menuActive("/task/list")
 $(function () {
 	ttable=$('#dataTable').DataTable({
 		"paging"      : false,
@@ -62,7 +62,7 @@ $(function () {
 				return dt.format("yyyy-MM-dd hh:mm:ss")
 			}},
 			{ "data": "s_username"},
-			{ "data": "s_descript","width":"25%"},
+			{ "data": "s_descript","width":"20%"},
 			{ "data": "i_runstate",render:function(val){
 				if(val==1){
 					return "<i class='fa fa-fw fa-circle text-green'></i>运行中"
@@ -70,14 +70,28 @@ $(function () {
 					return "<i class='fa fa-fw fa-circle text-danger'></i>未启动"
 				}
 			}},
-			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+			{ "data":"_id","width":"17%",render:function(val){
 				tmp = '<div>'+
-					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
-					'<a class="btn btn-sm btn-success opr" opr="start" onclick="start(\''+row._id+'\')">启动</a>　'+
-					'<a class="btn btn-sm btn-info opr" opr="stop" onclick="stop(\''+row._id+'\')">停止</a>　'+
+					'<a class="btn btn-sm btn-primary" href="/task/logic/preProcess?taskid='+val+'">预处理</a>  '+
+					'<a class="btn btn-sm btn-info" href="/task/logic/match?taskid='+val+'">匹配</a>  '+
+					'<a class="btn btn-sm btn-warning" href="/task/logic/field?taskid='+val+'">字段</a>'+
+					'</div>';
+				return  tmp
+			}},
+			{ "data":"_id","width":"20%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>  '+
+					'<a class="btn btn-sm btn-success opr" opr="start" onclick="start(\''+row._id+'\')">启动</a>  '+
+					'<a class="btn btn-sm btn-info opr" opr="stop" onclick="stop(\''+row._id+'\')">停止</a>  '+
 					'<a class="btn btn-sm btn-danger opr" opr="del" onclick="del(\''+val+'\')">删除</a>'+
 					'</div>';
 				return  tmp
+			}},
+			{ "data":"_id",render:function(val){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary" href="#">查看</a>  '+
+					'</div>';
+				return  tmp
 			}}
        	]
 	});
@@ -99,58 +113,8 @@ $(function () {
 					{label:"源表",s_label:"s_mgocoll",must:true},
 					{label:"保存表",s_label:"s_mgosavecoll",placeholder:"127.0.0.1:27080/dataprocess/bidding(数据库地址/数据库/表)",must:true},
 					{label:"描述",s_label:"s_descript",type:"tpl_text"},
-					/*
-					{label:"是否追踪",s_label:"i_track",type:"tpl_list_local",must:true,list:[{"s_name":"是","_id":1},{"s_name":"否","_id":0}],default:0,fun:function(){
-						var to=$("#i_track")					
-						to.val($(this).attr("_id"))
-						$("#s_show",to.closest("div")).val($(this).text())
-						//追踪表样式
-						$("#s_trackcoll").attr("must",$(this).attr("_id")==1)
-						$("#s_trackcoll").closest("div.row").find("label").css("color",$(this).attr("_id")==1?"red":"")
-					}},
-					{label:"追踪记录表",s_label:"s_trackcoll",must:function(){
-						return  obj&&obj.i_track
-					}()},
-					{label:"是否统计",s_label:"i_count",type:"tpl_list_local",must:true,list:[{"s_name":"是","_id":1},{"s_name":"否","_id":0}],default:0},
-					*/
 					//{label:"使用版本",s_label:"s_version",type:"tpl_list_local",must:true,url:"/admin/task/getversion"},
 					{label:"并发数量",s_label:"i_process",placeholder:"5",must:true},
-					{label:"预处理逻辑",s_label:"s_preprocesslogic",type:"tpl_list_local",url:""},
-					{label:"匹配逻辑",s_label:"s_class",type:"tpl_list_ajax",url:"",fun:function(){
-						var ids=$("#s_class").data("ids")
-						var parentDiv=$("#s_class").closest("div")
-						ids=ids||{}
-						var tid=$(this).attr("_id")
-						if(!ids[tid]){
-							ids[tid]=true
-							$("#s_class").data("ids",ids)
-							var tpl1=$('<div class="alert alert-dismissible alert-success" style="min-width:50px;max-width:250px;font-size:10px;padding:3px;margin:5px;display:inline-block"><button type="button" class="close" data-dismiss="alert" style="right:0px;">&times;</button><span></span></div>')
-							tpl1.find("span").text($(this).text())							
-							tpl1.attr("tid",tid)
-							tpl1.find("button").click(function(){
-								var ttid=$(this).closest(".alert").attr("tid");
-								var iids=$("#s_class").data("ids")
-								delete iids[ttid]
-								$("#s_class").val(function(){
-									var strid=[]
-									for(var k in iids){
-										strid.push(k)
-									}
-									return strid.join(",")
-								}())
-							})
-							$("#s_show",parentDiv).append(tpl1)
-							$("#s_class").val(function(){
-								var strid=[]
-								for(var k in ids){
-									strid.push(k)
-								}
-								return strid.join(",")
-							}())
-						}
-					}},
-					//{label:"匹配逻辑",s_label:"s_matchlogic",type:"tpl_list_local",url:""},
-					{label:"字段",s_label:"s_field",placeholder:"title"},
 					{label:"起始id",s_label:"s_extlastid",must:true},
 					{s_label:"_id",type:"tpl_hidden"},
 				];

dataprocess/src/web/templates/com/task_match_list.html

@@ -0,0 +1,167 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<div class="content-wrapper" id="showbtn">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增匹配逻辑</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  	<li><a href="/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>	
+			<li><a class="active" href="/task/logic/match?taskid={{.T.taskid}}">匹配逻辑</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+		                <th>逻辑名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+</div>
+{{include "com/dialog.html"}}
+{{include "com/footer.html"}}
+<script>
+menuActive("task/list")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : false,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/task/logic/match",
+			"type": "post",
+			"data":{"taskid":{{.T.taskid}}}
+		 },
+		"language": {
+            "url": "/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+            { "data": "s_taskname"},
+			{ "data": "l_comeintime",render:function(val){
+				var dt=new Date()
+				dt.setTime(parseInt(val)*1000)
+				return dt.format("yyyy-MM-dd hh:mm:ss")
+			}},
+			{ "data": "s_username"},
+			{ "data": "s_descript","width":"25%"},
+			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+					'<a class="btn btn-sm btn-danger opr" opr="del" onclick="del(\''+val+'\')">删除</a>'+
+					'</div>';
+				return  tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$("#showbtn").on('click','a.opr',function(){
+			var n=$(this).attr("opr");
+			var taskid=$(this).attr("taskid");
+			var htmlObj={},obj,tag=[],bts=[];
+			var _tit="";
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();	
+			case "new":
+				/*表单*/
+				addtask=[
+					{label:"任务名称",s_label:"s_taskname",placeholder:"数据清洗",must:true},
+					{label:"源库连接",s_label:"s_mgoaddr",must:true},
+					{label:"源数据库",s_label:"s_mgodb",must:true},
+					{label:"源表",s_label:"s_mgocoll",must:true},
+					{label:"保存表",s_label:"s_mgosavecoll",placeholder:"127.0.0.1:27080/dataprocess/bidding(数据库地址/数据库/表)",must:true},
+					{label:"描述",s_label:"s_descript",type:"tpl_text"},
+					//{label:"使用版本",s_label:"s_version",type:"tpl_list_local",must:true,url:"/admin/task/getversion"},
+					{label:"并发数量",s_label:"i_process",placeholder:"5",must:true},
+					{label:"起始id",s_label:"s_extlastid",must:true},
+					{s_label:"_id",type:"tpl_hidden"},
+				];
+				/*testtask=[
+					{label:"起始id",s_label:"s_startid",must:true},
+					{label:"数据数量",s_label:"s_datanum",placeholder:"5",must:true}
+				];*/
+				/*按钮*/
+				//新增保存按钮
+				addtaskbtn=[
+					{label:"保存",class:"btn-primary",
+						fun:function(){
+							var obj={}
+							var bcon=true
+							$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+								var val=$(el).val();
+								if(el.id!="_id"&&$(el).attr("must")&&!val){
+									bcon=false
+									return false
+								}
+								obj[el.id]=$(el).val()
+							})
+							if (bcon){								
+								$.post("/task/save",obj,function(data){
+									if(data&&data.rep){
+										//window.location.href="/task/list"
+										$('#myModal').modal('hide');
+										ttable.ajax.reload();
+									}else{
+										alert(data.msg)
+									}
+								},'json')
+							}else{
+								alert("红色标签的表单不能为空！")
+							}
+						}
+					}
+				];
+				if(n == "new"){
+					_tit="新增抽取任务";
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}else if(n == "edit"){
+					_tit="编辑-"+obj.s_taskname;
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+	
+})
+function del(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+</script>

dataprocess/src/web/templates/com/task_preprocess_list.html

@@ -0,0 +1,156 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<div class="content-wrapper" id="showbtn">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增预处理逻辑</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  	<li><a href="/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>	
+			<li><a class="active" href="/task/logic/preProcess?taskid={{.T.taskid}}">预处理逻辑</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+		                <th>逻辑名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+</div>
+{{include "com/dialog.html"}}
+{{include "com/footer.html"}}
+<script>
+menuActive("task/list")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : false,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/task/logic/preProcess",
+			"type": "post",
+			"data":{"taskid":{{.T.taskid}}}
+		 },
+		"language": {
+            "url": "/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+            { "data": "s_taskname"},
+			{ "data": "l_comeintime",render:function(val){
+				var dt=new Date()
+				dt.setTime(parseInt(val)*1000)
+				return dt.format("yyyy-MM-dd hh:mm:ss")
+			}},
+			{ "data": "s_username"},
+			{ "data": "s_descript","width":"25%"},
+			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+					'<a class="btn btn-sm btn-danger opr" opr="del" onclick="del(\''+val+'\')">删除</a>'+
+					'</div>';
+				return  tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$("#showbtn").on('click','a.opr',function(){
+			var n=$(this).attr("opr");
+			var taskid=$(this).attr("taskid");
+			var htmlObj={},obj,tag=[],bts=[];
+			var _tit="";
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();	
+			case "new":
+				/*表单*/
+				addtask=[
+					{label:"任务名称",s_label:"s_taskname",placeholder:"预处理逻辑",must:true},
+					{label:"描述",s_label:"s_descript",type:"tpl_text"},
+					{s_label:"_id",type:"tpl_hidden"},
+				];
+				/*按钮*/
+				//新增保存按钮
+				addtaskbtn=[
+					{label:"保存",class:"btn-primary",
+						fun:function(){
+							var obj={}
+							var bcon=true
+							$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+								var val=$(el).val();
+								if(el.id!="_id"&&$(el).attr("must")&&!val){
+									bcon=false
+									return false
+								}
+								obj[el.id]=$(el).val()
+							})
+							if (bcon){								
+								$.post("/task/preprocess/save",obj,function(data){
+									if(data&&data.rep){
+										//window.location.href="/task/list"
+										$('#myModal').modal('hide');
+										ttable.ajax.reload();
+									}else{
+										alert(data.msg)
+									}
+								},'json')
+							}else{
+								alert("红色标签的表单不能为空！")
+							}
+						}
+					}
+				];
+				if(n == "new"){
+					_tit="新增抽取任务";
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}else if(n == "edit"){
+					_tit="编辑-"+obj.s_taskname;
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+	
+})
+function del(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+</script>

fullproject/src_v1/main.go

@@ -41,8 +41,13 @@ func DealSign() {
 			log.Println("receive:", sign)
 			if v, ok := sign.(syscall.Signal); ok && v == os.Interrupt {
 				log.Println("receice signal..,start close iter")
-				queryClose <- true
-				<-queryCloseOver
+				if P_QL.Brun {
+					queryClose <- true
+					select {
+					case <-queryCloseOver:
+					case <-time.After(30 * time.Second):
+					}
+				}
 				util.ReadConfig(&Sysconfig)
 				log.Println("signal deal over")
 			}

fullproject/src_v1/task.go

@@ -58,6 +58,7 @@ type ProjectTask struct {
 	//	LockPoolLock sync.Mutex
 	//	m1, m23, m4  map[int]int
 	//	l1, l23, l4  map[int]*sync.Mutex
+	Brun bool
 }
 
 func NewPT() *ProjectTask {
@@ -83,6 +84,7 @@ func NewPT() *ProjectTask {
 }
 
 var P_QL *ProjectTask
+var sp = make(chan bool, 5)
 
 //初始化全量合并对象
 func init() {
@@ -95,7 +97,6 @@ func init() {
 func (p *ProjectTask) updateAllQueue() {
 	arru := make([][]map[string]interface{}, p.saveSize)
 	indexu := 0
-	sp := make(chan bool, 5)
 	for {
 		select {
 		case v := <-p.updatePool:
@@ -282,15 +283,15 @@ func (p *ProjectTask) taskZl(udpInfo map[string]interface{}) {
 		//生成查询语句执行
 		p.enter(db, coll, q)
 	}
-	for {
-		if len(P_QL.updatePool) > 0 {
-			log.Println("等待调用udp", len(P_QL.updatePool))
-			time.Sleep(1 * time.Second)
-		} else {
-			break
-		}
-	}
 	if udpInfo["stop"] == nil {
+		for i := 0; i < 5; i++ {
+			sp <- true
+		}
+		for i := 0; i < 5; i++ {
+			<-sp
+		}
+		log.Println("保存完成，生索引", p.pici)
+		time.Sleep(5 * time.Second)
 		nextNode(udpInfo, p.pici)
 	}
 }
@@ -318,6 +319,10 @@ func nextNode(mapInfo map[string]interface{}, pici int64) {
 
 func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 	defer util.Catch()
+	defer func() {
+		p.Brun = false
+	}()
+	p.Brun = true
 	count, taskcount := 0, 0
 
 	pool := make(chan bool, p.thread)

src/jy/extract/extractInit.go

@@ -808,6 +808,7 @@ func (e *ExtractTask) InitClearFn(isSite bool) {
 	var cleanupdb string
 	if isSite {
 		cleanupdb = "site_cleanup"
+		e.SiteClearFn = map[string][]string{}
 	} else {
 		cleanupdb = "cleanup"
 	}

src/jy/pretreated/analystep.go

@@ -6,7 +6,6 @@ package pretreated
 import (
 	"encoding/json"
 	"jy/util"
-	//"log"
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"

src/jy/util/script.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"log"
 	qu "qfw/util"
+	"regexp"
+	"strings"
 
 	ljson "github.com/yuin/gopher-json"
 	"github.com/yuin/gopher-lua"
@@ -59,7 +61,73 @@ func (s *LuaScript) RunScript(stype string) map[string]interface{} {
 	data := map[string]interface{}{}
 	qu.Try(func() {
 		s.L = lua.NewState()
+		//s.L.DoFile("lua/comm.lua")
 		s.L.PreloadModule("json", ljson.Loader)
+		s.L.SetGlobal("findOneText", s.L.NewFunction(func(S *lua.LState) int {
+			nodetype := S.ToString(-3)
+			gpath := S.ToString(-2)
+			content := S.ToString(-1)
+			ret := FindOneText(gpath, content, nodetype)
+			S.Push(ret)
+			return 1
+		}))
+		s.L.SetGlobal("findOneHtml", s.L.NewFunction(func(S *lua.LState) int {
+			nodetype := S.ToString(-3)
+			gpath := S.ToString(-2)
+			content := S.ToString(-1)
+			ret := FindOneHtml(gpath, content, nodetype)
+			S.Push(ret)
+			return 1
+		}))
+		s.L.SetGlobal("findListText", s.L.NewFunction(func(S *lua.LState) int {
+			gpath := S.ToString(-2)
+			content := S.ToString(-1)
+			ret := s.L.NewTable()
+			FindListText(gpath, content, ret)
+			S.Push(ret)
+			return 1
+		}))
+		s.L.SetGlobal("findListHtml", s.L.NewFunction(func(S *lua.LState) int {
+			gpath := S.ToString(-2)
+			content := S.ToString(-1)
+			ret := s.L.NewTable()
+			FindListHtml(gpath, content, ret)
+			S.Push(ret)
+			return 1
+		}))
+		s.L.SetGlobal("findMap", s.L.NewFunction(func(S *lua.LState) int {
+			qmap := S.ToTable(-2)
+			content := S.ToString(-1)
+			ret := s.L.NewTable()
+			FindMap(qmap, content, ret)
+			S.Push(ret)
+			return 1
+		}))
+		//支持正则
+		s.L.SetGlobal("regexp", s.L.NewFunction(func(S *lua.LState) int {
+			index := int(S.ToNumber(-1))
+			regstr := S.ToString(-2)
+			text := S.ToString(-3)
+			reg := regexp.MustCompile(regstr)
+			reps := reg.FindAllStringSubmatchIndex(text, -1)
+			ret := s.L.NewTable()
+			number := 0
+			for _, v := range reps {
+				number++
+				ret.Insert(number, lua.LString(text[v[index]:v[index+1]]))
+			}
+			S.Push(ret)
+			return 1
+		}))
+		//支持替换
+		s.L.SetGlobal("replace", s.L.NewFunction(func(S *lua.LState) int {
+			text := S.ToString(-3)
+			old := S.ToString(-2)
+			repl := S.ToString(-1)
+			text = strings.Replace(text, old, repl, -1)
+			S.Push(lua.LString(text))
+			return 1
+		}))
 		defer s.L.Close()
 		if err := s.L.DoString(s.Script); err != nil {
 			data["err"] = err.Error()
@@ -83,6 +151,7 @@ func (s *LuaScript) RunScript(stype string) map[string]interface{} {
 					Protect: true,
 				}, lua.LString(s.Code), tab, lua.LString(block), kvMap); err != nil {
 					data["err"] = err.Error()
+					log.Println(err.Error())
 				}
 			} else if stype == "back" {
 				result := MapToLuaTable2(s.L, s.Result)

src/web/templates/admin/site_rule_logicore.html

@@ -202,7 +202,7 @@ $(function () {
 					_tit="新增规则"
 					if(n=="newlua"){
 						_tit="新增脚本"
-						obj={"s_luascript":"--code脚本代码，doc数据源，block块对象，kvs抽取kv对象，返回kvs对象，kvs结构不可改变\nfunction main(code,doc,block,kvs)\n\t--自定义抽取\n\treturn kvs\nend","vid":"{{.vid}}","pid":"{{.pid}}","sid":"{{.sid}}","s_type":"1"}
+						obj={"s_luascript":"--code脚本代码，doc数据源，block块对象，kvs抽取kv对象，返回kvs对象，kvs结构不可改变\n--doc[\"contenthtml\"]为文章html内容\n--返回map对象，key为英文值，value为抽取内容\nlocal com=require \"lua.comm\"\nfunction main(code,doc,block,kvs)\n\t--自定义抽取\n\treturn kvs\nend","vid":"{{.vid}}","pid":"{{.pid}}","sid":"{{.sid}}","s_type":"1"}
 						tag = com.pushArry(tag,luatag)
 						tag = com.pushArry(tag,hiddentag)
 						islua=true

udpcreateindex/src/biddingindex.go

@@ -325,20 +325,22 @@ func DealInfo(obj, update *map[string]interface{}) {
 	var m [][]string
 	select {
 	case <-func() <-chan bool {
-		select {
-		case keypool <- true:
-			defer func() {
-				<-keypool
-			}()
-			ret, _ := client.Call("", mu.UUID(8), 4010, mu.SENDTO_TYPE_RAND_RECIVER, title, 1)
-			json.Unmarshal(ret, &m)
-		case <-time.After(5 * time.Millisecond):
-		}
 		ch := make(chan bool, 1)
-		ch <- true
+		go func(chan bool) {
+			select {
+			case keypool <- true:
+				defer func() {
+					<-keypool
+				}()
+				ret, _ := client.Call("", mu.UUID(8), 4010, mu.SENDTO_TYPE_RAND_RECIVER, title, 1)
+				json.Unmarshal(ret, &m)
+			case <-time.After(10 * time.Millisecond):
+			}
+			ch <- true
+		}(ch)
 		return ch
 	}():
-	case <-time.After(20 * time.Millisecond):
+	case <-time.After(40 * time.Millisecond):
 	}
 	arr := []string{}
 	keyword := []string{}

udpcreateindex/src/projectindex.go

@@ -37,13 +37,14 @@ func projectTask(data []byte, mapInfo map[string]interface{}) {
 	i := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
 		delete(tmp, "package")
+		delete(tmp, "winnerorder")
 		if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
 			tmp["budget"] = nil
 		}
 		if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
 			tmp["bidamount"] = nil
 		}
-		go IS.Add("project")
+		//go IS.Add("project")
 		arr[i] = tmp
 		n++
 		if i == savesizei-1 {