Merge branch 'dev3.3' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.3

dataprocess/src/task/task.go

@@ -18,6 +18,13 @@ type Task struct {
 	taskDel   xweb.Mapper `xweb:"/task/del"`   //删除任务
 	taskStope xweb.Mapper `xweb:"/task/start"` //启动任务
 	taskStop  xweb.Mapper `xweb:"/task/stop"`  //停止任务
+	//逻辑配置
+	taskPreProcessLogic xweb.Mapper `xweb:"/task/logic/preProcess"` //预处理逻辑
+	taskPreProcessSave  xweb.Mapper `xweb:"/task/preprocess/save"`  //预处理逻辑保存
+	askPreProcessDel    xweb.Mapper `xweb:"/task/preprocess/del"`   //预处理逻辑删除
+
+	taskMatchLogic xweb.Mapper `xweb:"/task/logic/match"` //匹配逻辑
+	taskfieldLogic xweb.Mapper `xweb:"/task/logic/field"` //字段逻辑
 }
 
 func (t *Task) TaskList() {
@@ -81,3 +88,45 @@ func (t *Task) TaskDel() {
 	})
 
 }
+
+func (t *Task) TaskPreProcessLogic() {
+	defer qu.Catch()
+	taskid := t.GetString("taskid")
+	if t.Method() == "POST" {
+		data, _ := Mgo.Find("task", `{"delete":false}`, nil, nil, false, -1, -1)
+		t.ServeJson(map[string]interface{}{
+			"data": data,
+		})
+	} else {
+		t.T["taskid"] = taskid
+		t.Render("com/task_preprocess_list.html", &t.T)
+	}
+}
+
+func (t *Task) TaskMatchLogic() {
+	defer qu.Catch()
+	taskid := t.GetString("taskid")
+	if t.Method() == "POST" {
+		data, _ := Mgo.Find("task", `{"delete":false}`, nil, nil, false, -1, -1)
+		t.ServeJson(map[string]interface{}{
+			"data": data,
+		})
+	} else {
+		t.T["taskid"] = taskid
+		t.Render("com/task_match_list.html", &t.T)
+	}
+}
+
+func (t *Task) TaskfieldLogic() {
+	defer qu.Catch()
+	taskid := t.GetString("taskid")
+	if t.Method() == "POST" {
+		data, _ := Mgo.Find("task", `{"delete":false}`, nil, nil, false, -1, -1)
+		t.ServeJson(map[string]interface{}{
+			"data": data,
+		})
+	} else {
+		t.T["taskid"] = taskid
+		t.Render("com/task_field_list.html", &t.T)
+	}
+}

dataprocess/src/web/templates/com/task_field_list.html

@@ -0,0 +1,167 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<div class="content-wrapper" id="showbtn">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增字段逻辑</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  	<li><a href="/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>	
+			<li><a class="active" href="/task/logic/field?taskid={{.T.taskid}}">字段逻辑</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+		                <th>逻辑名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+</div>
+{{include "com/dialog.html"}}
+{{include "com/footer.html"}}
+<script>
+menuActive("task/list")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : false,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/task/logic/field",
+			"type": "post",
+			"data":{"taskid":{{.T.taskid}}}
+		 },
+		"language": {
+            "url": "/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+            { "data": "s_taskname"},
+			{ "data": "l_comeintime",render:function(val){
+				var dt=new Date()
+				dt.setTime(parseInt(val)*1000)
+				return dt.format("yyyy-MM-dd hh:mm:ss")
+			}},
+			{ "data": "s_username"},
+			{ "data": "s_descript","width":"25%"},
+			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+					'<a class="btn btn-sm btn-danger opr" opr="del" onclick="del(\''+val+'\')">删除</a>'+
+					'</div>';
+				return  tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$("#showbtn").on('click','a.opr',function(){
+			var n=$(this).attr("opr");
+			var taskid=$(this).attr("taskid");
+			var htmlObj={},obj,tag=[],bts=[];
+			var _tit="";
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();	
+			case "new":
+				/*表单*/
+				addtask=[
+					{label:"任务名称",s_label:"s_taskname",placeholder:"数据清洗",must:true},
+					{label:"源库连接",s_label:"s_mgoaddr",must:true},
+					{label:"源数据库",s_label:"s_mgodb",must:true},
+					{label:"源表",s_label:"s_mgocoll",must:true},
+					{label:"保存表",s_label:"s_mgosavecoll",placeholder:"127.0.0.1:27080/dataprocess/bidding(数据库地址/数据库/表)",must:true},
+					{label:"描述",s_label:"s_descript",type:"tpl_text"},
+					//{label:"使用版本",s_label:"s_version",type:"tpl_list_local",must:true,url:"/admin/task/getversion"},
+					{label:"并发数量",s_label:"i_process",placeholder:"5",must:true},
+					{label:"起始id",s_label:"s_extlastid",must:true},
+					{s_label:"_id",type:"tpl_hidden"},
+				];
+				/*testtask=[
+					{label:"起始id",s_label:"s_startid",must:true},
+					{label:"数据数量",s_label:"s_datanum",placeholder:"5",must:true}
+				];*/
+				/*按钮*/
+				//新增保存按钮
+				addtaskbtn=[
+					{label:"保存",class:"btn-primary",
+						fun:function(){
+							var obj={}
+							var bcon=true
+							$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+								var val=$(el).val();
+								if(el.id!="_id"&&$(el).attr("must")&&!val){
+									bcon=false
+									return false
+								}
+								obj[el.id]=$(el).val()
+							})
+							if (bcon){								
+								$.post("/task/save",obj,function(data){
+									if(data&&data.rep){
+										//window.location.href="/task/list"
+										$('#myModal').modal('hide');
+										ttable.ajax.reload();
+									}else{
+										alert(data.msg)
+									}
+								},'json')
+							}else{
+								alert("红色标签的表单不能为空！")
+							}
+						}
+					}
+				];
+				if(n == "new"){
+					_tit="新增抽取任务";
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}else if(n == "edit"){
+					_tit="编辑-"+obj.s_taskname;
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+	
+})
+function del(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+</script>

dataprocess/src/web/templates/com/task_list.html

@@ -10,7 +10,7 @@
 			<small><a class="btn btn-primary opr" opr="new">新增任务</a></small>
 		</h1>
 		<ol class="breadcrumb">
-		  <li><a href="/admin/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>		  
+		  <li><a href="/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>		  
 		</ol>
     </section>
   <!-- Main content -->
@@ -22,7 +22,7 @@
 		            <table id="dataTable" class="table table-bordered table-hover">
 		              <thead>
 		              <tr>
-		                <th>任务名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>运行状态</th> <th>操作</th>
+		                <th>任务名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>运行状态</th> <th>逻辑配置</th> <th>操作</th> <th>流程图</th>
 		              </tr>
 		              </thead>
 		            </table>
@@ -37,7 +37,7 @@
 {{include "com/dialog.html"}}
 {{include "com/footer.html"}}
 <script>
-menuActive("list")
+menuActive("/task/list")
 $(function () {
 	ttable=$('#dataTable').DataTable({
 		"paging"      : false,
@@ -62,7 +62,7 @@ $(function () {
 				return dt.format("yyyy-MM-dd hh:mm:ss")
 			}},
 			{ "data": "s_username"},
-			{ "data": "s_descript","width":"25%"},
+			{ "data": "s_descript","width":"20%"},
 			{ "data": "i_runstate",render:function(val){
 				if(val==1){
 					return "<i class='fa fa-fw fa-circle text-green'></i>运行中"
@@ -70,14 +70,28 @@ $(function () {
 					return "<i class='fa fa-fw fa-circle text-danger'></i>未启动"
 				}
 			}},
-			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+			{ "data":"_id","width":"17%",render:function(val){
 				tmp = '<div>'+
-					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
-					'<a class="btn btn-sm btn-success opr" opr="start" onclick="start(\''+row._id+'\')">启动</a>　'+
-					'<a class="btn btn-sm btn-info opr" opr="stop" onclick="stop(\''+row._id+'\')">停止</a>　'+
+					'<a class="btn btn-sm btn-primary" href="/task/logic/preProcess?taskid='+val+'">预处理</a>  '+
+					'<a class="btn btn-sm btn-info" href="/task/logic/match?taskid='+val+'">匹配</a>  '+
+					'<a class="btn btn-sm btn-warning" href="/task/logic/field?taskid='+val+'">字段</a>'+
+					'</div>';
+				return  tmp
+			}},
+			{ "data":"_id","width":"20%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>  '+
+					'<a class="btn btn-sm btn-success opr" opr="start" onclick="start(\''+row._id+'\')">启动</a>  '+
+					'<a class="btn btn-sm btn-info opr" opr="stop" onclick="stop(\''+row._id+'\')">停止</a>  '+
 					'<a class="btn btn-sm btn-danger opr" opr="del" onclick="del(\''+val+'\')">删除</a>'+
 					'</div>';
 				return  tmp
+			}},
+			{ "data":"_id",render:function(val){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary" href="#">查看</a>  '+
+					'</div>';
+				return  tmp
 			}}
        	]
 	});
@@ -99,58 +113,8 @@ $(function () {
 					{label:"源表",s_label:"s_mgocoll",must:true},
 					{label:"保存表",s_label:"s_mgosavecoll",placeholder:"127.0.0.1:27080/dataprocess/bidding(数据库地址/数据库/表)",must:true},
 					{label:"描述",s_label:"s_descript",type:"tpl_text"},
-					/*
-					{label:"是否追踪",s_label:"i_track",type:"tpl_list_local",must:true,list:[{"s_name":"是","_id":1},{"s_name":"否","_id":0}],default:0,fun:function(){
-						var to=$("#i_track")					
-						to.val($(this).attr("_id"))
-						$("#s_show",to.closest("div")).val($(this).text())
-						//追踪表样式
-						$("#s_trackcoll").attr("must",$(this).attr("_id")==1)
-						$("#s_trackcoll").closest("div.row").find("label").css("color",$(this).attr("_id")==1?"red":"")
-					}},
-					{label:"追踪记录表",s_label:"s_trackcoll",must:function(){
-						return  obj&&obj.i_track
-					}()},
-					{label:"是否统计",s_label:"i_count",type:"tpl_list_local",must:true,list:[{"s_name":"是","_id":1},{"s_name":"否","_id":0}],default:0},
-					*/
 					//{label:"使用版本",s_label:"s_version",type:"tpl_list_local",must:true,url:"/admin/task/getversion"},
 					{label:"并发数量",s_label:"i_process",placeholder:"5",must:true},
-					{label:"预处理逻辑",s_label:"s_preprocesslogic",type:"tpl_list_local",url:""},
-					{label:"匹配逻辑",s_label:"s_class",type:"tpl_list_ajax",url:"",fun:function(){
-						var ids=$("#s_class").data("ids")
-						var parentDiv=$("#s_class").closest("div")
-						ids=ids||{}
-						var tid=$(this).attr("_id")
-						if(!ids[tid]){
-							ids[tid]=true
-							$("#s_class").data("ids",ids)
-							var tpl1=$('<div class="alert alert-dismissible alert-success" style="min-width:50px;max-width:250px;font-size:10px;padding:3px;margin:5px;display:inline-block"><button type="button" class="close" data-dismiss="alert" style="right:0px;">&times;</button><span></span></div>')
-							tpl1.find("span").text($(this).text())							
-							tpl1.attr("tid",tid)
-							tpl1.find("button").click(function(){
-								var ttid=$(this).closest(".alert").attr("tid");
-								var iids=$("#s_class").data("ids")
-								delete iids[ttid]
-								$("#s_class").val(function(){
-									var strid=[]
-									for(var k in iids){
-										strid.push(k)
-									}
-									return strid.join(",")
-								}())
-							})
-							$("#s_show",parentDiv).append(tpl1)
-							$("#s_class").val(function(){
-								var strid=[]
-								for(var k in ids){
-									strid.push(k)
-								}
-								return strid.join(",")
-							}())
-						}
-					}},
-					//{label:"匹配逻辑",s_label:"s_matchlogic",type:"tpl_list_local",url:""},
-					{label:"字段",s_label:"s_field",placeholder:"title"},
 					{label:"起始id",s_label:"s_extlastid",must:true},
 					{s_label:"_id",type:"tpl_hidden"},
 				];

dataprocess/src/web/templates/com/task_match_list.html

@@ -0,0 +1,167 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<div class="content-wrapper" id="showbtn">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增匹配逻辑</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  	<li><a href="/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>	
+			<li><a class="active" href="/task/logic/match?taskid={{.T.taskid}}">匹配逻辑</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+		                <th>逻辑名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+</div>
+{{include "com/dialog.html"}}
+{{include "com/footer.html"}}
+<script>
+menuActive("task/list")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : false,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/task/logic/match",
+			"type": "post",
+			"data":{"taskid":{{.T.taskid}}}
+		 },
+		"language": {
+            "url": "/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+            { "data": "s_taskname"},
+			{ "data": "l_comeintime",render:function(val){
+				var dt=new Date()
+				dt.setTime(parseInt(val)*1000)
+				return dt.format("yyyy-MM-dd hh:mm:ss")
+			}},
+			{ "data": "s_username"},
+			{ "data": "s_descript","width":"25%"},
+			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+					'<a class="btn btn-sm btn-danger opr" opr="del" onclick="del(\''+val+'\')">删除</a>'+
+					'</div>';
+				return  tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$("#showbtn").on('click','a.opr',function(){
+			var n=$(this).attr("opr");
+			var taskid=$(this).attr("taskid");
+			var htmlObj={},obj,tag=[],bts=[];
+			var _tit="";
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();	
+			case "new":
+				/*表单*/
+				addtask=[
+					{label:"任务名称",s_label:"s_taskname",placeholder:"数据清洗",must:true},
+					{label:"源库连接",s_label:"s_mgoaddr",must:true},
+					{label:"源数据库",s_label:"s_mgodb",must:true},
+					{label:"源表",s_label:"s_mgocoll",must:true},
+					{label:"保存表",s_label:"s_mgosavecoll",placeholder:"127.0.0.1:27080/dataprocess/bidding(数据库地址/数据库/表)",must:true},
+					{label:"描述",s_label:"s_descript",type:"tpl_text"},
+					//{label:"使用版本",s_label:"s_version",type:"tpl_list_local",must:true,url:"/admin/task/getversion"},
+					{label:"并发数量",s_label:"i_process",placeholder:"5",must:true},
+					{label:"起始id",s_label:"s_extlastid",must:true},
+					{s_label:"_id",type:"tpl_hidden"},
+				];
+				/*testtask=[
+					{label:"起始id",s_label:"s_startid",must:true},
+					{label:"数据数量",s_label:"s_datanum",placeholder:"5",must:true}
+				];*/
+				/*按钮*/
+				//新增保存按钮
+				addtaskbtn=[
+					{label:"保存",class:"btn-primary",
+						fun:function(){
+							var obj={}
+							var bcon=true
+							$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+								var val=$(el).val();
+								if(el.id!="_id"&&$(el).attr("must")&&!val){
+									bcon=false
+									return false
+								}
+								obj[el.id]=$(el).val()
+							})
+							if (bcon){								
+								$.post("/task/save",obj,function(data){
+									if(data&&data.rep){
+										//window.location.href="/task/list"
+										$('#myModal').modal('hide');
+										ttable.ajax.reload();
+									}else{
+										alert(data.msg)
+									}
+								},'json')
+							}else{
+								alert("红色标签的表单不能为空！")
+							}
+						}
+					}
+				];
+				if(n == "new"){
+					_tit="新增抽取任务";
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}else if(n == "edit"){
+					_tit="编辑-"+obj.s_taskname;
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+	
+})
+function del(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+</script>

dataprocess/src/web/templates/com/task_preprocess_list.html

@@ -0,0 +1,156 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "com/menu.html"}}
+
+<div class="content-wrapper" id="showbtn">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary opr" opr="new">新增预处理逻辑</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  	<li><a href="/task/list"><i class="fa fa-dashboard"></i> 任务管理</a></li>	
+			<li><a class="active" href="/task/logic/preProcess?taskid={{.T.taskid}}">预处理逻辑</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+		                <th>逻辑名称</th> <th>创建时间</th> <th>创建人</th> <th>描述</th><th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+</div>
+{{include "com/dialog.html"}}
+{{include "com/footer.html"}}
+<script>
+menuActive("task/list")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : false,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/task/logic/preProcess",
+			"type": "post",
+			"data":{"taskid":{{.T.taskid}}}
+		 },
+		"language": {
+            "url": "/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+            { "data": "s_taskname"},
+			{ "data": "l_comeintime",render:function(val){
+				var dt=new Date()
+				dt.setTime(parseInt(val)*1000)
+				return dt.format("yyyy-MM-dd hh:mm:ss")
+			}},
+			{ "data": "s_username"},
+			{ "data": "s_descript","width":"25%"},
+			{ "data":"_id","width":"25%",render:function(val,a,row,pos){
+				tmp = '<div>'+
+					'<a class="btn btn-sm btn-primary opr" opr="edit" row="'+pos.row+'" >编辑</a>　'+
+					'<a class="btn btn-sm btn-danger opr" opr="del" onclick="del(\''+val+'\')">删除</a>'+
+					'</div>';
+				return  tmp
+			}}
+       	]
+	});
+	ttable.on('init.dt', function () {
+		$("#showbtn").on('click','a.opr',function(){
+			var n=$(this).attr("opr");
+			var taskid=$(this).attr("taskid");
+			var htmlObj={},obj,tag=[],bts=[];
+			var _tit="";
+			switch(n){
+			case "edit":			
+				obj=ttable.row($(this).closest("tr")).data();	
+			case "new":
+				/*表单*/
+				addtask=[
+					{label:"任务名称",s_label:"s_taskname",placeholder:"预处理逻辑",must:true},
+					{label:"描述",s_label:"s_descript",type:"tpl_text"},
+					{s_label:"_id",type:"tpl_hidden"},
+				];
+				/*按钮*/
+				//新增保存按钮
+				addtaskbtn=[
+					{label:"保存",class:"btn-primary",
+						fun:function(){
+							var obj={}
+							var bcon=true
+							$("#_con").find("input[id!=s_show],textarea").each(function(i,el){
+								var val=$(el).val();
+								if(el.id!="_id"&&$(el).attr("must")&&!val){
+									bcon=false
+									return false
+								}
+								obj[el.id]=$(el).val()
+							})
+							if (bcon){								
+								$.post("/task/preprocess/save",obj,function(data){
+									if(data&&data.rep){
+										//window.location.href="/task/list"
+										$('#myModal').modal('hide');
+										ttable.ajax.reload();
+									}else{
+										alert(data.msg)
+									}
+								},'json')
+							}else{
+								alert("红色标签的表单不能为空！")
+							}
+						}
+					}
+				];
+				if(n == "new"){
+					_tit="新增抽取任务";
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}else if(n == "edit"){
+					_tit="编辑-"+obj.s_taskname;
+					tag = com.pushArry(tag,addtask);
+					bts = com.pushArry(bts,addtaskbtn);
+				}
+				htmlObj={
+					title:_tit,
+					tag:tag,
+					bts:bts
+				}
+			OpenDialog(htmlObj,obj)
+			break;
+			}
+		});
+	})
+	
+})
+function del(_id){
+	showConfirm("确定启动？", function() {
+		$.ajax({
+			url:"/task/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				ttable.ajax.reload();
+			}
+		})
+	});
+}
+</script>

udpfilterdup/src/config.json

@@ -1,13 +1,13 @@
 {
-    "udpport": ":1112",
-    "dupdays": 2,
+    "udpport": ":1485",
+    "dupdays": 5,
     "mongodb": {
         "addr": "192.168.3.207:27082",
-        "pool": 5,
+        "pool": 15,
         "db": "extract_kf",
         "extract": "bidding_20190910_01",
         "extract_copy": "bidding_20190910_01_copy",
-        "bidding": "bidding"
+        "bidding": "bidding_126"
     },
     "jkmail": {
         "to": "renzheng@topnet.net.cn",
@@ -27,7 +27,13 @@
             "memo": "创建招标数据索引"
         }
     ],
+    "site": [
+        {
+            "addr": "信息网"
+        },
+        {
+            "addr": "招标网"
+        }
+    ],
     "specialwords": "[(（]?[0-9一二三四五六七八九十零１２３４５６７８９再][)）]?[子分]?[次批标包]|重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研"
-}
-
-
+}

udpfilterdup/src/datamap.go

@@ -54,7 +54,6 @@ func NewDatamap(days int, lastid string) *datamap {
 	if lastid == "" {
 		return dm
 	}
-
 	//初始化加载数据
 	sess := mgo.GetMgoConn()
 	defer mgo.DestoryMongoConn(sess)
@@ -62,10 +61,9 @@ func NewDatamap(days int, lastid string) *datamap {
 	now1 := int64(0)
 	n, continuSum := 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
-		//|| qutil.ObjToString(tmp["subtype"]) == "变更"  //变更的数据打开
+		//|| qutil.ObjToString(tmp["subtype"]) == "变更"
 		if qutil.IntAll(tmp["repeat"]) == 1  {
 			continuSum++
-
 		} else {
 			cm := tmp["comeintime"] //时间单位
 			//cm := tmp["publishtime"]
@@ -84,7 +82,6 @@ func NewDatamap(days int, lastid string) *datamap {
 				data := dm.data[k]
 				if data == nil {
 					data = []*Info{}
-					//log.Println(k)
 				}
 				data = append(data, info)
 				dm.data[k] = data
@@ -123,16 +120,11 @@ func NewInfo(tmp map[string]interface{}) *Info {
 	//info.winner = qutil.ObjToString(tmp["winner"])
 	info.publishtime = qutil.Int64All(tmp["publishtime"])
 
-
-
-
 	info.bidopentime  = qutil.Int64All(tmp["bidopentime"])
 	info.agencyaddr	 = qutil.ObjToString(tmp["agencyaddr"])
 	info.detail		= qutil.ObjToString(tmp["detail"])
 	info.site	 = qutil.ObjToString(tmp["site"])
 
-
-
 	return info
 }
 
@@ -213,7 +205,7 @@ L:
 								break L
 							}
 						}
-					}else {//非变更数据判重处理
+					}else {
 						n:=0 //三要素
 						m:=0 //二要素
 						x:=0 //四要素
@@ -241,8 +233,16 @@ L:
 						c_3 :=conditionTAB(v.title,info.title,v.buyer,info.buyer) //标题+采购单位
 
 
-						//同站点判断
-						if info.site != "" && v.site == info.site {
+						site_b := false
+						if info.site != "" {
+							for _,v := range siteArr {
+								if info.site==v["addr"] {
+									site_b=true
+									break
+								}
+							}
+						}
+						if info.site != "" && v.site == info.site&&site_b {
 							if n>1||c_1||c_2 {
 								b = true
 								id = v.id
@@ -343,11 +343,6 @@ L:
 	}
 
 
-
-
-
-
-
 	//往预存数据 d 添加
 	if !b {
 		ct, _ := strconv.ParseInt(info.id[:8], 16, 64)

udpfilterdup/src/main.go

@@ -30,24 +30,25 @@ var (
 	dupdays      = 5                      //初始化判重范围
 	DM           *datamap                 //判重数据
 	FilterRegexp = regexp.MustCompile("^_$")
-	lastid       = "5d767728a5cb26b9b7748868"  //起始数据1条
+	lastid       = "5d767728a5cb26b9b7748868"
 
 
 	FilterRegTitle = regexp.MustCompile("[0-9一二三四五六七八九十零１２３４５６７８９](次|包|标段)")
-
+	inV_n int   //无效数据数量
+	siteArr     []map[string]interface{} //站点
 	//56404035af5374672e00059c
 	//5d767728a5cb26b9b7748868
-
-	inV_n int
 )
 
 func init() {
 	//flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
-	//flag.Parse()// 类似于命令台 scanf
-	////172.17.145.163:27080
+	//flag.Parse()
+	//172.17.145.163:27080
 	util.ReadConfig(&Sysconfig)
 	nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
+	siteArr = util.ObjArrToMapArr(Sysconfig["site"].([]interface{}))
 	mconf = Sysconfig["mongodb"].(map[string]interface{})
+
 	mgo = &mongodb.MongodbSim{
 		MongodbAddr: mconf["addr"].(string),
 		DbName:      mconf["db"].(string),
@@ -60,7 +61,6 @@ func init() {
 	mgo.InitPool()
 
 
-
 	//测试临时注释
 	dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
 	//加载数据
@@ -72,35 +72,25 @@ func init() {
 }
 
 //新增一个方法 判断
-func main22()  {
+func mainTest()  {
 
 	sess := mgo.GetMgoConn()
 	defer mgo.DestoryMongoConn(sess)
-
 	res, _ := mgo.Find(extract, nil, nil, nil, false, -1, -1)
 	res_copy, _ := mgo.Find(extract_copy, nil, nil, nil, false, -1, -1)
-
-	//fmt.Println(len(*res),len(*res_copy))
-
 	m1 :=map[string]int{} //新版本
 	m2 :=map[string]int{} //老版本
 	for _,v:=range *res{
 		m1[(v["_id"].(bson.ObjectId).Hex())]= util.IntAll(v["repeat"])
 	}
-
 	for _,v:=range *res_copy{
 		m2[(v["_id"].(bson.ObjectId).Hex())]= util.IntAll(v["repeat"])
 	}
 	fmt.Println(len(m1),len(m2))
-
-
 	n1:=0
 	n2:=0
-
-
 	var arr1 []string
 	var arr2 []string
-
 	for k,v:=range m2{
 		if m1[k]!=v{
 			if m1[k]==0{
@@ -113,7 +103,6 @@ func main22()  {
 			}
 		}
 	}
-
 	for _,v:=range arr1 {
 		log.Println(v)
 	}
@@ -128,8 +117,6 @@ func main22()  {
 
 	log.Println("0:1 对应数量",n1)
 	log.Println("1:0 对应数量",n2)
-
-
 }
 
 
@@ -197,9 +184,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
 
 	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
 
-		//fmt.Println(util.ObjToString(tmp["subtype"]))
 		if util.ObjToString(tmp["subtype"]) == "变更" {
-			//fmt.Println("变更信息")
 			//continue
 		}
 
@@ -217,7 +202,6 @@ func task(data []byte, mapInfo map[string]interface{}) {
 
 			//是否为无效数据
 			if invalidData(info.buyer,info.projectname,info.projectcode) {
-				//
 				inV_n++
 				mapLock.Lock()
 				updateExtract = append(updateExtract, []map[string]interface{}{
@@ -233,23 +217,14 @@ func task(data []byte, mapInfo map[string]interface{}) {
 
 				if len(updateExtract) > 500 {
 					mgo.UpdateBulk(extract, updateExtract...)
-					//mgo.UpdateBulk(bidding, updateBidding...)
-					//updateExtract, updateBidding = [][]map[string]interface{}{}, [][]map[string]interface{}{}
 					updateExtract = [][]map[string]interface{}{}
 				}
 				mapLock.Unlock()
-
 			}else  {
 				b, id := DM.check(info)
 				if b { //有重复,生成更新语句，更新抽取和更新招标
 
-					//重复数据，处理方式   合并操作--------------
-					//重复数据，处理方式   合并操作--------------
-					//重复数据，处理方式   合并操作--------------
-
-
 					repeateN++
-					//log.Println("变更数据重复",tmp["_id"])
 					mapLock.Lock()
 					updateExtract = append(updateExtract, []map[string]interface{}{
 						map[string]interface{}{
@@ -265,11 +240,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
 
 					if len(updateExtract) > 500 {
 						mgo.UpdateBulk(extract, updateExtract...)
-						//mgo.UpdateBulk(bidding, updateBidding...)
-						//updateExtract, updateBidding = [][]map[string]interface{}{}, [][]map[string]interface{}{}
 						updateExtract = [][]map[string]interface{}{}
-
-
 					}
 					mapLock.Unlock()
 				} else {

udps/main.go

@@ -35,7 +35,7 @@ func main() {
 	flag.StringVar(&startDate, "start", "", "开始日期2006-01-02")
 	flag.StringVar(&endDate, "end", "", "结束日期2006-01-02")
 	flag.StringVar(&ip, "ip", "127.0.0.1", "ip")
-	flag.StringVar(&port, "port", "1112", "dup端口")
+	flag.StringVar(&port, "port", "1485", "dup端口")
 	flag.StringVar(&stype, "stype", "", "stype")
 	flag.Parse()
 	var startid, endid bson.ObjectId