抽取导出功能

src/jy/admin/admin.go

@@ -63,7 +63,7 @@ func init() {
 	})
 	//获取字段列表
 	Admin.POST("/getversions", func(c *gin.Context) {
-		list, b := Mgo.Find("version", `{}`, `{"_id":-1}`, `{"version":1}`, false, -1, -1)
+		list, b := Mgo.Find("version", `{"delete":false}`, `{"_id":-1}`, `{"version":1}`, false, -1, -1)
 		if b && list != nil {
 			for _, v := range *list {
 				v["_id"] = qu.BsonIdToSId(v["_id"])

src/jy/admin/task/task.go

@@ -4,9 +4,9 @@ package task
 
 import (
 	. "jy/admin"
-
 	"jy/extract"
 	. "jy/mongodbutil"
+	"net/http"
 	qu "qfw/util"
 	"time"
 
@@ -111,4 +111,61 @@ func init() {
 		data, _ := Mgo.Find("tasktestlog", nil, `{"_id":-1}`, nil, false, -1, -1)
 		c.JSON(200, gin.H{"data": data})
 	})
+
+	//抽取导出相关
+	Admin.GET("/task/export", func(c *gin.Context) {
+		list, b := Mgo.Find("version", `{"delete":false}`, `{"_id":-1}`, `{"version":1}`, false, -1, -1)
+		if b && list != nil {
+			for _, v := range *list {
+				v["_id"] = qu.BsonIdToSId(v["_id"])
+				v["s_name"] = v["version"]
+			}
+		}
+		c.HTML(
+			http.StatusOK, "task_export.html",
+			gin.H{"vlist": list},
+		)
+	})
+	Admin.POST("/task/export/data", func(c *gin.Context) {
+		data, _ := Mgo.Find("task_export", `{}`, `{"_id":-1}`, nil, false, -1, -1)
+		c.JSON(200, gin.H{"data": data})
+	})
+	Admin.POST("task/export/save", func(c *gin.Context) {
+		_id, _ := c.GetPostForm("_id")
+		name, _ := c.GetPostForm("name")
+		v1id, _ := c.GetPostForm("v1")
+		v2id, _ := c.GetPostForm("v2")
+		v1name, _ := c.GetPostForm("v1name")
+		v2name, _ := c.GetPostForm("v2name")
+		dbaddr, _ := c.GetPostForm("dbaddr")
+		dbname, _ := c.GetPostForm("dbname")
+		query, _ := c.GetPostForm("query")
+		table, _ := c.GetPostForm("table")
+		limit, _ := c.GetPostForm("limit")
+		tmpfields, _ := c.GetPostFormMap("fields")
+		fields := []string{}
+		for _, v := range tmpfields {
+			fields = append(fields, v)
+		}
+		data := map[string]interface{}{
+			"name": name,
+			"v1id": v1id, "v1name": v1name, "v2id": v2id, "v2name": v2name,
+			"dbaddr": dbaddr, "dbname": dbname, "table": table,
+			"query": query, "limit": limit, "fields": fields,
+		}
+		if _id != "" {
+			Mgo.UpdateById("task_export", _id, map[string]interface{}{"$set": data})
+		} else {
+			data["state"] = 0 //0待执行，1已完成
+			data["intime"] = time.Now().Unix()
+			Mgo.Save("task_export", data)
+		}
+		c.JSON(200, gin.H{"rep": true})
+	})
+	Admin.POST("/task/export/del", func(c *gin.Context) {
+		_id, _ := c.GetPostForm("_id")
+		b := Mgo.Del("task_export", `{"_id":"`+_id+`"}`)
+		c.JSON(200, gin.H{"rep": b})
+	})
+
 }

src/jy/admin/version.go

@@ -38,7 +38,9 @@ func init() {
 			data["s_username"] = s_username
 			data["delete"] = false
 			Mgo.Save("version", data)
-			copyComRules(version, s_pversionid, s_username)
+			if s_pversionid != "" {
+				copyComRules(version, s_pversionid, s_username)
+			}
 			c.JSON(200, gin.H{"rep": true})
 		}
 	})

src/jy/extract/exportask.go

@@ -0,0 +1,134 @@
+// exportask
+package extract
+
+import (
+	"fmt"
+	db "jy/mongodbutil"
+	"log"
+	qu "qfw/util"
+	"time"
+
+	"github.com/tealeg/xlsx"
+)
+
+//执行导出定时任务
+func Export() {
+	tk, _ := db.Mgo.Find("task_export", `{"state":0}`, nil, nil, false, -1, -1)
+	for _, t := range *tk {
+		extractAndExport("v1", t)
+		extractAndExport("v2", t)
+		//生成excel
+		filename := createFile(t)
+		if filename != "" {
+			db.Mgo.UpdateById("task_export", qu.BsonIdToSId(t["_id"]),
+				map[string]interface{}{
+					"$set": map[string]interface{}{
+						"state":    1,
+						"filename": filename,
+					},
+				})
+		}
+	}
+	time.AfterFunc(30*time.Minute, Export)
+}
+
+func extractAndExport(v string, t map[string]interface{}) {
+	db.Mgo.Del("task_export_result_"+v, `{}`) //清除历史数据
+	e := &ExtractTask{}
+	e.IsRun = true
+	e.TaskInfo = &TaskInfo{
+		Name:        t["name"].(string),
+		Version:     t[v+"name"].(string),
+		VersionId:   t[v+"id"].(string),
+		FromDbAddr:  t["dbaddr"].(string),
+		FromDB:      t["dbname"].(string),
+		FromColl:    t["table"].(string),
+		TestColl:    "task_export_result_" + v,
+		TrackColl:   "task_export_tract_" + v,
+		IsEtxLog:    false,
+		ProcessPool: make(chan bool, 5),
+	}
+	e.TaskInfo.DB = db.MgoFactory(1, 3, 120, fmt.Sprint(t["dbaddr"]), fmt.Sprint(t["dbname"]))
+	e.InitRulePres()
+	e.InitRuleBacks()
+	e.InitRuleCore()
+	e.InitTag()
+	e.InitClearFn()
+	query := t["query"]
+	limit := qu.IntAll(t["limit"])
+	list, _ := e.TaskInfo.DB.Find(e.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
+	for _, v := range *list {
+		j := PreInfo(v)
+		e.TaskInfo.ProcessPool <- true
+		go e.ExtractProcess(j)
+	}
+}
+
+func createFile(info map[string]interface{}) string {
+	filename := ""
+	if fields, ok := info["fields"].([]interface{}); ok {
+		qfield := ""
+		for k, v := range fields {
+			if k == 0 {
+				qfield += `"` + fmt.Sprint(v) + `":1`
+			} else {
+				qfield += `,"` + fmt.Sprint(v) + `":1`
+			}
+		}
+		list_1, _ := db.Mgo.Find("task_export_result_v1", `{}`, nil, "{"+qfield+"}", false, -1, -1)
+		result := map[string]map[string]interface{}{}
+		for _, v := range *list_1 {
+			id := qu.BsonIdToSId(v["_id"])
+			tmp := map[string]interface{}{}
+			for _, f := range fields {
+				field := fmt.Sprint(f)
+				tmp["v1_"+field] = qu.ObjToString(v[field])
+			}
+			result[id] = tmp
+		}
+		list_2, _ := db.Mgo.Find("task_export_result_v2", `{}`, nil, "{"+qfield+"}", false, -1, -1)
+		for _, v := range *list_2 {
+			id := qu.BsonIdToSId(v["_id"])
+			tmp := map[string]interface{}{}
+			if result[id] != nil {
+				tmp = result[id]
+			}
+			for _, f := range fields {
+				field := fmt.Sprint(f)
+				tmp["v2_"+field] = qu.ObjToString(v[field])
+			}
+			result[id] = tmp
+		}
+		sheetmaps := map[string][]map[string]interface{}{}
+		for _, f := range fields {
+			field := fmt.Sprint(f)
+			tmps := []map[string]interface{}{}
+			for _, v := range result {
+				tmp := map[string]interface{}{
+					"v1": v["v1_"+field],
+					"v2": v["v2_"+field],
+				}
+				tmps = append(tmps, tmp)
+			}
+			sheetmaps[field] = tmps
+		}
+		fx := xlsx.NewFile()
+		for k, v := range sheetmaps {
+			sheet := xlsx.Sheet{}
+			title := sheet.AddRow()
+			title.AddCell().SetString(fmt.Sprint(info["v1name"]))
+			title.AddCell().SetString(fmt.Sprint(info["v2name"]))
+			for _, d := range v {
+				row := sheet.AddRow()
+				row.AddCell().SetString(fmt.Sprint(d["v1"]))
+				row.AddCell().SetString(fmt.Sprint(d["v2"]))
+			}
+			fx.AppendSheet(sheet, k)
+		}
+		t := time.Now()
+		filename = fmt.Sprint(info["name"]) + "_" + t.Format("20060102150405") + ".xlsx"
+		fx.Save("./web/res/down/" + filename)
+		log.Println("任务完成", filename)
+	}
+	return filename
+}

src/jy/extract/extract.go

@@ -238,7 +238,7 @@ func (e *ExtractTask) ExtractProcess(j *ju.Job) {
 		//log.Println("抽取结果", j.Title, j.SourceMid, string(bs))
 
 		//抽取省份城市县
-		fmt.Println("-----------", j.Province, j.City, j.Title)
+		//fmt.Println("-----------", j.Province, j.City, j.Title)
 		//ExtractPC()
 		//分析抽取结果并保存 todo
 		AnalysisSaveResult(j.Data, j.Result, e.TaskInfo)

src/main.go

@@ -4,6 +4,7 @@ import (
 	_ "jy/admin"
 	_ "jy/admin/buyermanager"
 	_ "jy/admin/task"
+	"jy/extract"
 	_ "jy/front"
 	. "jy/router"
 	"jy/util"
@@ -18,6 +19,7 @@ func init() {
 }
 
 func main() {
+	go extract.Export()
 	go Router.Run(":" + qu.ObjToString(util.Config["port"]))
 	go log.Println("启动..", qu.ObjToString(util.Config["port"]))
 	time.Sleep(99999 * time.Hour)

src/web/res/doublebox/css/hdw.css

@@ -1,6 +1,6 @@
 
 #selectclear select { 
-	width:154px; 
+	width:190px; 
 	height:167px;
 	padding:5px;
 }
@@ -11,7 +11,7 @@
 #selectclear .move{
 	display: flex;
     flex-direction: column;
-	margin: 20px 60px
+	margin: 20px 25px
 }
 #selectclear .move button{
 	margin: 1px 0px;

src/web/templates/admin/com_memu.html

@@ -4,8 +4,18 @@
       <ul class="sidebar-menu" data-widget="tree">
         <li class="header">HEADER</li>
         <!-- Optionally, you can add icons to the links -->
-		<li><a href="/admin/task"><i class="fa fa-clock-o"></i> <span>任务管理</span></a></li>
-	    <li><a href="/admin/version"><i class="fa fa-navicon"></i><span>版本管理</span></a></li>
+		<li class="treeview">
+          	<a href="#"><i class="fa fa-clock-o"></i> <span>任务管理</span>
+            <span class="pull-right-container">
+                <i class="fa fa-angle-left pull-right"></i>
+            </span>
+          	</a>
+          	<ul class="treeview-menu">
+	            <li><a href="/admin/task"><i class="fa fa-link"></i>抽取任务</a></li>
+				<li><a href="/admin/task/export"><i class="fa fa-link"></i>导出任务</a></li>
+			</ul>
+        </li>
+		<li><a href="/admin/version"><i class="fa fa-navicon"></i><span>版本管理</span></a></li>
 		<li class="treeview">
           	<a href="#"><i class="fa fa-laptop"></i> <span>中标单位库</span>
             <span class="pull-right-container">

src/web/templates/admin/task_export.html

@@ -0,0 +1,305 @@
+{{template "inc"}}
+<!-- Main Header -->
+{{template "header"}}
+<!-- Left side column. 权限菜单 -->
+{{template "memu"}}
+	<link href="/res/doublebox/css/hdw.css" rel="stylesheet" type="text/css">
+	<script src="/res/doublebox/js/hdw.js"></script>
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper">
+	<section class="content-header">
+		<h1>
+			<small><a class="btn btn-primary" onclick="editorexport()">创建导出任务</a></small>
+		</h1>
+		<ol class="breadcrumb">
+		  <li><a href="/admin/task/export"><i class="fa fa-dashboard"></i>导出任务</a></li>
+		</ol>
+    </section>
+  <!-- Main content -->
+  <section class="content">
+      <div class="row">
+	      <div class="col-xs-12">
+	        <div class="box">
+		        <div class="box-body">
+		            <table id="dataTable" class="table table-bordered table-hover">
+		              <thead>
+		              <tr>
+		                <th>任务名称</th>
+						<th>导出属性</th>
+						<th>导出条件</th>
+						<th>导出数量</th>
+						<th>是否完成</th>
+						<th>操作</th>
+		              </tr>
+		              </thead>
+		            </table>
+		        </div>
+	          <!-- /.box-body -->
+	        </div>
+        <!-- /.box -->
+		</div>
+	</div>
+  </section>
+</div>
+<div class="modal fade" id="modal-info-export">
+  	<div class="modal-dialog">
+	    <form id="dataform" class="form-horizontal" role="form">
+		<div class="modal-content">
+		    <div class="modal-header">
+		        <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+		          <span aria-hidden="true">&times;</span></button>
+		        <h4 class="modal-title">导出</h4>
+		    </div>
+		    <div class="modal-body">
+				<div class="form-group">
+				    <label for="code" class="col-sm-2 control-label">名称:</label>
+				    <div class="col-sm-10">
+				      <input name="name" type="text" value="" class="form-control" placeholder="任务名称">
+				    </div>
+				</div>
+				<div class="form-group">
+				    <label for="code" class="col-sm-2 control-label">对比任务:</label>
+				    <div class="col-sm-5">
+				      	<select id="v1" name="v1" class="form-control">
+							<option value="">请选择版本</option>
+							{{range .vlist}}
+								<option value="{{._id}}">{{.s_name}}</option>
+							{{end}}
+						</select>
+					</div>
+					<div class="col-sm-5">
+				    <select id="v2" name="v2" class="form-control">
+							<option value="">请选择版本</option>
+							{{range .vlist}}
+								<option value="{{._id}}">{{.s_name}}</option>
+							{{end}}
+						</select>
+					</div>
+				</div>
+				<div class="form-group">
+				    <label for="code" class="col-sm-2 control-label">数据库:</label>
+				    <div class="col-sm-5">
+				      <input name="dbaddr" type="text" value="" class="form-control" placeholder="数据库连接地址">
+				    </div>
+					<div class="col-sm-5">
+				      <input name="dbname" type="text" value="" class="form-control" placeholder="数据库名称">
+				    </div>
+				</div>
+				<div class="form-group">
+				    <label for="code" class="col-sm-2 control-label">查询条件:</label>
+					<div class="col-sm-10">
+				      <input name="query" type="text" value="{}" class="form-control" placeholder="{}">
+				    </div>
+				</div>
+				<div class="form-group">
+				    <label for="code" class="col-sm-2 control-label">信息源:</label>
+					<div class="col-sm-5">
+				      <input name="table" type="text" value="" class="form-control" placeholder="信息表(bidding)">
+				    </div>
+					<div class="col-sm-5">
+				      <input name="limit" type="text" value="" class="form-control" placeholder="信息量">
+				    </div>
+				</div>
+				<div class="form-group">
+				   <label for="code" class="col-sm-2 control-label">导出属性:</label>
+				    <div class="col-sm-10" id="selectclear">
+						<div class="doublebox">
+							<select multiple="multiple" id="select1" style="overflow-x: scroll;"></select>
+						</div>
+						<div class="move">
+							<button type="button" id="right" class="btn btn-primary">右移</button>
+							<button type="button" id="left" class="btn btn-primary">左移</button>
+						</div>	
+						<div class="doublebox">
+						  	<select multiple="multiple" id="select2" style="overflow-x: scroll;"></select>
+						</div>
+				    </div>
+				</div>
+		    </div>
+		    <div class="modal-footer">
+		        <button type="button" class="btn btn-default cancel" data-dismiss="modal" onclick="reset()">取消</button>
+		        <button type="button" class="btn btn-primary save" onclick="save()">保存</button>
+	    	</div>
+		</div>
+	    <!-- /.modal-content -->
+		<input type="hidden" name="_id" value="">
+	    </form>
+  	</div>
+  <!-- /.modal-dialog -->
+</div>	
+<!-- footer -->
+{{template "dialog"}}
+{{template "footer"}}
+
+<script>
+menuActive("export")
+$(function () {
+	ttable=$('#dataTable').DataTable({
+		"paging"      : false,
+		"lengthChange": false,
+		"searching"   : true,
+		"ordering"    : false,
+		"info"        : true,
+		"autoWidth"   : false,
+		"ajax": {
+			"url": "/admin/task/export/data",
+			"type": "post",
+			"data":{}
+		},
+		"language": {
+            "url": "/res/dist/js/dataTables.chinese.lang"
+        },
+		"columns": [
+            { "data": "name"},
+			{ "data": "fields"},
+			{ "data": "query"},
+			{ "data": "limit"},
+			{ "data": "state",render:function(val,a,row){
+				if(row.state!=0){
+					return "已完成"
+				}else{
+					return "待执行"
+				}
+			}},
+			{"data":"_id",render:function(val,a,row){
+				if(row.state!=0){
+					return '<a class="btn btn-sm btn-success" href="../../res/down/'+row.filename+'"><i class="fa fa-fw fa-cloud-download"></i>下载</a>&nbsp;&nbsp;<a class="btn btn-sm btn-danger" href="#" onclick="del(\''+val+'\')">删&nbsp;&nbsp;除</a>'
+				}else{
+					return '<a class="btn btn-sm btn-success edit" href="#">编&nbsp;&nbsp;辑</a> &nbsp;&nbsp;<a class="btn btn-sm btn-danger" href="#" onclick="del(\''+val+'\')">删&nbsp;&nbsp;除</a>'
+				}	
+			}}
+       	]
+	});
+	
+	ttable.on('init.dt', function () {
+		$(".edit").click(function(){
+			$("#v1 option:first").attr("selected",true).siblings("option").attr("selected",false);
+			$("#v2 option:first").attr("selected",true).siblings("option").attr("selected",false);
+			var obj=ttable.row($(this).closest("tr")).data();
+			$.setForm('#dataform',obj);	
+			$.ajax({
+				url:"/admin/getfields",
+				type:"post",
+				success:function(r){
+					if(r.data){	
+						$("#select1").empty();
+						$("#select2").empty();
+						for(var i in r.data){
+							if(i==0){
+								continue;
+							}
+							f=r.data[i]
+							var right=false
+							for(var k in obj["fields"]){
+								if (f._id==obj["fields"][k]){
+									right=true
+								}
+							}
+							if (right){									
+								$("#select2").append("<option title='"+f.s_name+"' value='"+f._id+"'>"+f.s_name+"</option>");
+							}else{
+								$("#select1").append("<option title='"+f.s_name+"' value='"+f._id+"'>"+f.s_name+"</option>");									
+							}
+						}		
+					}
+				}
+			})
+			$("#v1").find("option[value='"+obj["v1id"]+"']").attr("selected",true);
+			$("#v2").find("option[value='"+obj["v2id"]+"']").attr("selected",true);
+			if(obj["state"]!=0){
+				$(".save").attr({"disabled":"disabled"});
+			}
+			$("input[name='_id']").val(obj["_id"]);
+			$("#modal-info-export").modal("show");
+		})
+	})
+})
+
+function del(_id){
+	showConfirm("确定删除?", function() {
+		$.ajax({
+			url:"/admin/task/export/del",
+			type:"post",
+			data:{"_id":_id},
+			success:function(r){
+				if(r.rep){				
+					ttable.ajax.reload();
+				}else{
+					showTip("删除失败", 1000, function() {});
+				}
+			}
+		})
+	});
+}
+
+function editorexport(){
+	$("#v1 option:first").attr("selected",true).siblings("option").attr("selected",false);
+	$("#v2 option:first").attr("selected",true).siblings("option").attr("selected",false);
+	$(".save").removeAttr("disabled");
+	$.ajax({
+		url:"/admin/getfields",
+		type:"post",
+		success:function(r){
+			if(r.data){	
+				$("#select1").empty();
+				$("#select2").empty();
+				for(var i in r.data){
+					if(i==0){
+						continue;
+					}
+					f=r.data[i]
+					$("#select1").append("<option title='"+f.s_name+"' value='"+f._id+"'>"+f.s_name+"</option>");
+				}	
+				$("#modal-info-export").modal("show");		
+			}
+		}
+	})
+}
+
+function save(){
+	var name=$("input[name='name']").val();
+	if(name==""){
+		showTip("请输入任务名称", 1000, function() {});
+		return false;
+	}
+	var v1=$("#v1").find("option:selected").val();
+	var v1name=$("#v1").find("option:selected").text();
+	var v2=$("#v2").find("option:selected").val();
+	var v2name=$("#v2").find("option:selected").text();
+	if(v1==v2||v1==""||v2==""){
+		showTip("请输入正确的版本", 1000, function() {});
+		return false;
+	}
+	var dbaddr=$("input[name='dbaddr']").val();
+	var dbname=$("input[name='dbname']").val();
+	if(dbaddr==""||dbname==""){
+		showTip("请输入数据库信息", 1000, function() {});
+		return false;
+	}
+	var fields = {};
+	$("#select2 option").each(function(i,val){
+		fields[i]=this.value
+	})
+	if(fields.length<1){
+		showTip("请输入导出属性", 1000, function() {});
+		return false;
+	}
+	var query=$("input[name='query']").val();
+	var table=$("input[name='table']").val();
+	var limit=$("input[name='limit']").val();
+	var _id=$("input[name='_id']").val();
+	$.ajax({
+		url:"/admin/task/export/save",
+		type:"post",
+		data:{"_id":_id,"name":name,"v1":v1,"v1name":v1name,"v2":v2,"v2name":v2name,"dbaddr":dbaddr,"dbname":dbname,"fields":fields,"query":query,"table":table,"limit":limit},
+		success:function(r){
+			if(r){
+				showTip("任务已创建", 500, function() {});
+				window.location.href="/admin/task/export"	
+			}else{
+				showTip("任务创建失败！", 1000, function() {});
+			}
+		}
+	})
+}
+</script>

src/web/templates/admin/task_list.html

@@ -37,6 +37,7 @@
 {{template "dialog"}}
 {{template "footer"}}
 <script>
+menuActive("task")
 $(function () {
 	ttable=$('#dataTable').DataTable({
 		"paging"      : false,