新增过滤数据展示列表

src/front/front.go

@@ -66,6 +66,9 @@ type Front struct {
 	uploadFile   xweb.Mapper `xweb:"/front/lua/uploadfile"`   //上传附件
 	luaDataSend  xweb.Mapper `xweb:"/front/lua/datasend"`     //数据更新、保存
 	spiderUpdate xweb.Mapper `xweb:"/front/lua/spiderupdate"` //批量更新状态
+	filterList   xweb.Mapper `xweb:"/front/lua/filterlist"`   //过滤数据列表
+	filterEdit   xweb.Mapper `xweb:"/front/lua/filteredit"`   //过滤数据详情
+	filterSave   xweb.Mapper `xweb:"/front/lua/filtersave"`   //过滤数据保存
 
 	//data
 	//dataFind   xweb.Mapper `xweb:"/front/lua/datafind"`   //数据查找页面

src/front/lua.go

@@ -336,6 +336,98 @@ func TagToSpiderWarnErr(state int, id, reason string, update map[string]interfac
 	MgoS.UpdateById("spider_warn_err", id, map[string]interface{}{"$set": set})
 }
 
+func (f *Front) FilterList() {
+	defer qu.Catch()
+	if f.Method() == "POST" {
+		defer qu.Catch()
+		draw, _ := f.GetInteger("draw")
+		start, _ := f.GetInteger("start")
+		limit, _ := f.GetInteger("length")
+		searchStr := f.GetString("search[value]")
+		search := strings.TrimSpace(searchStr)
+		sort := `{"%s":%d}`
+		orderIndex := f.GetString("order[0][column]")
+		orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
+		orderType := 1
+		if f.GetString("order[0][dir]") != "asc" {
+			orderType = -1
+		}
+		sort = fmt.Sprintf(sort, orderName, orderType)
+		query := map[string]interface{}{
+			"biddingid": map[string]interface{}{
+				"$exists": false,
+			},
+		}
+		if search != "" {
+			query["$or"] = []interface{}{
+				map[string]interface{}{"site": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"spidercode": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"title": map[string]interface{}{"$regex": search}},
+			}
+		}
+		count := MgoS.Count("spider_filterdata", query)
+		qu.Debug("query:", query, "sort:", sort, count)
+		fields := map[string]interface{}{
+			"spidercode": 1,
+			"title":      1,
+			"href":       1,
+			"site":       1,
+		}
+		list, _ := MgoS.Find("spider_filterdata", query, sort, fields, false, start, limit)
+		f.ServeJson(map[string]interface{}{
+			"data":            list,
+			"draw":            draw,
+			"recordsFiltered": count,
+			"recordsTotal":    count,
+		})
+	} else {
+		f.Render("lua/filterlist.html", &f.T)
+	}
+}
+
+func (f *Front) FilterEdit() {
+	defer qu.Catch()
+	id := f.GetString("id")
+	one, _ := MgoS.FindById("spider_filterdata", id, SaveFields)
+	if one != nil && len(*one) > 0 {
+		detail := qu.ObjToString((*one)["detail"])
+		contenthtml := qu.ObjToString((*one)["contenthtml"])
+		summary := qu.ObjToString((*one)["summary"])
+		f.T["detail"] = detail
+		f.T["contenthtml"] = contenthtml
+		f.T["summary"] = summary
+		delete(*one, "_id")
+		delete(*one, "detail")
+		delete(*one, "contenthtml")
+		delete(*one, "summary")
+		delete(*one, "l_np_publishtime")
+	}
+	f.T["id"] = id
+	f.T["data"] = one
+	f.Render("lua/filteredit.html", &f.T)
+}
+
+func (f *Front) FilterSave() {
+	defer qu.Catch()
+	id := f.GetString("id")
+	msg := "保存失败"
+	success := false
+	data, _ := MgoS.FindById("spider_filterdata", id, SaveFields)
+	if len(*data) > 0 {
+		biddingId := MgoB.Save("bidding", data)
+		if biddingId != "" {
+			msg = "保存成功"
+			success = true
+		}
+		shaid := ShaDetail(qu.ObjToString((*data)["detail"]))
+		(*data)["s_sha"] = shaid
+		(*data)["extracttype"] = 0
+		RedisClusterSet(shaid, "", 86400*365)
+		MgoS.UpdateById("spider_filterdata", id, map[string]interface{}{"$set": map[string]interface{}{"biddingid": biddingId, "s_sha": shaid}})
+	}
+	f.ServeJson(map[string]interface{}{"msg": msg, "success": success})
+}
+
 //func (f *Front) SaveUpdateLog(s_type int, reasons, bid string, old_data, update, modifyinfo map[string]interface{}) {
 //	//日志记录
 //	user := f.GetSession("user").(map[string]interface{})

src/util/util.go

@@ -29,10 +29,30 @@ var Fields = map[string]interface{}{
 	"T":           1,
 	"dataging":    1,
 	"infoformat":  1,
+	"summary":     1,
 	//"publishdept": 1,
 	//"type":        1,
 }
-
+var SaveFields = map[string]interface{}{
+	"publishtime": 1,
+	"detail":      1,
+	"contenthtml": 1,
+	"title":       1,
+	"href":        1,
+	"competehref": 1,
+	"area":        1,
+	"city":        1,
+	"district":    1,
+	"site":        1,
+	"channel":     1,
+	"spidercode":  1,
+	"jsondata":    1,
+	"projectinfo": 1,
+	"dataging":    1,
+	"infoformat":  1,
+	"type":        1,
+	"summary":     1,
+}
 var ImportDataFieldsCheck = []string{"href", "title", "site", "channel", "area", "city", "district", "spidercode", "publishtime"}
 
 const GTEID = "5a862f0640d2d9bbe88e3cec"

src/web/templates/lua/filteredit.html

@@ -0,0 +1,184 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "menu/menu.html"}}
+
+<style>
+    /* 方法1：设置textarea合适的宽高 */
+    #jsonTextarea {
+        float: left;
+        margin-right: 20px;
+        width: 40%;
+        height: 70vh;
+        outline: none;
+        padding: 5px;
+    }
+
+    /* 方法2：自定义高亮样式 */
+    #jsonPre {
+        float: left;
+        width: 40%;
+        height: 70vh;
+        outline: 1px solid #ccc;
+        padding: 5px;
+        overflow: scroll;
+    }
+</style>
+<div class="modal fade" id="uploadfile" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                    <span aria-hidden="true">&times;</span>
+                </button>
+                <h4>
+                    <span class="fa fa-cloud-upload" id="code-assign" aria-hidden="true"></span>
+                    <span class="info">附件</span>
+                </h4>
+
+            </div>
+            <div class="modal-body" style="max-height: 70vh;overflow-y: scroll;">
+                <form class="form-horizontal" role="form">
+                    <div id="TaskDiv">
+                        <div id="itemDiv" style="display: none">
+                            <div class="form-group">
+                                <label class="col-sm-2 control-label">附件地址:</label>
+                                <div class="col-sm-5">
+                                    <input type="text" class="furl form-control" placeholder="附件下载地址" value="">
+                                </div>
+                            </div>
+                            <div class="form-group">
+                                <label class="col-sm-2 control-label">本地文件:</label>
+                                <div class="col-sm-5">
+                                    <input type="file" name="file" class="ffile" id="uploadfile-add-0">
+                                </div>
+                                <div style="float: right;margin-right: 158px">
+                                    <input type="button" class="btn btn-info" value="删除" onclick="removeFile(this)">
+                                </div>
+                            </div>
+<!--                            <div class="form-group">-->
+<!--                                <label class="col-sm-2 control-label">附件名称:</label>-->
+<!--                                <div class="col-sm-5">-->
+<!--                                    <input type="text" class="fname form-control" placeholder="附件名称.类型" value="">-->
+<!--                                </div>-->
+<!--                            </div>-->
+                        </div>
+                    </div>
+                    <div class="group-item">
+                        <div class="form-group">
+                            <label class="col-sm-2 control-label">附件地址:</label>
+                            <div class="col-sm-5">
+                                <input type="text" class="furl form-control" id="downloadurl" placeholder="附件下载地址" value="">
+                            </div>
+                        </div>
+                        <div class="form-group">
+                            <label class="col-sm-2 control-label">本地文件:</label>
+                            <div class="col-sm-5">
+                                <input type="file" name="file" class="ffile" id="uploadfile-add">
+                            </div>
+                            <div style="float: right;margin-right: 100px">
+                                <input type="button" class="btn btn-info" value="新增" onclick="addFile()">
+                                <input type="button" class="btn btn-info" value="上传" onclick="uploadFile()">
+                            </div>
+                        </div>
+<!--                        <div class="form-group">-->
+<!--                            <label class="col-sm-2 control-label">附件名称:</label>-->
+<!--                            <div class="col-sm-5">-->
+<!--                                <input type="text" class="fname form-control" id="filename" placeholder="附件名称.类型" value="">-->
+<!--                            </div>-->
+<!--                            <input type="button" class="btn btn-info" value="新增" onclick="addFile()">-->
+<!--                            <input type="button" class="btn btn-info" value="上传" onclick="uploadFile()">-->
+<!--                        </div>-->
+                    </div>
+                </form>
+                <!--</div>	-->
+            </div>
+        </div>
+    </div>
+</div>
+
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>
+            {{if ne .T.bid ""}}
+                修复数据
+            {{else}}
+                新增数据
+            {{end}}
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="#"><i class="fa fa-dashboard"></i> 爬虫数据维护</a></li>
+            <li><a href="front/lua/warnlist"> 异常数据管理</a></li>
+            <li><a href="#">数据维护</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="nav-tabs-custom">
+            <ul class="nav nav-tabs edit-step">
+<!--                <button class="btn btn-primary btn-sm" style="float: left;margin-top: 7px;margin-left: 10px" onclick="showUploadFileModal()"><i class="fa fa-fw fa-file-text fa-lg"></i>补充附件</button>-->
+                <button class="btn btn-primary btn-sm" style="float: right;margin-top: 7px;margin-right: 10px" onclick="saveFileterData()">保存数据</button>
+
+            </ul>
+            <form class="form-horizontal">
+                <div class="box-body">
+                    <textarea id="jsonTextarea" readonly="true" style="width: 100%;height: 300px"></textarea>
+                </div>
+                <div class="box-body">
+                    <label class="col-sm-2 control-left"><span style="color:red;">* </span>请填写detail</label>
+                    <textarea id="jsonDetail" readonly="true" style="width: 100%;height: 300px;padding: 5px"></textarea>
+                    <label class="col-sm-2 control-left"><span style="color:red;">* </span>请填写contenthtml</label>
+                    <textarea id="jsonContentHtml" readonly="true" style="width: 100%;height: 300px;padding: 5px"></textarea>
+                </div>
+                <div class="box-body" id="summaryDiv">
+                    <label class="col-sm-2 control-left">请填写summary</label>
+                    <textarea id="summary" readonly="true" style="width: 100%;height: 200px;padding: 5px"></textarea>
+                </div>
+            </form>
+        </div>
+    </section>
+</div>
+
+{{include "com/footer.html"}}
+<script>
+    menuActive("warnlist");
+    var fileNameReg = new RegExp("(.jpg|.png|.gif|.pdf|.doc|.docx|.xlsx|.xls|.zip|.rar)$");
+    var edit_data = {{.T.data}}
+    var edit_contentHtml = {{.T.contenthtml}}
+    var edit_detail = {{.T.detail}}
+    var edit_summary = {{.T.summary}}
+    var bid = {{.T.bid}}
+    var edit_data_tmp = {{.T.data}}//记录汇总数据
+    edit_data_tmp["detail"] = edit_detail
+    edit_data_tmp["contenthtml"] = edit_contentHtml
+    edit_data_tmp["summary"] = edit_summary
+    function parse(str) {
+        return JSON.stringify(str, null, "\t")
+    }
+    $('#jsonTextarea').val(parse(edit_data)); //其他属性
+    $('#jsonDetail').val(edit_detail); //其他属性
+    $('#jsonContentHtml').val(edit_contentHtml); //其他属性
+    $('#summary').val(edit_summary);
+
+    function saveFileterData(){
+        showConfirm("确定保存？", function() {
+            $.ajax({
+                url: "/front/lua/filtersave",
+                type: 'POST',
+                data: {
+                    "id": {{.T.id}}
+                },
+                success: function (r) {
+                    if (r.success) {
+                        showMsg(r.msg,function (){
+                            window.location.href="/front/lua/filterlist"
+                        })
+                    } else {
+                        showTip(r.msg);
+                    }
+                }
+            })
+        });
+    }
+</script>

src/web/templates/lua/filterlist.html

@@ -0,0 +1,152 @@
+{{include "com/inc.html"}}
+<!-- Main Header -->
+{{include "com/header.html"}}
+<!-- Left side column. 权限菜单 -->
+{{include "menu/menu.html"}}
+
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>
+            <small>
+                <button type="button" class="btn btn-primary" data-toggle="modal" onclick="batchprocess(2)">批量删除</button>
+            </small>
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="#"><i class="fa fa-dashboard"></i> 爬虫数据维护</a></li>
+            <li><a href="/front/lua/filterlist"> 过滤数据管理</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="row">
+            <div class="col-xs-12">
+                <div class="box">
+                    <div class="box-body">
+                       <!-- <div>
+                            <div id="status-div" style="width: 205px;float: right">
+                                <span class="input-group date date-picker" id="starttime" data-provide="datepicker">
+                                    <input type="text" class="form-control form-filter input-sm" readonly name="starttime" placeholder="开始日期" />
+                                     <span class="input-group-addon">
+                                         <i class="fa fa-calendar"></i>
+                                      </span>
+                                </span>
+                            </div>
+                            <div style="display: flex;float: right;margin-right: 10px;align-items: center">
+                                <label style="white-space: nowrap;margin-bottom: 0;" for='name'>异常类型：</label>
+                                <select id='errtype' onchange='checkclick(this.value)' class='form-control input-sm'>
+                                    <option value='-1'>全部</option>
+                                    <option value='1'>警告</option>
+                                    <option value='2'>错误</option>
+                                </select>
+                            </div>
+                        </div>-->
+
+                        <table id="filterdata" class="table table-bordered table-hover">
+                            <thead>
+                            <tr>
+                                <th><input type="checkbox" id="selrow" onclick="selectrow(this)"/></th>
+                                <th>站点</th>
+                                <th>爬虫</th>
+                                <th>标题</th>
+                                <th>操作</th>
+                            </tr>
+                            </thead>
+                        </table>
+                    </div>
+                    <!-- /.box-body -->
+                </div>
+                <!-- /.box -->
+            </div>
+        </div>
+    </section>
+</div>
+
+
+<!-- footer -->
+{{include "com/footer.html"}}
+
+<script>
+    menuActive("filterlist");
+    $(function () {
+        ttable = $('#filterdata').DataTable({
+            "ajax": {
+                "url": "/front/lua/filterlist",
+                "type": "post",
+                "data": {}
+            },
+            "language": {
+                "url": "/dist/js/dataTables.chinese.lang"
+            },
+            "columnDefs": [
+                { "orderable": false, "targets": [0,3,4] } //设置列不可排序
+            ],
+            "order": [[2,"desc"]],
+            "paging": true,
+            "lengthChange": false,
+            "searching": true,
+            "ordering": true,
+            "info": true,
+            "serverSide": true,
+            "autoWidth":false,
+            "bLengthChange":true,//支持分页数选择
+            "columns": [
+                {"data": "_id",render:function(val,a,row){
+            				return "<input type='checkbox' value='"+val+"' />"
+            			}},
+                {"data": "site"},
+                {"data": "spidercode"},
+                {"data": "title", render: function (val, a, row) {
+                        return val="<a href='"+row.href+"' title='"+row.title+"' target='_blank'>"+val+"</a>"
+                    }},
+                {"data": "_id", render: function (val, a, row) {
+                     return "<a href='/front/lua/filteredit?id="+val+"'><i class='fa fa-fw fa-edit text-yellow'></i></a>";
+                }}
+            ]
+        });
+    });
+    //批量删除
+    function batchprocess(state){
+      var ids=[];
+    		$("#filterdata td input[type=checkbox]").each(function(){
+    			if($(this).prop("checked")){
+    				ids.push($(this).attr("value"));
+    			}
+    		});
+      if(ids.length > 0){
+        showConfirm("确定批量处理？", function() {
+          $.ajax({
+            url: "/front/lua/spiderupdate",
+            type: "post",
+            data: {"ids":ids.join(","),"state":state},
+            success: function(r){
+              if(r&&r.success){
+                $("#selrow").prop('checked',false);
+                 ttable.ajax.reload();
+              }
+            }
+          })
+        })
+      }else{
+          if(state == 2){
+            alert("请选择要删除的数据！")
+          }else if(state == 4){
+              alert("请选择要发布的数据！")
+          }
+      }
+    }
+    
+    function checkclick(){
+        ttable.ajax.reload();
+    }
+    function selectrow(me){
+    		var sel=$(me);
+    		var isSelected=sel.prop('checked');
+    		if(isSelected){
+    			$("#filterdata td input[type=checkbox]").prop("checked",true);
+    		}else{
+    			$("#filterdata td input[type=checkbox]").prop("checked",false);
+    		}
+    	}
+        
+</script>