剑鱼异常数据维护修改

src/lua/spiderwarn.go

@@ -31,21 +31,20 @@ type Lua struct {
 	dataFind   xweb.Mapper `xweb:"/lua/datafind"`   //数据查找页面
 	searchData xweb.Mapper `xweb:"/lua/searchdata"` //通过title、href查找
 	dataView   xweb.Mapper `xweb:"/lua/dataview"`   //查看数据详情
-
 }
 
 func (l *Lua) SpiderWarn() {
 	defer qu.Catch()
 	if l.Method() == "POST" {
 		defer qu.Catch()
-		level, _ := l.GetInteger("level")
+		repeat, _ := l.GetInteger("repeat")
 		start, _ := l.GetInteger("start")
 		limit, _ := l.GetInteger("length")
 		draw, _ := l.GetInteger("draw")
 		startTime, _ := l.GetInt("starttime")
 		searchStr := l.GetString("search[value]")
 		search := strings.TrimSpace(searchStr)
-		qu.Debug(startTime, start, limit, draw, level)
+		qu.Debug(startTime, start, limit, draw)
 		sort := `{"%s":%d}`
 		orderIndex := l.GetString("order[0][column]")
 		orderName := l.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
@@ -57,24 +56,15 @@ func (l *Lua) SpiderWarn() {
 		query := map[string]interface{}{
 			"ok": false,
 		}
+		if repeat != -1 {
+			query["repeat"] = repeat == 1
+		}
 		if startTime > 0 {
 			query["comeintime"] = map[string]interface{}{
 				"$gte": startTime,
 				"$lt":  startTime + 86400,
 			}
 		}
-		if level != -1 {
-			query["level"] = level
-			//错误中区分中文错误和其他错误
-			if level == 2 {
-				query["info"] = "Field Value Not Contains Chinese"
-			} else if level == 3 {
-				query["level"] = 2
-				query["info"] = map[string]interface{}{
-					"$ne": "Field Value Not Contains Chinese",
-				}
-			}
-		}
 		if search != "" {
 			query["$or"] = []interface{}{
 				map[string]interface{}{"site": map[string]interface{}{"$regex": search}},
@@ -107,7 +97,7 @@ func (l *Lua) SpiderEdit() {
 	id := l.GetString("id")
 	data := map[string]interface{}{}
 	bidId := ""
-	one, _ := MgoS.FindById("spider_warn_err", id, map[string]interface{}{"data": 1, "field": 1, "level": 1})
+	one, _ := MgoS.FindById("spider_warn_err", id, map[string]interface{}{"data": 1, "field": 1})
 	if one != nil && len(*one) > 0 {
 		data, _ = (*one)["data"].(map[string]interface{})
 		detail := qu.ObjToString((data)["detail"])
@@ -124,17 +114,17 @@ func (l *Lua) SpiderEdit() {
 			1、异常类型为错误时，原信息未入bidding库
 			2、异常类型为警告时，原信息也已经入bidding库
 		*/
-		if qu.IntAll((*one)["level"]) == 1 {
-			bidData, _ := JYMgo.FindOne("bidding", map[string]interface{}{"s_sha": sha})
-			if bidData != nil && len(*bidData) > 0 {
-				bidId = mongodb.BsonIdToSId((*bidData)["_id"])
-				data = *bidData
-			} else {
-				qu.Debug("异常信息id:", id)
-				l.Render("com/err.html")
-				return
-			}
-		}
+		//if qu.IntAll((*one)["level"]) == 1 {
+		//	bidData, _ := JYMgo.FindOne("bidding", map[string]interface{}{"s_sha": sha})
+		//	if bidData != nil && len(*bidData) > 0 {
+		//		bidId = mongodb.BsonIdToSId((*bidData)["_id"])
+		//		data = *bidData
+		//	} else {
+		//		qu.Debug("异常信息id:", id)
+		//		l.Render("com/err.html")
+		//		return
+		//	}
+		//}
 		delete(data, "_id")
 		delete(data, "detail")
 		delete(data, "contenthtml")
@@ -187,7 +177,6 @@ func (l *Lua) SpiderSave() {
 			}
 			FormatNumber(update) //解决超大金额转成科学计数法的问题
 		}
-		qu.Debug(modifyinfo)
 		if stype == 1 { //修复更新
 			if bid == "" {
 				rep = "bidding id 为空"

src/web/templates/com/header.html

@@ -108,6 +108,25 @@
   	</div>
   <!-- /.modal-dialog -->
 </div>
+
+<div class="modal fade" id="loading" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true" data-backdrop="static" data-keyboard="false">>
+	<div class="modal-dialog">
+		<div class="modal-content" style="margin-top: 220px">
+			<div class="modal-header">
+				<div class="modal-header">
+					<div class="edit-form">
+						<form class="form-horizontal" role="form">
+							<div class="form-group">
+								<span class="info">正在加载...</span>
+								<span class="fa fa-spin fa-spinner" aria-hidden="true"></span>
+							</div>
+						</form>
+					</div>
+				</div>
+			</div>
+		</div><!-- /.modal-content -->
+	</div><!-- /.modal -->
+</div>
 <style>
 
 
@@ -133,10 +152,10 @@ showLoading = function (text){
 	if(text){
 		$("#loadText").html(text)
 	}
-	$('#loadingModal').modal({backdrop: 'static', keyboard: false});
+	$('#loading').modal({backdrop: 'static', keyboard: false});
 }
 //隐藏掉loading框
 hideLoading = function (){
-	$('#loadingModal').modal('hide');
+	$('#loading').modal('hide');
 }
 </script>

src/web/templates/lua/datafind.html

@@ -37,7 +37,6 @@
         right: 40px;
     }
 </style>
-
 <div class="content-wrapper" id="showbtn">
     <section class="content-header">
         <ol class="breadcrumb">
@@ -68,6 +67,7 @@
                                 <th>标题</th>
                                 <th>链接</th>
                                 <th>来源</th>
+                                <th>索引</th>
                                 <th>操作</th>
                             </tr>
                             </thead>
@@ -123,6 +123,13 @@
                         }
                     },  width: "10%"},
                 {"data": "by"},
+                {"data": function (row) {
+                        if(row.extracttype >= 0){
+                            return "已生成"
+                        }else{
+                            return "未生成"
+                        }
+                    }},
                 {"data": function (row) {
                         return "<a href='/lua/dataview?coll="+row.by+"&id="+row._id+"' target='_blank' class='btn btn-sm btn-primary'>详情</a>"
                     }},
@@ -141,12 +148,13 @@
             alert("内容不能为空")
             return
         }
-
+        showLoading("正在加载...");
         $.ajax({
             url:"/lua/searchdata",
             type:"post",
             data:{"text": text,"stype":stype},
             success:function(r){
+                hideLoading();
                 if (r&&r.data.length>0) {
                     $('#dataTable').dataTable().fnClearTable();
                     $('#dataTable').dataTable().fnAddData(r.data);

src/web/templates/lua/spiderwarn.html

@@ -47,11 +47,13 @@
                             <thead>
                             <tr>
                                 <th><input type="checkbox" id="selrow" onclick="selectrow(this)"/></th>
-                                <th>ID</th>
+                                <th>标题</th>
                                 <th>爬虫</th>
                                 <th>站点</th>
                                 <th>类型</th>
                                 <th>字段</th>
+                                <th>重复</th>
+                                <th>来源</th>
                                 <th>错误原因</th>
                                 <th>操作</th>
                             </tr>
@@ -82,7 +84,7 @@
                 "url": "/dist/js/dataTables.chinese.lang"
             },
             "columnDefs": [
-                { "orderable": false, "targets": [0,1,3,6,7] } //设置列不可排序
+                { "orderable": false, "targets": [0,1,3,6,7,8,9] } //设置列不可排序
             ],
             "order": [[2,"desc"]],
             "paging": true,
@@ -98,11 +100,11 @@
             				return "<input type='checkbox' value='"+val+"' />"
             			}},
                 {
-                    "data": "_id", render: function (val, a, row) {
-                        return val="<a href='"+row.href+"' title='"+row.title+"' target='_blank'>"+row._id+"</a>"
+                    "data": "title", render: function (val, a, row) {
+                        return val="<a href='"+row.href+"' title='"+row.title+"' target='_blank'>"+val+"</a>"
                     }
                 },
-                {"data": "code"},
+                {"data": "spidercode"},
                 {"data": "site"},
                 {"data": "level", render: function (val, a, row) {
                     if(val == 2 ){
@@ -111,6 +113,14 @@
                     return "警告";
                 }},
                 {"data": "field"},
+                {"data": "repeat",render: function(val){
+                    if(val){
+                        return "是"
+                    }else{
+                        return "否"
+                    }
+                    }},
+                {"data": "from"},
                 {"data": "info"},
                 {
                   "data": "_id", render: function (val, a, row) {
@@ -130,12 +140,20 @@
                 var nst = Date.parse(new Date().toLocaleDateString())/1000;
                 e.starttime = nst
               }
-              var level = $("#errtype").val();
-              if(level){
-                  e.level = level;
+              //repeat
+              var repeat = $("#repeat").val();
+              if(repeat){
+                  e.repeat = repeat;
               }else{
-                  e.level= -1;
+                  e.repeat= -1;
               }
+              //来源
+                var repeat = $("#repeat").val();
+                if(repeat){
+                    e.repeat = repeat;
+                }else{
+                    e.repeat= -1;
+                }
             }
         });
         ttable.on('init.dt', function () {
@@ -148,20 +166,27 @@
             "</span>"+
          "</div>&nbsp;"
           
-          var selectStype="<div class='form-group' style='margin-top: -4px'><label for='name'>异常类型：</label>"+
-              "<select id='errtype' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+          var selectRepeat="<div class='form-group' style='margin-top: -4px'><label for='name'>是否重复：</label>"+
+              "<select id='repeat' onchange='checkclick(this.value)' class='form-control input-sm'>"+
               "<option value='-1'>全部</option>"+
-              "<option value='1'>警告</option>"+
-              "<option value='2'>中文错误</option>"+
-              "<option value='3'>其他错误</option>"+
+              "<option value='1'>是</option>"+
+              "<option value='0'>否</option>"+
+              "</select></div>"
+          var selectFrom="<div class='form-group' style='margin-top: -4px'><label for='name'>来源：</label>"+
+              "<select id='from' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+              "<option value='-1'>全部</option>"+
+              "<option value='list'>list</option>"+
+              "<option value='warn'>warn</option>"+
               "</select></div>"
           $("#spiderwarn_filter").prepend(errStype);
           $("#spiderwarn_filter").prepend("&nbsp;");
-          $("#spiderwarn_filter").prepend(selectStype);
-          $("#spiderwarn_length").parent().css("width", "30%")
-          $("#spiderwarn_filter").parent().css("width", "70%")
+          $("#spiderwarn_filter").prepend(selectRepeat);
+          $("#spiderwarn_filter").prepend("&nbsp;&nbsp");
+          $("#spiderwarn_filter").prepend(selectFrom);
+          $("#spiderwarn_length").parent().css("width", "10%")
+          $("#spiderwarn_filter").parent().css("width", "90%")
           $("#spiderwarn_filter").css({"display": "flex","text-align": "right","align-items": "center","justify-content": "flex-end"})
-        
+
           $('.date-picker').datepicker({
               language: 'zh-CN',
               autoclose: true,