xg

src/front/front.go

@@ -58,6 +58,7 @@ type Front struct {
 	findName    xweb.Mapper `xweb:"/center/findname"`        //即时查询名称
 	checkrepeat xweb.Mapper `xweb:"/center/spider/isrepeat"` //脚本代码判重
 	heart       xweb.Mapper `xweb:"/center/heart"`           //心跳监控
+	spiderCopy  xweb.Mapper `xweb:"/center/spider/copy"`     //复制爬虫
 
 	Base      Base
 	OtherBase OtherBase
@@ -943,3 +944,54 @@ func (f *Front) GetCity() {
 	cityArr = u.Province[area]
 	f.ServeJson(cityArr)
 }
+
+func (f *Front) SpiderCopy() {
+	defer qu.Catch()
+	code := f.GetString("code")
+	code = strings.TrimSpace(code)
+	success := false
+	text := ""
+	encode := ""
+	if code != "" {
+		lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
+		if len(lua) > 0 {
+			codeOld := qu.ObjToString(lua["code"])
+			codeNew := codeOld + "_bu"
+			luaTmp := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
+			if len(luaTmp) > 0 {
+				f.ServeJson(map[string]interface{}{"success": false, "text": "已存在补漏爬虫"})
+				return
+			}
+			lua["code"] = codeNew
+			if param_common := lua["param_common"].([]interface{}); len(param_common) < 13 {
+				f.ServeJson(map[string]interface{}{"success": false, "text": "脚本异常，请联系管理员"})
+				return
+			} else {
+				param_common = param_common[:13]
+				param_common[0] = codeNew                  //code
+				param_common = append(param_common, true)  //历史补漏
+				param_common = append(param_common, false) //强制下载
+				lua["param_common"] = param_common
+			}
+			str_list := qu.ObjToString(lua["str_list"])
+			lua["str_list"] = strings.ReplaceAll(str_content, codeOld, codeNew)
+			lua["comeintime"] = time.Now().Unix()
+			lua["modifytime"] = time.Now().Unix()
+			lua["spidertype"] = "increment"
+			lua["event"] = 7000
+			lua["state"] = 6
+			id := mgdb.Save("luaconfig", lua)
+			if id != "" {
+				success = true
+				encode = util.Se.Encode2Hex(codeNew)
+			} else {
+				text = "新爬虫保存失败"
+			}
+		} else {
+			text = "未查询到复制原"
+		}
+	} else {
+		text = "脚本不能为空"
+	}
+	f.ServeJson(map[string]interface{}{"success": success, "text": text, "encode": encode})
+}

src/luaerrdata/errlua.go

@@ -0,0 +1,84 @@
+package luaerrdata
+
+import (
+	qu "qfw/util"
+	mgdb "qfw/util/mongodb"
+	mgu "qfw/util/mongodbutil"
+	"spider"
+	util "spiderutil"
+	"time"
+
+	"github.com/go-xweb/xweb"
+)
+
+type ErrorLua struct {
+	*xweb.Action
+	errorLuaIndex xweb.Mapper `xweb:"/center/errorLua"`          //加载错误爬虫信息
+	reUpSpider    xweb.Mapper `xweb:"/center/errorLua/upSpider"` //重新上架
+}
+
+func (el *ErrorLua) ErrorLuaIndex() {
+	defer qu.Catch()
+	if el.Method() == "POST" {
+		auth := qu.IntAll(el.GetSession("auth"))
+		username := el.GetSession("loginuser").(string)
+		query := map[string]interface{}{
+			// "modifytime": map[string]interface{}{
+			// 	"$exists": false,
+			// },
+		}
+		if auth == role_dev {
+			if username != "" {
+				query["modifyuser"] = username
+			} else {
+				el.Write("数据错误")
+			}
+		}
+		qu.Debug("query:", query)
+		list := *mgu.Find("spider_loadfail", "spider", "spider", query, map[string]interface{}{"_id": -1}, nil, false, -1, -1)
+		for _, l := range list {
+			l["encode"] = util.Se.Encode2Hex(qu.ObjToString(l["code"]))
+		}
+		el.ServeJson(map[string]interface{}{"data": list})
+	} else {
+		el.Render("errlualist.html")
+	}
+}
+
+func (el *ErrorLua) ReUpSpider() {
+	defer qu.Catch()
+	code := el.GetString("code")
+	id := el.GetString("id")
+	lua := *mgdb.FindOne("luaconfig", `{"code":"`+code+`"}`)
+	success := false
+	text := ""
+	if len(lua) > 0 {
+		event := qu.IntAll(lua["event"])
+		if qu.IntAll(lua["state"]) == 5 { //是上架状态
+			//重新上架
+			b, err := spider.UpdateSpiderByCodeState(code, "5", event)
+			if b && err == nil {
+				success = true
+				go mgu.Update("spider_loadfail", "spider", "spider",
+					map[string]interface{}{
+						"_id": qu.StringTOBsonId(id),
+					},
+					map[string]interface{}{
+						"$set": map[string]interface{}{
+							"modifytime": time.Now().Unix(),
+						},
+					}, false, false)
+			} else {
+				success = false
+				text = err.Error()
+			}
+		} else {
+			success = false
+			text = "爬虫暂时无法上架，请联系管理人员"
+		}
+	} else {
+		success = false
+		text = "没有对应爬虫"
+	}
+	el.ServeJson(map[string]interface{}{"success": success, "text": text})
+}

src/main.go

@@ -68,6 +68,7 @@ func init() {
 	xweb.AddAction(&taskManager.TaskM{})
 	xweb.AddAction(&quesManager.QuesM{})
 	xweb.AddAction(&luaerrdata.ErrorData{})
+	xweb.AddAction(&luaerrdata.ErrorLua{})
 	xweb.AddAction(&front.LuaMove{})
 	xweb.RootApp().AppConfig.SessionTimeout = 1 * time.Hour
 	xweb.RootApp().Logger.SetOutputLevel(4)

src/taskManager/taskManager.go

@@ -475,7 +475,7 @@ func (t *TaskM) Mytask() {
 		}
 		sort.Strings(events)
 		t.T["events"] = events
-
+		//查询当前未通过任务条数
 		failedtasknum := 0
 		if SessionFailuer {
 			query["s_modifyid"] = userid
@@ -484,7 +484,18 @@ func (t *TaskM) Mytask() {
 			failedtasknum = len(task)
 			SessionFailuer = false
 		}
+		//查询是否有错误爬虫
+		modifyuser := qu.ObjToString(t.GetSession("loginuser"))
+		errluanum := mgu.Count("spider_loadfail", "spider", "spider",
+			map[string]interface{}{
+				"modifyuser": modifyuser,
+				"modifytime": map[string]interface{}{
+					"$exists": false,
+				},
+			},
+		)
 		t.T["failedtasknum"] = failedtasknum
+		t.T["errluanum"] = errluanum
 		t.Render("mytask.html", &t.T)
 	}
 }

src/timetask/timetask.go

@@ -185,6 +185,7 @@ func UpdateSiteInfo() {
 }
 
 func UpdateCodeHeart() {
+	qu.Debug("定时更新爬虫心跳信息...")
 	defer qu.Catch()
 	query := map[string]interface{}{
 		"state": map[string]interface{}{
@@ -197,4 +198,5 @@ func UpdateCodeHeart() {
 			"$set": map[string]interface{}{"del": true},
 		}, false, false)
 	}
+	qu.Debug("定时更新爬虫心跳信息完成...")
 }

src/web/templates/errlualist.html

@@ -0,0 +1,138 @@
+{{include "head.html"}}
+<link rel="stylesheet"  href="/css/otherStyle.css">
+<style>
+</style>
+<div class="content-wrapper">
+		<section class="content-header">
+		   <h1>
+		     错误爬虫列表
+		     <small>
+			   </small>
+		   </h1>
+        <!--时间-->
+		   <ol class="breadcrumb">
+		     <li><a href="#"><i class="fa fa-dashboard"></i> 错误信息</a></li>
+		     <li class="active">列表</li>
+		   </ol>
+		 </section>
+		<section id="taskerrmsg" class="content hide" style="min-height:0px;">
+			<h5>错误信息：</h5>
+		</section>
+		<section class="content">
+			<div class="box">
+       <div id="timeTopBox">
+         
+       </div>
+				<div class="box-body">
+					<table id="errdata" class="table table-bordered table-striped">
+						<thead>
+							<tr>
+                <th>爬虫代码</th>
+                <th>错误信息</th>
+                <th>维护人</th>
+                <th>创建日期</th>
+                <th>维护日期</th>
+                <th>节点</th>
+                <th>功能</th>
+								<th>操作</th>
+							</tr>
+						</thead>
+						<tbody>
+							
+						</tbody>
+					</table>
+				</div>
+			</div>
+		</section>
+<script>
+	$(function(){
+    common.setActive("index_errlua");
+	  errluatable = $('#errdata').DataTable({
+			"language": {
+        "url": "/js/dataTables.chinese.lang"
+      },
+			"ajax": {
+      	"url": "/center/errorLua",
+     		"type": "POST", 
+ 		  },
+      "columnDefs": [
+        {"targets":[7], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
+          $(cell).click(function () {
+            $(this).html("<button type='button' onclick='upspider(\""+rowData.code+"\",\""+rowData._id+"\")' class='btn btn-sm btn-warning'>上架</button>");
+          });
+        }}
+      ],
+			"lengthChange":false,
+			"serverSide": true,
+			"searching": false,
+			"ordering": false,
+			"info": false,
+			"autoWidth": true,
+      "paging": false,
+      "processing": true,
+			"columns": [
+				{"data": "code"},
+        {"data": "err"},
+        {"data": "modifyuser"},
+        {"data": "updatetime",render:function(val,a,row){
+          var dt = new Date()
+          dt.setTime(parseInt(val) * 1000);
+          return dt.format("yyyy-MM-dd hh:mm:ss");
+        }},
+        {"data": "modifytime",render:function(val,a,row){
+          if(val == null){
+            return ""
+          }
+          var dt = new Date()
+          dt.setTime(parseInt(val) * 1000);
+          return dt.format("yyyy-MM-dd hh:mm:ss");
+        }},
+        {"data": "event"},
+        {"data": "encode","width":"92px",render:function(val,a,row){
+          var div=$("<div><div class=\"btn-group\"></div></div>")
+          var buttonDownload=$('<a type="button" class="btn btn-sm btn-info">下载</a>');
+					buttonDownload.attr("href","/center/spider/download/"+row["code"]);
+					div.find(".btn-group").append(buttonDownload);
+          var button=$('<a type="button" target="_blank" class="btn btn-sm btn-primary">编辑</a>');
+					button.attr("href","/center/spider/edit/"+val+"__");
+					div.find(".btn-group").append(button);
+          return div.html()
+				}},
+        {"data": "encode",render:function(val,a,row){
+          console.log(row.modifytime)
+          var button
+           var div=$("<div><div class=\"btn-group\"></div></div>")
+          if(row.modifytime==null){
+            button=$('<button type="button" class="btn btn-sm btn-danger">完成</button>');
+          }else{
+             button=$('<button type="button" disabled class="btn btn-sm btn-success">已修改</button>');
+          }
+					div.find(".btn-group").append(button);
+          return div.html()
+				}}
+			]
+	  });
+	})
+	//修改节点
+  function upspider(code,id){
+    showConfirm("确定重新上架", function() {
+      $.ajax({
+				url:"/center/errorLua/upSpider",
+        data:{"code":code,"id":id},
+				type:"post",
+				success:function(r){
+					if(r&&r.success){
+						errluatable.ajax.reload(null,false);
+					}else{
+						showTip(r.text,2000, function() {});
+					}
+				}
+			})
+    });
+  };
+  function updatefinish(){
+    
+  }
+</script>
+</div>
+{{include "bottom.html"}}

src/web/templates/head.html

@@ -567,6 +567,7 @@
 			<li data="index"><a href="/center"><i class="glyphicon glyphicon-eye-open"></i> <span>爬虫列表</span></a></li>
 			<li data="index_rwlb"><a href="/center/mytask"><i class="glyphicon glyphicon-tasks"></i> <span>任务列表</span></a></li>
 		{{end}}
+    <li data="index_errlua"><a href="/center/errorLua"><i class="fa fa-warning"></i> <span>错误爬虫管理</span></a></li>
     <li data="index_errdata"><a href="/center/errorData"><i class="glyphicon glyphicon-exclamation-sign"></i> <span>错误信息管理</span></a></li>
 		<li data="index_wtgl"><a href="/center/managerQues"><i class="glyphicon glyphicon-question-sign"></i> <span>问题管理</span></a></li>
 		<li ><a href="/center/logout" target="_self"><i class="fa fa-sign-out"></i> <span>退出</span></a></li>

src/web/templates/index.html

@@ -73,11 +73,45 @@
         </div><!-- /.modal-content -->
     </div><!-- /.modal -->
 </div>
+<!-- 模态框（Modal） -->
+<div class="modal fade" id="modal-spidercopy" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+			<div class="modal-header">
+				<div class="modal-header">
+	                <button type="button" id="assign-close" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
+					<!--<div class="edit-form">-->
+						<div class="edit-info">
+							<span class="fa fa-copy" id="code-assign" aria-hidden="true"></span>
+							<span class="info">复制爬虫<span>
+						</div>
+						<form class="form-horizontal" role="form">
+							<div class="form-group">
+							    <label for="modify" class="col-sm-2 control-label">爬虫:</label>
+							    <div class="col-sm-10">
+									<input class="form-control" id="spidercopy"></input>
+							    </div>
+							</div>
+							<div class="form-group" id="assign-style">
+							    <div class="col-sm-offset-2 col-sm-10">
+							      	<input type="button" onclick="comfirm_spidercopy()" class="btn btn-primary" value="确定">
+									<input type="button" onclick="spidercopy_cancel()" class="btn btn-default" value="取消">
+							    </div>
+							</div>
+						</form>
+					<!--</div>	-->
+				</div>
+		 	</div>
+        </div><!-- /.modal-content -->
+    </div><!-- /.modal -->
+</div>
+
 <div class="content-wrapper">
 <section class="content-header">
    <h1>
      脚本列表
      <small>
+      <a class="btn btn-default btn-sm" onclick="spidercopy()">复制爬虫</a>
 			{{if gt (session "auth") 2}}
 			<a class="btn btn-default btn-sm" href="/center/spider">新建爬虫</a>
 			<a class="btn btn-default btn-sm" id="assign" onclick="assign()">分配</a>
@@ -509,6 +543,10 @@ $(function(){
 	//		})
 	//	});
     //}
+  function spidercopy(){
+    $("#modal-spidercopy").modal("show");
+  }  
+    
 	function assign(){
 		var names=[];
 		$("#spider td input[type=checkbox]").each(function(){
@@ -555,6 +593,27 @@ $(function(){
 	function disables(){
 		$("#modal-disables").modal("show");
 	}
+  //复制爬虫
+  function comfirm_spidercopy(){
+    var code = $("#spidercopy").val();
+    if(code != ""){
+      $.ajax({
+				url:"/center/spider/copy",
+				type:"post",
+				data:{"code": code},
+				success: function(r){
+				  if(r&&r.success){
+            $("#modal-spidercopy").modal("hide");
+            window.open("/center/spider/edit/"+r.encode+"__")
+          }else{
+            alert(r.text)
+          }
+				}
+			})
+    }else{
+      alert("爬虫不能为空！")
+    }
+  }
 	//确认作废
 	function comfirm_disables(){
 		$("#modal-disables").modal("hide");
@@ -604,6 +663,9 @@ $(function(){
 	function disables_cancel(){
 		$("#modal-disables").modal("hide");
 	}
+  function spidercopy_cancel(){
+		$("#modal-spidercopy").modal("hide");
+	}
 	//分配爬虫
 	function assign_save(){
 		$("#modal-assign").modal("hide");

src/web/templates/mytask.html

@@ -201,10 +201,18 @@
 				setCookie("mytask",$("[type=search]").val());
 			})
    	});
-    //每次登陆提示还有多少条未通过的任务  
+    //每次登陆提示还有多少条未通过的任务和错误爬虫个数  
     var failedtasknum = {{.T.failedtasknum}};
+    var errluanum = {{.T.errluanum}};
+    var tips = "";
     if(failedtasknum>0){
-      showTip("你当前有未通过的任务"+failedtasknum.toString()+"条",5000);
+      tips = "你当前有未通过的任务"+failedtasknum.toString()+"条；";
+    }
+    if(errluanum>0){
+      tips = tips + "你当前有错误爬虫"+errluanum.toString()+"个";
+    }
+    if(tips!=""){
+       showTip(tips,5000);
     }
 	});