心跳监控/重复连接附件过滤

src/front/front.go

@@ -57,13 +57,15 @@ type Front struct {
 	oldedit     xweb.Mapper `xweb:"/center/oldedit"`         //老文件编辑
 	findName    xweb.Mapper `xweb:"/center/findname"`        //即时查询名称
 	checkrepeat xweb.Mapper `xweb:"/center/spider/isrepeat"` //脚本代码判重
-	Base        Base
-	OtherBase   OtherBase
-	Step1       Step1
-	Step2       Step2
-	Step3       Step3
-	StepRe3     StepRe3
-	U           U
+	heart       xweb.Mapper `xweb:"/center/heart"`           //心跳监控
+
+	Base      Base
+	OtherBase OtherBase
+	Step1     Step1
+	Step2     Step2
+	Step3     Step3
+	StepRe3   StepRe3
+	U         U
 
 	luaList       xweb.Mapper `xweb:"/center/lualist.html"`       //脚本管理
 	user          xweb.Mapper `xweb:"/center/user.html"`          //用户管理

src/front/spider.go

@@ -1236,6 +1236,69 @@ func (f *Front) LuaList() {
 	}
 }
 
+//心跳监控
+func (f *Front) Heart() {
+	auth := qu.IntAll(f.GetSession("auth"))
+	if auth != role_admin {
+		f.ServeJson("没有权限！")
+		return
+	}
+	if f.Method() == "POST" {
+		// start, _ := f.GetInteger("start")
+		// limit, _ := f.GetInteger("length")
+		// draw, _ := f.GetInteger("draw")
+		searchStr := f.GetString("search[value]")
+		search := strings.TrimSpace(searchStr) //只能搜索code
+		//qu.Debug("search:", search, "start:", start, "limit:", limit, "draw:", draw)
+		result := []map[string]interface{}{}
+		if search != "" {
+			if tmp, ok := spinfos.Load(search); ok {
+				info := tmp.(*spinfo)
+				result = append(result, map[string]interface{}{
+					"modifytime": info.lastHeartbeat,
+					"lstate":     info.lstate,
+				})
+			}
+		} else {
+			timeCode := map[string]string{}
+			spinfos.Range(func(k, v interface{}) bool {
+				code := k.(string)
+				info, ok := v.(*spinfo)
+				if ok {
+					timeCode[code] = info.lastHeartbeat
+				}
+				return true
+			})
+			vs := u.MapStringValueSort(timeCode)
+			n := 0
+			for _, c := range vs.Keys {
+				if strings.HasPrefix(c, "jzgc_") { //过滤企业爬虫
+					continue
+				}
+				if n > 300 {
+					break
+				}
+				d := mgdb.FindOne("luaconfig", map[string]interface{}{"code": c})
+				state := qu.IntAll((*d)["state"])
+				if state != 5 && state != 6 {
+					continue
+				}
+				n++
+				result = append(result, map[string]interface{}{
+					"code":       c,
+					"modifytime": timeCode[c],
+					"state":      (*d)["state"],
+					"event":      (*d)["event"],
+				})
+			}
+			vs = nil
+		}
+		f.ServeJson(map[string]interface{}{"data": result})
+	} else {
+		f.Render("heart.html")
+	}
+}
+
 //爬虫信息
 type spinfo struct {
 	code                                   string

src/res/util/comm.lua

@@ -517,6 +517,7 @@ function common.getFilesLinkByTag(href,tags,content,withend)
 	--dhtml=dhtml.."<a href='/123.doc'>123.doc</a>"
 	local alist = findListHtml(tags.." a", content)
 	local flist={}
+	local ahref={}--对href进行判重处理
 	for k,v in pairs(alist) do
 		local item={}
 		item["href"]="a:eq("..tostring(k-1).."):attr(href)"
@@ -550,8 +551,16 @@ function common.getFilesLinkByTag(href,tags,content,withend)
 		end
 	
 		if statehref~=nil and item["title"]~="" then
-			
-			table.insert(flist,item)
+			local has = false
+			for _,tmphref in pairs(ahref) do
+				if tmphref == item["href"] then
+					has = true
+				end
+			end
+			if not has then
+				table.insert(ahref,item["href"])
+				table.insert(flist,item)
+			end
 		end
 	end
 	return flist
@@ -568,7 +577,6 @@ function common.getFileAttachmentsArrayWithTag(href,tags,content,withend,param,h
 	--local nameTypeArr={"jpg","JPG","bid","pdf","PDF","doc","docx","xls","xlsx","zip","rar","swf","DOCX","DOC","PDF","XLS","XLSX","ZIP","RAR","SWF"}
 	local titleAndHrefList = common.getFilesLinkByTag(href,tags,content,withend)
 	for i,v in ipairs(titleAndHrefList) do
-		
 		local end_type = string.find(v["title"],v["ftype"].."$")
 		local file_name = ""
 		if end_type==nil or end_type=="" then

src/timetask/timetask.go

@@ -29,7 +29,6 @@ func TimeTask() {
 	c.AddFunc("0 20 9 ? * MON-FRI", CheckCreateTask)
 	c.AddFunc("0 0 */1 ? * *", CheckLuaMove)
 	c.AddFunc("0 30 23 * * *", UpdateSiteInfo) //定时更新站点信息
-	c.Start()
 }
 
 //监测爬虫由历史转增量时未成功的

src/util/util.go

@@ -5,6 +5,7 @@ import (
 	mgo "mongodb"
 	qu "qfw/util"
 	"regexp"
+	"sort"
 	sp "spiderutil"
 	"strings"
 	"time"
@@ -301,3 +302,41 @@ func GetLuasInfoBySite(site string) (domain, status, event, platform, area, city
 	}
 	return
 }
+
+type StringValSorter struct {
+	Keys []string
+	Vals []string
+}
+
+func MapStringValueSort(m map[string]string) *StringValSorter {
+	vs := NewStringValSorter(m)
+	vs.Sort()
+	return vs
+}
+
+func NewStringValSorter(m map[string]string) *StringValSorter {
+	vs := &StringValSorter{
+		Keys: make([]string, 0, len(m)),
+		Vals: make([]string, 0, len(m)),
+	}
+	for k, v := range m {
+		vs.Keys = append(vs.Keys, k)
+		vs.Vals = append(vs.Vals, v)
+	}
+	return vs
+}
+
+func (vs *StringValSorter) Sort() {
+	sort.Sort(vs)
+}
+
+func (vs *StringValSorter) Len() int {
+	return len(vs.Vals)
+}
+func (vs *StringValSorter) Less(i, j int) bool {
+	return vs.Vals[i] < vs.Vals[j]
+}
+func (vs *StringValSorter) Swap(i, j int) {
+	vs.Vals[i], vs.Vals[j] = vs.Vals[j], vs.Vals[i]
+	vs.Keys[i], vs.Keys[j] = vs.Keys[j], vs.Keys[i]
+}

src/web/templates/head.html

@@ -15,6 +15,7 @@
   <link rel="stylesheet" href="https://cdn.staticfile.org/ionicons/2.0.1/css/ionicons.css">
   <!-- Theme style -->
   <link rel="stylesheet" href="/css/AdminLTE.min.css">
+   <link rel="stylesheet" href="/css/font-awesome.min.css">
   <link rel="stylesheet" href="/dist/css/skins/skin-blue.min.css">
 	<link rel="stylesheet" href="/plugins/datatables/css/dataTables.bootstrap.css">
 	<link rel="stylesheet" href="/css/style.css">
@@ -558,6 +559,7 @@
       <li data="index_pcdr"><a href="/center/importdata"><i class="glyphicon glyphicon-import"></i> <span>爬虫导入</span></a></li>
       <li data="index_site"><a href="/center/site/sitelist"><i class="glyphicon glyphicon-home"></i> <span>站点列表</span></a></li>
       <li data="index_pcqy"><a href="/center/luamove"><i class="glyphicon glyphicon-move"></i> <span>爬虫迁移管理</span></a></li>
+      <li data="index_heart"><a href="/center/heart"><i class="fa fa-heartbeat"></i> <span>心跳监控</span></a></li>
 		{{else if eq (session "auth") 2}}
 			<li data="index"><a href="/center"><i class="glyphicon glyphicon-eye-open"></i> <span>爬虫审核</span></a></li>
 			<li data="index_rwsh"><a href="/center/task/audit"><i class="glyphicon glyphicon glyphicon-tasks"></i> <span>任务审核</span></a></li>

src/web/templates/heart.html

@@ -0,0 +1,127 @@
+{{include "head.html"}}
+<div class="content-wrapper">
+<section class="content-header">
+   <h1>
+    爬虫列表
+	 <small>
+				<!--<a class="btn btn-default btn-sm" id="delRedis" onclick="delRedis()">清理Redis</a>-->
+	 </small>
+   </h1>
+   <ol class="breadcrumb">
+     <li><a href="#"><i class="fa fa-dashboard"></i> 首页</a></li>
+     <li class="active">列表</li>
+   </ol>
+ </section>
+ <section class="content">
+	<div class="box">
+            <!-- /.box-header -->
+            <div class="box-body">
+              <table id="spider" class="table table-bordered table-striped">
+                <thead>
+                <tr>
+                  <th>节点</th>
+          				<th>爬虫</th>
+          				<th>心跳</th>
+                  <th class="hidden-xs">操作</th>
+                </tr>
+                </thead>
+              </table>
+            </div>
+            <!-- /.box-body -->
+          </div>
+</section>
+
+<script>
+	var first=1;
+	$(function(){
+		taskConst=false;
+		ttable=$('#spider').DataTable({
+		"language": {
+              "url": "/js/dataTables.chinese.lang"
+          },
+		"columnDefs": [
+		   
+		],
+		"processing":true,
+		"serverSide": true,
+		"searching": true,
+		"ordering": false,
+		"info": false,
+    "paging": false,
+		"autoWidth": true,
+		"ajax": {
+			"url": "/center/heart",
+			"type": "POST"
+		 },
+		"columns": [
+			{ "data": "code"},
+      { "data": "event"},
+			{ "data": "modifytime"},
+			{ "data": "state",render:function(val,a,row){
+				var div=$("<div><div class=\"btn-group\"></div></div>")
+				if(val==5){
+					var dbutton=$('<button class="btn btn-sm btn-danger" onclick="myconfirm(this,\'下架\');">下架</button>');
+					dbutton.attr("hrefs","/center/spider/upstate?code="+row["code"]+"&state=6");
+					div.find(".btn-group").append(dbutton);
+				}else if(val==6){
+					var dbutton=$('<button class="btn btn-sm btn-warning" onclick="myconfirm(this,\'上架\');">上架</button>');
+					dbutton.attr("hrefs","/center/spider/upstate?code="+row["code"]+"&state=5");
+					div.find(".btn-group").append(dbutton);
+				}
+				return div.html()
+			}}
+    ],
+	});
+		common.setActive("index_heart");
+		ttable.on( 'init.dt', function () {
+			var search=getCookie("input");
+			if(search){
+				$("[type=search]").val(search);
+			}
+      /**
+			var opt="<option value='-1'>全部</option>"+
+				"<option value='3'>已通过</option>"+
+				"<option value='5'>已上架</option>"+
+				"<option value='6'>已下架</option>";
+			var select="<div class='form-group'><label for='name'>状态：</label>"+
+				"<select id='type_state' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+				opt+
+				"</select></div>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"
+			var opt2="<option value='-1'>全部</option>"
+			var events={{.T.events}}
+			for(k in events){
+				opt2+="<option value='"+events[k]+"'>"+events[k]+"</option>"
+			}
+			var select2="<div class='form-group'><label for='name'>节点：</label>"+
+				"<select id='eventid' onchange='checkclick(this.value)' class='form-control input-sm'>"+
+				opt2+
+				"</select></div>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"
+			$("#spider_filter").prepend(select);
+			$("#spider_filter").prepend(select2);
+      */
+		});
+	})
+	
+	function myconfirm(me,content){
+		showConfirm("确定"+content+"？", function() {
+			common.maskShow("正在"+content+"...");
+			$.ajax({
+				url:$(me).attr("hrefs"),
+				type:"post",
+				success:function(r){
+					common.maskHide();
+					if(r["err"]==""){
+						showTip(content+"成功", 500, function() {});
+						ttable.ajax.reload(null,false);
+					}else{
+						showMsg(r["err"], function() {});
+					}
+				}
+			})
+		});
+  }
+	
+	
+</script>
+</div>
+{{include "bottom.html"}}