新增周补采模块

src/front/front.go

@@ -85,7 +85,8 @@ type Front struct {
 	getCity xweb.Mapper `xweb:"/center/getCity"` //获取城市
 
 	//补采信息
-	supplementList xweb.Mapper `xweb:"/center/supplement/list"` //脚本管理
+	supplementDayList  xweb.Mapper `xweb:"/center/supplement/daylist"`  //日补采列表
+	supplementWeekList xweb.Mapper `xweb:"/center/supplement/weeklist"` //周补采列表
 }
 
 const Sp_state_0, Sp_state_1, Sp_state_2, Sp_state_3, Sp_state_4, Sp_state_5, Sp_state_6, Sp_state_7, Sp_state_8, Sp_state_9, Sp_state_10 = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 //0待完成，1待审核，2打回，3发布，4作废，5已上架，6已下架，7无发布，8需登录，9转python，10已删除

src/front/supplement.go

@@ -4,10 +4,11 @@ import (
 	"fmt"
 	qu "qfw/util"
 	"strings"
+	"time"
 	u "util"
 )
 
-func (f *Front) SupplementList() {
+func (f *Front) SupplementDayList() {
 	auth := qu.IntAll(f.GetSession("auth"))
 	if auth != u.Role_Admin {
 		f.ServeJson("没有权限！")
@@ -25,6 +26,7 @@ func (f *Front) SupplementList() {
 				"$gte": starttime,
 				"$lte": starttime + 12*3600,
 			},
+			"stype": "day",
 		}
 		if search != "" {
 			query["$or"] = []interface{}{
@@ -40,11 +42,68 @@ func (f *Front) SupplementList() {
 			orderType = -1
 		}
 		sort = fmt.Sprintf(sort, orderName, orderType)
-		qu.Debug("supplement query:", query)
+		qu.Debug("supplement day query:", query)
 		count := u.MgoS.Count("spider_supplement", query)
 		list, _ := u.MgoS.Find("spider_supplement", query, sort, nil, false, start, limit)
 		f.ServeJson(map[string]interface{}{"draw": draw, "data": list, "recordsFiltered": count, "recordsTotal": count})
 	} else {
-		f.Render("supplement.html", &f.T)
+		f.Render("supplement_day.html", &f.T)
 	}
 }
+
+func (f *Front) SupplementWeekList() {
+	auth := qu.IntAll(f.GetSession("auth"))
+	if auth != u.Role_Admin {
+		f.ServeJson("没有权限！")
+		return
+	}
+	if f.Method() == "POST" {
+		start, _ := f.GetInteger("start")
+		limit, _ := f.GetInteger("length")
+		draw, _ := f.GetInteger("draw")
+		searchStr := f.GetString("search[value]") //搜索内容
+		search := strings.TrimSpace(searchStr)
+		starttime, _ := f.GetInt("starttime") //当天凌晨时间
+		starttime = getMondayInt64(starttime)
+		query := map[string]interface{}{
+			"comeintime": map[string]interface{}{
+				"$gte": starttime,
+				"$lte": starttime + 12*3600,
+			},
+			"stype": "week",
+		}
+		if search != "" {
+			query["$or"] = []interface{}{
+				map[string]interface{}{"spidercode": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"channel": map[string]interface{}{"$regex": search}},
+			}
+		}
+		sort := `{"%s":%d}`
+		orderIndex := f.GetString("order[0][column]")
+		orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
+		orderType := 1
+		if f.GetString("order[0][dir]") != "asc" {
+			orderType = -1
+		}
+		sort = fmt.Sprintf(sort, orderName, orderType)
+		qu.Debug("supplement week query:", query)
+		count := u.MgoS.Count("spider_supplement", query)
+		list, _ := u.MgoS.Find("spider_supplement", query, sort, nil, false, start, limit)
+		f.ServeJson(map[string]interface{}{"draw": draw, "data": list, "recordsFiltered": count, "recordsTotal": count})
+	} else {
+		f.Render("supplement_week.html", &f.T)
+	}
+}
+
+func getMondayInt64(t int64) int64 {
+	// 获取当前时间的星期几
+	currentTime := time.Unix(t, 0)
+	weekday := int(currentTime.Weekday())
+	// 计算今天距离周一的天数差
+	daysSinceMonday := (weekday + 6) % 7
+	// 计算本周一的时间戳
+	nowTime := currentTime.AddDate(0, 0, -daysSinceMonday)
+	timeStr := qu.FormatDate(&nowTime, qu.Date_Short_Layout)
+	result, _ := time.ParseInLocation(qu.Date_Short_Layout, timeStr, time.Local)
+	return result.Unix()
+}

src/timetask/timetask.go

@@ -28,13 +28,13 @@ func TimeTask() {
 	c.AddFunc("0 */10 * * * *", SpiderMoveEvent) //7000节点转增量爬虫
 }
 
-//检测创建任务失败的爬虫
+// 检测创建任务失败的爬虫
 func CheckCreateTask() {
 	defer qu.Catch()
 	qu.Debug("开始检测任务创建...")
 	query := map[string]interface{}{
 		"comeintime": map[string]interface{}{
-			"$gte": GetTime(0),
+			"$gte": util.GetTime(0),
 		},
 	}
 	codes := []string{}
@@ -58,15 +58,6 @@ func CheckCreateTask() {
 	}
 }
 
-//获取第day天凌晨的时间戳
-func GetTime(day int) int64 {
-	defer qu.Catch()
-	nowTime := time.Now().AddDate(0, 0, day)
-	timeStr := qu.FormatDate(&nowTime, qu.Date_Short_Layout)
-	t, _ := time.ParseInLocation(qu.Date_Short_Layout, timeStr, time.Local)
-	return t.Unix()
-}
-
 func UpdateSiteInfo() {
 	defer qu.Catch()
 	qu.Debug("定时更新站点信息开始...")
@@ -120,7 +111,7 @@ func UpdateCodeHeart() {
 	qu.Debug("定时更新爬虫心跳信息完成...")
 }
 
-//监测爬虫由历史转增量时未成功的
+// 监测爬虫由历史转增量时未成功的
 func CheckLuaMove() {
 	defer qu.Catch()
 	qu.Debug("开始检测爬虫节点移动...")

src/util/util.go

@@ -534,3 +534,12 @@ func (m *MyWrite) Reader(p []byte) (n int, err error) {
 	n, err = m.Byte.Read(p)
 	return
 }
+
+// 获取第day天凌晨的时间戳
+func GetTime(day int) int64 {
+	defer qu.Catch()
+	nowTime := time.Now().AddDate(0, 0, day)
+	timeStr := qu.FormatDate(&nowTime, qu.Date_Short_Layout)
+	t, _ := time.ParseInLocation(qu.Date_Short_Layout, timeStr, time.Local)
+	return t.Unix()
+}

src/web/templates/head.html

@@ -621,7 +621,16 @@
 		    <li data="index_pcqy"><a href="/center/luamove"><i class="glyphicon glyphicon-move"></i> <span>爬虫迁移管理</span></a></li>
 <!--		    <li data="index_newrwgl"><a href="/center/newtask/tasklist"><i class="glyphicon glyphicon glyphicon-tasks"></i> <span>新版任务管理</span></a></li>-->
 			{{if eq (session "identity") 1}}
-			<li data="index_bcgl"><a href="/center/supplement/list"><i class="glyphicon glyphicon-list-alt"></i> <span>补采管理</span></a></li>
+			  <li class="treeview" data="index_bcgl">
+				  <a href="#"><i class="glyphicon glyphicon-list-alt"></i> <span>补采管理</span>
+					  <span class="pull-right-container"><i class="fa fa-angle-left pull-right"></i></span>
+				  </a>
+				  <ul class="treeview-menu">
+					  <li><a href="/center/supplement/daylist"><i class="fa fa-circle-o"></i>日补采</a></li>
+					  <li><a href="/center/supplement/weeklist"><i class="fa fa-circle-o"></i>周补采</a></li>
+				  </ul>
+			  </li>
+<!--			<li data="index_bcgl"><a href="/center/supplement/daylist"><i class="glyphicon glyphicon-list-alt"></i> <span>补采管理</span></a></li>-->
 			{{end}}
 		{{else if eq (session "auth") 2}}
 			<li data="index"><a href="/center"><i class="glyphicon glyphicon-eye-open"></i> <span>爬虫审核</span></a></li>

src/web/templates/supplement.html → src/web/templates/supplement_day.html

@@ -2,7 +2,7 @@
 <div class="content-wrapper">
     <section class="content-header">
         <h1>
-            补采列表
+            日补采列表
             <small>
             </small>
         </h1>
@@ -25,6 +25,7 @@
                         <th>补采量</th>
                         <th>成功量</th>
                         <th>失败量</th>
+                        <th>有效量</th>
                         <th>时间异常量</th>
                     </tr>
                     </thead>
@@ -50,7 +51,7 @@
                 "info": true,
                 "autoWidth": true,
                 "ajax": {
-                    "url": "/center/supplement/list",
+                    "url": "/center/supplement/daylist",
                     "type": "POST"
                 },
                 "columns": [
@@ -61,6 +62,7 @@
                     { "data": "savenum"},
                     { "data": "success"},
                     { "data": "failed"},
+                    { "data": "effectivenum"},
                     { "data": "ptimezeronum"}
                 ],
                 "fnDrawCallback": function(table) {
@@ -79,11 +81,11 @@
                         if (st != null) {
                             var s = st.toLocaleDateString()
                             var sc = Date.parse(new Date(s).toString())/1000
-                            e.starttime = sc - 2*3600;
+                            e.starttime = sc;
                         }
                     }else{
                         var nst = Date.parse(new Date().toLocaleDateString())/1000;
-                        e.starttime = nst - 2*3600;
+                        e.starttime = nst;
                     }
                     // console.log(e.starttime)
                 }
@@ -124,7 +126,7 @@
             //     setCookie("request",JSON.stringify(request));
             //      console.log("1",getCookie("request"))
             // })
-            common.setActive("index_fbgl");
+            common.setActive("index_bcgl");
         })
     </script>
 </div>

src/web/templates/supplement_week.html

@@ -0,0 +1,133 @@
+{{include "head.html"}}
+<div class="content-wrapper">
+    <section class="content-header">
+        <h1>
+            周补采列表
+            <small>
+            </small>
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="#"><i class="fa fa-dashboard"></i> 首页</a></li>
+            <li class="active">列表</li>
+        </ol>
+    </section>
+    <section class="content">
+        <div class="box">
+            <!-- /.box-header -->
+            <div class="box-body">
+                <table id="supplement" class="table table-bordered table-striped">
+                    <thead>
+                    <tr>
+                        <th>爬虫</th>
+                        <th>网站</th>
+                        <th>栏目</th>
+                        <th>作者</th>
+                        <th>补采量</th>
+                        <th>成功量</th>
+                        <th>失败量</th>
+                        <th>有效量</th>
+                        <th>时间异常量</th>
+                    </tr>
+                    </thead>
+                </table>
+            </div>
+            <!-- /.box-body -->
+        </div>
+    </section>
+    <script>
+        $(function(){
+            ttable=$('#supplement').DataTable({
+                "language": {
+                    "url": "/js/dataTables.chinese.lang"
+                },
+                "columnDefs": [
+                    { "orderable": false, "targets": [0,1,2] },
+                ],
+                "order": [[4,"desc"]],
+                "processing": true,
+                "serverSide": true,
+                "searching": true,
+                "ordering": true,
+                "info": true,
+                "autoWidth": true,
+                "ajax": {
+                    "url": "/center/supplement/weeklist",
+                    "type": "POST"
+                },
+                "columns": [
+                    { "data": "spidercode"},
+                    { "data": "site"},
+                    { "data": "channel"},
+                    { "data": "modifyuser"},
+                    { "data": "savenum"},
+                    { "data": "success"},
+                    { "data": "failed"},
+                    { "data": "effectivenum"},
+                    { "data": "ptimezeronum"}
+                ],
+                "fnDrawCallback": function(table) {
+                    $("ul.pagination").prepend("&nbsp;&nbsp;&nbsp;转到第 <input type='text' id='changePage'   style='width:20px;'> 页    <a type='text' href='javascript:void(0);' id='dataTable-btn' style='text-align:center'>GO</a>");
+                    $('#dataTable-btn').click(function(e) {
+                        var redirectpage=0;
+                        if($("#changePage").val() && $("#changePage").val() > 0) {
+                            var redirectpage = $("#changePage").val() - 1;
+                        }
+                        ttable.page(redirectpage).draw(false);
+                    });
+                },
+                "fnServerParams": function (e) {
+                    if($('#starttime').length !=0){
+                        var st = $('#starttime').datepicker('getDate')
+                        if (st != null) {
+                            var s = st.toLocaleDateString()
+                            var sc = Date.parse(new Date(s).toString())/1000
+                            e.starttime = sc;
+                        }
+                    }else{
+                        var nst = Date.parse(new Date().toLocaleDateString())/1000;
+                        e.starttime = nst;
+                    }
+                    // console.log(e.starttime)
+                }
+            });
+            ttable.on('init.dt', function () {
+                var selectTime = "<div style='margin-top: -4px'><label for='name'>时间：</label>"+
+                    "<span class='input-group date date-picker' id='starttime' data-provide='datepicker'>"+
+                    "<input type='text' class='form-control form-filter input-sm' readonly name='starttime' placeholder='开始日期' />"+
+                    "<span class='input-group-addon'>"+
+                    "<i class='fa fa-calendar'></i>"+
+                    "</span>"+
+                    "</span>"+
+                    "</div>&nbsp;"
+                $("#supplement_filter").prepend(selectTime);
+                $("#supplement_filter").prepend("&nbsp;");
+                // $("#spiderwarn_length").parent().css("width", "10%")
+                $("#supplement_filter").css({"display": "flex","text-align": "right","align-items": "center","justify-content": "flex-end"})
+
+                $('.date-picker').datepicker({
+                    language: 'zh-CN',
+                    autoclose: true,
+                    clearBtn: true, //清除按钮
+                    todayBtn: false, //今日按钮
+                    format: "yyyy-mm-dd"
+                });
+                $('#starttime').datepicker('setDate',new Date().toLocaleDateString());//设置初始时间
+                $('#starttime').datepicker().on('changeDate', function (e) {
+                    ttable.ajax.reload();
+                });
+            });
+            //发送到服务器的数据先进行处理
+            // ttable.on('preXhr.dt', function ( e, settings, data ) {
+            //     var request = {};
+            //     var search = data.search.value;//搜索框值
+            //     var starttime = data.starttime;//时间
+            //     request["search"] = search;
+            //     request["starttime"] = starttime;
+            //     setCookie("request",JSON.stringify(request));
+            //      console.log("1",getCookie("request"))
+            // })
+            common.setActive("index_bcgl");
+        })
+    </script>
+</div>
+{{include "bottom.html"}}