|
|
@@ -8,6 +8,7 @@ import (
|
|
|
"net/smtp"
|
|
|
qu "qfw/util"
|
|
|
mgdb "qfw/util/mongodb"
|
|
|
+ mgu "qfw/util/mongodbutil"
|
|
|
sp "spiderutil"
|
|
|
"strconv"
|
|
|
"strings"
|
|
|
@@ -29,6 +30,7 @@ func TimeTask() {
|
|
|
c.AddFunc("0 20 9 ? * MON-FRI", CheckCreateTask)
|
|
|
c.AddFunc("0 0 */1 ? * *", CheckLuaMove)
|
|
|
c.AddFunc("0 30 23 * * *", UpdateSiteInfo) //定时更新站点信息
|
|
|
+ c.AddFunc("0 0 23 * * *", UpdateCodeHeart) //定时更新爬虫心跳信息
|
|
|
}
|
|
|
|
|
|
//监测爬虫由历史转增量时未成功的
|
|
|
@@ -181,3 +183,18 @@ func UpdateSiteInfo() {
|
|
|
qu.Debug("定时更新站点信息完成...")
|
|
|
|
|
|
}
|
|
|
+
|
|
|
+func UpdateCodeHeart() {
|
|
|
+ defer qu.Catch()
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "state": map[string]interface{}{
|
|
|
+ "$gt": 6,
|
|
|
+ },
|
|
|
+ }
|
|
|
+ list, _ := util.MgoE.Find("luaconfig", query, nil, map[string]interface{}{"code": 1}, false, -1, -1)
|
|
|
+ for _, l := range *list {
|
|
|
+ mgu.Update("spider_heart", "spider", "spider", map[string]interface{}{"code": l["code"]}, map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{"del": true},
|
|
|
+ }, false, false)
|
|
|
+ }
|
|
|
+}
|
