xg

src/spider/spider.go

@@ -250,7 +250,9 @@ func (s *Spider) DownListPageItem() (errs interface{}) {
 		max = 100 //设置最大页为100
 	}
 	for ; start <= max && !s.Stop; start++ {
-		UpdateHeart(s.Name, s.Channel, s.Code, s.MUserName, "list") //记录所有节点列表页心跳
+		if !s.Stop { //在下载详情页时爬虫下架，此时不再存心跳信息
+			UpdateHeart(s.Name, s.Channel, s.Code, s.MUserName, "list") //记录所有节点列表页心跳
+		}
 		//qu.Debug("重复页：", repeatPageNum, "	配置最大页：", tmpMax, "	最终最大页：", max, "	当前页：", start, "重复次数：", repeatPageTimes)
 		if start > tmpMax && isRunRepeatList && repeatPageTimes >= 5 { //重复次数超过5次，不再翻页
 			break
@@ -527,7 +529,9 @@ func (s *Spider) DownloadDetailItem(p interface{}, num *int) {
 		SaveHighListPageData(paramdata, href, num)
 		return
 	} else {
-		UpdateHeart(s.Name, s.Channel, s.Code, s.MUserName, "detail") //记录modal=0老模式采集三级页心跳
+		if !s.Stop {
+			UpdateHeart(s.Name, s.Channel, s.Code, s.MUserName, "detail") //记录modal=0老模式采集三级页心跳
+		}
 		isExist, _ := util.ExistRedis("title_repeat_judgement", 0, "url_repeat_"+href)
 		if isExist { //更新redis生命周期
 			util.PutRedis("title_repeat_judgement", 0, "url_repeat_"+href, href, 3600*24*30)
@@ -572,7 +576,9 @@ func (s *Spider) DownloadDetailItem(p interface{}, num *int) {
 	if t1 > time.Now().Unix() { //防止发布时间超前
 		data["publishtime"] = time.Now().Unix()
 	}
-	UpdateHeart(s.Name, s.Channel, s.Code, s.MUserName, "detailexcute") //记录modal=0老模式采集到数据心跳
+	if !s.Stop {
+		UpdateHeart(s.Name, s.Channel, s.Code, s.MUserName, "detailexcute") //记录modal=0老模式采集到数据心跳
+	}
 	delete(data, "state")
 	delete(data, "exit")
 	delete(data, "checkpublishtime")