|
@@ -394,10 +394,6 @@ func (s *Spider) DownListPageItem() (errs interface{}) {
|
|
}
|
|
}
|
|
downtimes = 0 //当前页下载无误,重置下载重试次数
|
|
downtimes = 0 //当前页下载无误,重置下载重试次数
|
|
util.TimeSleepFunc(100*time.Millisecond, TimeSleepChan)
|
|
util.TimeSleepFunc(100*time.Millisecond, TimeSleepChan)
|
|
- //临时用,更新开始页码,附件下载
|
|
|
|
- // mgu.Update("luaconfig2", "editor", "editor", `{"code":"`+s.Code+`"}`,
|
|
|
|
- // map[string]interface{}{"$inc": map[string]interface{}{"param_common.4": 1}},
|
|
|
|
- // true, false)
|
|
|
|
}
|
|
}
|
|
nowTime := time.Now()
|
|
nowTime := time.Now()
|
|
sDate := qu.FormatDate(&nowTime, qu.Date_Short_Layout)
|
|
sDate := qu.FormatDate(&nowTime, qu.Date_Short_Layout)
|
|
@@ -435,6 +431,7 @@ func (s *Spider) DownListPageItem() (errs interface{}) {
|
|
"date": sDate,
|
|
"date": sDate,
|
|
"spidercode": s.Code,
|
|
"spidercode": s.Code,
|
|
}
|
|
}
|
|
|
|
+ logger.Info(s.Code, "本轮列表页采集详情:", downloadAllNum, repeatAllNum, start)
|
|
Mgo.Update("spider_downloadrate", query, map[string]interface{}{
|
|
Mgo.Update("spider_downloadrate", query, map[string]interface{}{
|
|
"$set": set,
|
|
"$set": set,
|
|
"$inc": inc,
|
|
"$inc": inc,
|