|
|
@@ -876,9 +876,10 @@ func (s *Spider) DownloadDetail(stype string) {
|
|
|
UpdateHeart(s.Name, s.Channel, s.Code, s.MUserName, "detail") //记录modal=1采集三级页心跳
|
|
|
}
|
|
|
countNum := Mgo.Count("spider_highlistdata", q) //统计util.Config.DayNum天内未下载爬虫个数
|
|
|
+ logger.Info("Thread Info： Code:", s.SCode, " count:", countNum)
|
|
|
if countNum > 0 {
|
|
|
threadNum := countNum / util.Config.ThreadBaseNum //线程数
|
|
|
- logger.Info("Thread Info： Code:", s.SCode, " count:", countNum, " thread num:", threadNum)
|
|
|
+ //logger.Info("Thread Info： Code:", s.SCode, " count:", countNum, " thread num:", threadNum)
|
|
|
list, _ := Mgo.Find("spider_highlistdata", q, o, f, false, 0, 200)
|
|
|
if list != nil && len(*list) > 0 {
|
|
|
spChan := make(chan *Spider, threadNum+1) //初始化线程通道(+1表示基本的线程数)
|
