|
@@ -913,7 +913,7 @@ func (s *Spider) DownloadDetail(stype string) {
|
|
wg.Done()
|
|
wg.Done()
|
|
atomic.AddInt64(&AllThreadNum, -1)
|
|
atomic.AddInt64(&AllThreadNum, -1)
|
|
}()
|
|
}()
|
|
- if sp == nil { //初始化sp的时候sp可能为nil
|
|
|
|
|
|
+ if s.Stop || sp == nil { //爬虫下架或者初始化sp为nil时不再下载数据
|
|
return
|
|
return
|
|
}
|
|
}
|
|
_id := tmp["_id"]
|
|
_id := tmp["_id"]
|
|
@@ -1040,7 +1040,7 @@ func (s *Spider) DownloadDetail(stype string) {
|
|
data["dataging"] = 0
|
|
data["dataging"] = 0
|
|
data["iscompete"] = s.IsCompete //2021-11-01以后新增的爬虫不在展示原文链接(保存服务判断)
|
|
data["iscompete"] = s.IsCompete //2021-11-01以后新增的爬虫不在展示原文链接(保存服务判断)
|
|
Store(s.StoreMode, s.StoreToMsgEvent, s.Collection, s.CoverAttr, data, true)
|
|
Store(s.StoreMode, s.StoreToMsgEvent, s.Collection, s.CoverAttr, data, true)
|
|
- set := map[string]interface{}{"$set": map[string]interface{}{"state": 1, "updatetime": time.Now().Unix(), "date": data}} //下载成功state置为1
|
|
|
|
|
|
+ set := map[string]interface{}{"$set": map[string]interface{}{"state": 1, "updatetime": time.Now().Unix()}} //下载成功state置为1
|
|
update = append(update, query)
|
|
update = append(update, query)
|
|
update = append(update, set)
|
|
update = append(update, set)
|
|
spLock.Lock()
|
|
spLock.Lock()
|
|
@@ -1061,7 +1061,7 @@ func (s *Spider) DownloadDetail(stype string) {
|
|
sp.L.Close()
|
|
sp.L.Close()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
- if stype == "highlist" { //高性能模式下载完三级页数据,sp对象需要重载
|
|
|
|
|
|
+ if !s.Stop && stype == "highlist" { //高性能模式下载完三级页数据,sp对象需要重载
|
|
//重载主线程sp
|
|
//重载主线程sp
|
|
s.LoadScript(&s.Name, &s.Channel, &s.MUserName, s.Code, s.ScriptFile, true, false)
|
|
s.LoadScript(&s.Name, &s.Channel, &s.MUserName, s.Code, s.ScriptFile, true, false)
|
|
}
|
|
}
|