|
@@ -377,7 +377,7 @@ class Scheduler(threading.Thread):
|
|
|
self._begin_callback()
|
|
|
|
|
|
for parser in self._parsers:
|
|
|
- parser.start_callback()
|
|
|
+ parser.start_callback() # 任务开始回调
|
|
|
|
|
|
def spider_end(self):
|
|
|
if self._end_callback: # 任务结束回调
|
|
@@ -385,17 +385,15 @@ class Scheduler(threading.Thread):
|
|
|
|
|
|
for parser in self._parsers:
|
|
|
if not self._keep_alive:
|
|
|
- parser.close() # 爬虫可自定义close
|
|
|
+ parser.close() # 爬虫自定义 close
|
|
|
|
|
|
parser.end_callback() # 调用结束回调函数
|
|
|
|
|
|
if not self._keep_alive:
|
|
|
- # 关闭 webdriver 管理池
|
|
|
if Request.webdriver_pool:
|
|
|
- Request.webdriver_pool.close()
|
|
|
+ Request.webdriver_pool.close() # 关闭 webdriver 管理池
|
|
|
|
|
|
- # 关闭打点
|
|
|
- metrics.close()
|
|
|
+ metrics.close() # 关闭打点
|
|
|
else:
|
|
|
metrics.flush()
|
|
|
|