|
|
@@ -1,3 +1,4 @@
|
|
|
+import random
|
|
|
import time
|
|
|
|
|
|
import requests.exceptions
|
|
|
@@ -216,6 +217,7 @@ class DetailSpider:
|
|
|
self.crawl_response(response, item)
|
|
|
self._update_crawl_task(item["_id"], crawl_status='finished')
|
|
|
sc.crawl_counter(1)
|
|
|
+ next_task_interval = random.choice(range(3,9))
|
|
|
except YbwCrawlError as e:
|
|
|
if e.code == 10105:
|
|
|
# 抛出异常时,将es查询统计结果进行更新
|
