|
|
@@ -219,7 +219,10 @@ class DetailSpider:
|
|
|
sc.crawl_counter(1)
|
|
|
next_task_interval = random.choice(range(3,9))
|
|
|
except (YbwCrawlError, Exception) as e:
|
|
|
- if getattr(e, 'code', None) == 10105:
|
|
|
+ if getattr(e, 'code', None) is None:
|
|
|
+ err = YbwCrawlError(unknown_err=e)
|
|
|
+ sc.err_record(err)
|
|
|
+ elif e.code == 10105:
|
|
|
# 抛出异常时,将es查询统计结果进行更新
|
|
|
self._update_crawl_task(item["_id"], count=item['count'])
|
|
|
logger.info('[重复数据]{}-{}'.format(item['title'], item['publishtime']))
|
