|
@@ -167,12 +167,14 @@ class ListSpider:
|
|
|
break
|
|
|
item_size = self.crawl_response(response, menu)
|
|
|
logger.info(f'[采集成功]{menu.channel}-{region_name}-第{page}页-{item_size}条')
|
|
|
+ sc.crawl_counter(item_size)
|
|
|
if item_size < page_size:
|
|
|
'''当前页面的发布的条数小于网站页面固定发布数量,下一页不存在数据,直接跳过'''
|
|
|
break
|
|
|
else:
|
|
|
crawl_total += 1
|
|
|
except (JyBasicException, Exception) as e:
|
|
|
+ sc.crawl_counter(0)
|
|
|
logger.error('[采集失败]{}-{}-第{}页, 错误类型:{}'.format(
|
|
|
menu.channel,
|
|
|
region_name,
|