|
|
@@ -132,7 +132,8 @@ class ItemBuffer(threading.Thread):
|
|
|
elif isinstance(data, FailedTaskItem):
|
|
|
data.queue_name = self._tab_items # 采集任务队列名称
|
|
|
failed_times = data.to_dict.pop('failed_times', 0)
|
|
|
- if failed_times >= setting.SPIDER_MAX_RETRY_TIMES:
|
|
|
+ failed_retries = data.to_dict.pop('failed_retries', 0)
|
|
|
+ if failed_retries >= setting.SPIDER_MAX_RETRY_TIMES:
|
|
|
state = 4 # 待采集任务停止采集状态[4=停止采集]
|
|
|
# 更新完成采集的任务状态
|
|
|
update_item = UpdateItem(
|
|
|
@@ -146,6 +147,7 @@ class ItemBuffer(threading.Thread):
|
|
|
# 保存失败的采集任务详情
|
|
|
data.state = state
|
|
|
data.failed_times = failed_times
|
|
|
+ data.failed_retries = failed_retries
|
|
|
data.create_at = update_at
|
|
|
failed_task_items.append(data)
|
|
|
else:
|
|
|
@@ -153,9 +155,15 @@ class ItemBuffer(threading.Thread):
|
|
|
state=3, # 待采集任务失败采集状态[3=采集失败]
|
|
|
pyuuid=data.pyuuid,
|
|
|
update_at=update_at,
|
|
|
- failed_times=failed_times + 1
|
|
|
+ failed_times=failed_times + 1,
|
|
|
+ failed_retries=failed_retries + 1
|
|
|
)
|
|
|
- update_item.update_key = ['state', 'update_at', 'failed_times']
|
|
|
+ update_item.update_key = [
|
|
|
+ 'state',
|
|
|
+ 'update_at',
|
|
|
+ 'failed_times',
|
|
|
+ 'failed_retries',
|
|
|
+ ]
|
|
|
update_item.table_name = setting.TASK_REQUEST_PRODUCE
|
|
|
update_items.append(update_item)
|
|
|
|
