|
|
@@ -132,41 +132,26 @@ class ItemBuffer(threading.Thread):
|
|
|
|
|
|
elif isinstance(data, FailedTaskItem):
|
|
|
data.queue_name = self._tab_items # 采集任务队列名称
|
|
|
- failed_times = data.to_dict.pop('failed_times', 0)
|
|
|
- failed_retries = data.to_dict.pop('failed_retries', 0)
|
|
|
- if failed_retries >= setting.SPIDER_MAX_RETRY_TIMES:
|
|
|
+ if data.failed_retries >= setting.SPIDER_MAX_RETRY_TIMES:
|
|
|
state = 4 # 待采集任务停止采集状态[4=停止采集]
|
|
|
- # 更新完成采集的任务状态
|
|
|
- update_item = UpdateItem(
|
|
|
- state=state,
|
|
|
- pyuuid=data.pyuuid,
|
|
|
- update_at=update_at,
|
|
|
- )
|
|
|
- update_item.update_key = ['state', 'update_at']
|
|
|
- update_item.table_name = setting.TASK_REQUEST_PRODUCE
|
|
|
- update_items.append(update_item)
|
|
|
- # 保存失败的采集任务详情
|
|
|
+
|
|
|
+ '''记录失败的采集任务详情'''
|
|
|
data.state = state
|
|
|
- data.failed_times = failed_times
|
|
|
- data.failed_retries = failed_retries
|
|
|
data.create_at = update_at
|
|
|
failed_task_items.append(data)
|
|
|
else:
|
|
|
- update_item = UpdateItem(
|
|
|
- state=3, # 待采集任务失败采集状态[3=采集失败]
|
|
|
- pyuuid=data.pyuuid,
|
|
|
- update_at=update_at,
|
|
|
- failed_times=failed_times + 1,
|
|
|
- failed_retries=failed_retries + 1
|
|
|
- )
|
|
|
- update_item.update_key = [
|
|
|
- 'state',
|
|
|
- 'update_at',
|
|
|
- 'failed_times',
|
|
|
- 'failed_retries',
|
|
|
- ]
|
|
|
- update_item.table_name = setting.TASK_REQUEST_PRODUCE
|
|
|
- update_items.append(update_item)
|
|
|
+ state = 3 # 待采集任务失败采集状态[3=采集失败]
|
|
|
+
|
|
|
+ '''更新失败的采集任务状态'''
|
|
|
+ update_item = UpdateItem(
|
|
|
+ state=state,
|
|
|
+ pyuuid=data.pyuuid,
|
|
|
+ failed_retries=data.failed_retries,
|
|
|
+ update_at=update_at,
|
|
|
+ )
|
|
|
+ update_item.update_key = ['state', 'update_at', 'failed_retries']
|
|
|
+ update_item.table_name = setting.TASK_REQUEST_PRODUCE
|
|
|
+ update_items.append(update_item)
|
|
|
|
|
|
elif isinstance(data, Item):
|
|
|
if isinstance(data, BaseListItem):
|
