|
@@ -135,21 +135,29 @@ class ItemBuffer(threading.Thread):
|
|
|
if data.failed_retries >= setting.SPIDER_MAX_RETRY_TIMES:
|
|
|
state = 4 # 待采集任务停止采集状态[4=停止采集]
|
|
|
|
|
|
+ '''更新失败的采集任务状态'''
|
|
|
+ update_item = UpdateItem(
|
|
|
+ state=state,
|
|
|
+ pyuuid=data.pyuuid,
|
|
|
+ update_at=update_at,
|
|
|
+ failed_retries=data.failed_retries,
|
|
|
+ )
|
|
|
+ update_key = ['state', 'update_at', 'failed_retries']
|
|
|
+
|
|
|
'''记录失败的采集任务详情'''
|
|
|
data.state = state
|
|
|
data.create_at = update_at
|
|
|
failed_task_items.append(data)
|
|
|
else:
|
|
|
- state = 3 # 待采集任务失败采集状态[3=采集失败]
|
|
|
-
|
|
|
- '''更新失败的采集任务状态'''
|
|
|
- update_item = UpdateItem(
|
|
|
- state=state,
|
|
|
- pyuuid=data.pyuuid,
|
|
|
- failed_retries=data.failed_retries,
|
|
|
- update_at=update_at,
|
|
|
- )
|
|
|
- update_item.update_key = ['state', 'update_at', 'failed_retries']
|
|
|
+ '''更新失败的采集任务状态'''
|
|
|
+ update_item = UpdateItem(
|
|
|
+ state=3, # 待采集任务失败采集状态[3=采集失败]
|
|
|
+ pyuuid=data.pyuuid,
|
|
|
+ failed_retries=data.failed_retries,
|
|
|
+ )
|
|
|
+ update_key = ['state', 'failed_retries']
|
|
|
+
|
|
|
+ update_item.update_key = update_key
|
|
|
update_item.table_name = setting.TASK_REQUEST_PRODUCE
|
|
|
update_items.append(update_item)
|
|
|
|