|
|
@@ -6,6 +6,8 @@ import os
|
|
|
TAB_REQUESTS = "{redis_key}:z_requests"
|
|
|
# 详情待处理任务表模版
|
|
|
TAB_ITEMS = "{redis_key}:z_items"
|
|
|
+# 爬虫采集心跳
|
|
|
+TAB_SPIDER_HEARTBEAT = os.getenv("TAB_SPIDER_HEARTBEAT", "spider:z_heartbeats")
|
|
|
# 任务失败表
|
|
|
TAB_FAILED_REQUESTS = os.getenv("TAB_FAILED_REQUESTS", "spider:z_failed_requests")
|
|
|
# 数据保存失败表
|
|
|
@@ -61,7 +63,6 @@ COLLECTOR_SLEEP_TIME = 1 # 从任务队列中获取任务到内存队列的间
|
|
|
COLLECTOR_TASK_COUNT = 10 # 每次获取任务数量
|
|
|
|
|
|
# SPIDER
|
|
|
-SPIDER_HEARTBEAT = os.getenv("SPIDER_HEARTBEAT") # 爬虫心跳
|
|
|
SPIDER_THREAD_COUNT = 1 # 爬虫并发数
|
|
|
SPIDER_SLEEP_TIME = (
|
|
|
0
|
