|
|
@@ -286,6 +286,7 @@ func SaveObj(event int, checkAtrr string, data map[string]interface{}, saveredis
|
|
|
if !isExist {
|
|
|
id = mgu.Save("data_bak", "spider", "spider", data)
|
|
|
} else { //记录重复数据,spider_repeatdata
|
|
|
+ data["from"] = "spider"
|
|
|
mgu.Save("spider_repeatdata", "spider", "spider", data)
|
|
|
}
|
|
|
//保存服务未接收成功的数据会存入data_bak中，确保数据不丢失依赖补发程序
|
|
|
@@ -295,7 +296,7 @@ func SaveObj(event int, checkAtrr string, data map[string]interface{}, saveredis
|
|
|
//全量(判断是否已存在防止覆盖id)
|
|
|
isExist, _ := util.ExistRedis("title_repeat_fulljudgement", db, "url_repeat_"+href)
|
|
|
if !isExist {
|
|
|
- util.PutRedis("title_repeat_fulljudgement", db, "url_repeat_"+href, "", -1)
|
|
|
+ util.PutRedis("title_repeat_fulljudgement", db, "url_repeat_"+href, "lua_"+id, -1)
|
|
|
}
|
|
|
}
|
|
|
}
|
