|
|
@@ -284,6 +284,7 @@ func SaveObj(event int, checkAtrr string, data map[string]interface{}, saveredis
|
|
|
if !isExist {
|
|
|
id = MgoS.Save("data_bak", data)
|
|
|
} else { //记录重复数据,spider_repeatdata
|
|
|
+ data["from"] = "spider"
|
|
|
MgoS.Save("spider_repeatdata", data)
|
|
|
}
|
|
|
//保存服务未接收成功的数据会存入data_bak中，确保数据不丢失依赖补发程序
|
|
|
@@ -293,7 +294,7 @@ func SaveObj(event int, checkAtrr string, data map[string]interface{}, saveredis
|
|
|
//全量(判断是否已存在防止覆盖id)
|
|
|
isExist, _ := util.ExistRedis("title_repeat_fulljudgement", db, "url_repeat_"+href)
|
|
|
if !isExist {
|
|
|
- util.PutRedis("title_repeat_fulljudgement", db, "url_repeat_"+href, "", -1)
|
|
|
+ util.PutRedis("title_repeat_fulljudgement", db, "url_repeat_"+href, "lua_"+id, -1)
|
|
|
}
|
|
|
}
|
|
|
}
|
