|
|
@@ -139,12 +139,12 @@ func (s *Spider) DownloadDetailItem(p interface{}) {
|
|
|
} else if from == "lua" && tmphref != href { //三级页href替换导致前后href不同
|
|
|
logger.Debug("beforeHref:", href, " afterHref:", tmphref)
|
|
|
//增量
|
|
|
- util.PutRedis("title_repeat_judgement", 0, "url_repeat_"+tmphref, tmphref, 3600*24*30)
|
|
|
+ util.PutRedis("title_repeat_judgement", 0, "url_repeat_"+href, href, 3600*24*30)
|
|
|
//全量
|
|
|
- db := HexToBigIntMod(tmphref)
|
|
|
- isExist, _ := util.ExistRedis("title_repeat_fulljudgement", db, "url_repeat_"+tmphref)
|
|
|
+ db := HexToBigIntMod(href)
|
|
|
+ isExist, _ := util.ExistRedis("title_repeat_fulljudgement", db, "url_repeat_"+href)
|
|
|
if !isExist {
|
|
|
- util.PutRedis("title_repeat_fulljudgement", db, "url_repeat_"+tmphref, "", -1)
|
|
|
+ util.PutRedis("title_repeat_fulljudgement", db, "url_repeat_"+href, "", -1)
|
|
|
}
|
|
|
}
|
|
|
//jsondata处理
|
|
|
@@ -254,6 +254,7 @@ func (s *Spider) DownloadDetailPage(param map[string]interface{}, data map[strin
|
|
|
//
|
|
|
func UpdateState(id string, set map[string]interface{}) {
|
|
|
defer qu.Catch()
|
|
|
+ logger.Debug("++++++++++++++++++++++++", id, mongo.StringTOBsonId(id))
|
|
|
update := []map[string]interface{}{}
|
|
|
update = append(update, map[string]interface{}{"_id": mongo.StringTOBsonId(id)})
|
|
|
update = append(update, map[string]interface{}{"$set": set})
|
