|
@@ -365,7 +365,7 @@ func dataRepeat(infoformat int, tmp map[string]interface{}, href, hashHref, uuid
|
|
|
publishtime := qutil.Int64All(tmp["publishtime"])
|
|
publishtime := qutil.Int64All(tmp["publishtime"])
|
|
|
isExist := false //记录判重结果
|
|
isExist := false //记录判重结果
|
|
|
repeatby := "" //判重字段
|
|
repeatby := "" //判重字段
|
|
|
- if infoformat == NEWS_INFOFORMAT { //新闻数据判重
|
|
|
|
|
|
|
+ if infoformat == NEWS_INFOFORMAT { //新闻数据判重(采集时已有list链接db判重此处不再做判重处理)
|
|
|
if uuid != ANNEX_DETAIL_SHA && filterDetail != "无" && !DetailExists(filterDetail) {
|
|
if uuid != ANNEX_DETAIL_SHA && filterDetail != "无" && !DetailExists(filterDetail) {
|
|
|
isExist = tools.CheckStr("detail", uuid)
|
|
isExist = tools.CheckStr("detail", uuid)
|
|
|
if isExist {
|
|
if isExist {
|
