|
|
@@ -771,9 +771,9 @@ func (s *Spider) DownloadHighDetail() {
|
|
|
for _, tmp := range *list {
|
|
|
_id := tmp["_id"]
|
|
|
query := map[string]interface{}{"_id": _id}
|
|
|
+ href := qu.ObjToString(tmp["href"])
|
|
|
if isEsRepeat { //es数据title判重
|
|
|
title := qu.ObjToString(tmp["title"])
|
|
|
- href := qu.ObjToString(tmp["href"])
|
|
|
eTime := fmt.Sprint(GetTime(0))
|
|
|
sTime := fmt.Sprint(GetTime(-7))
|
|
|
esQuery := `{"query": {"filtered": {"filter": {"bool": {"must": [{"range": {"comeintime": {"gte": "` + sTime + `","lte": "` + eTime + `"}}}]}},"query": {"bool": {"must": [{"multi_match": {"query": "` + title + `","type": "phrase","fields": ["title"]}}]}}}}}`
|
|
|
@@ -789,7 +789,6 @@ func (s *Spider) DownloadHighDetail() {
|
|
|
success := true //数据是否下载成功的标志
|
|
|
delete(tmp, "_id")
|
|
|
delete(tmp, "times")
|
|
|
- href := qu.ObjToString(tmp["href"])
|
|
|
data := map[string]interface{}{}
|
|
|
var err interface{}
|
|
|
for k, v := range tmp {
|
|
|
@@ -908,9 +907,9 @@ func (s *Spider) DownloadListDetail() {
|
|
|
for _, tmp := range *list {
|
|
|
_id := tmp["_id"]
|
|
|
query := map[string]interface{}{"_id": _id}
|
|
|
+ href := qu.ObjToString(tmp["href"])
|
|
|
if isEsRepeat { //es数据title判重
|
|
|
title := qu.ObjToString(tmp["title"])
|
|
|
- href := qu.ObjToString(tmp["href"])
|
|
|
eTime := fmt.Sprint(GetTime(0))
|
|
|
sTime := fmt.Sprint(GetTime(-7))
|
|
|
esQuery := `{"query": {"filtered": {"filter": {"bool": {"must": [{"range": {"comeintime": {"gte": "` + sTime + `","lte": "` + eTime + `"}}}]}},"query": {"bool": {"must": [{"multi_match": {"query": "` + title + `","type": "phrase","fields": ["title"]}}]}}}}}`
|
|
|
@@ -925,7 +924,6 @@ func (s *Spider) DownloadListDetail() {
|
|
|
success := true //数据是否下载成功的标志
|
|
|
delete(tmp, "_id")
|
|
|
delete(tmp, "times")
|
|
|
- href := qu.ObjToString(tmp["href"])
|
|
|
data := map[string]interface{}{}
|
|
|
var err interface{}
|
|
|
for k, v := range tmp {
|
