xg

src/spider/spider.go

@@ -850,8 +850,7 @@ func (s *Spider) DownloadHighDetail() {
 			if !s.Stop { //在下载详情页时爬虫下架，此时不再存心跳信息
 				UpdateHeart(s.Name, s.Channel, s.Code, s.MUserName, "detail") //记录modal=1采集三级页心跳
 			}
-			list, _ := Mgo.Find("spider_highlistdata_test", q, o, f, false, 0, 100)
-			qu.Debug("----", len(*list))
+			list, _ := Mgo.Find("spider_highlistdata", q, o, f, false, 0, 100)
 			if list != nil && len(*list) > 0 {
 				for _, tmp := range *list {
 					_id := tmp["_id"]
@@ -862,7 +861,7 @@ func (s *Spider) DownloadHighDetail() {
 					isExist, _ := util.ExistRedis("title_repeat_judgement", 0, "url_repeat_"+href)
 					if isExist {
 						set := map[string]interface{}{"$set": map[string]interface{}{"state": 1, "exist": true, "updatetime": time.Now().Unix()}} //已存在state置为1
-						Mgo.Update("spider_highlistdata_test", query, set, false, false)
+						Mgo.Update("spider_highlistdata", query, set, false, false)
 						continue
 					}
 					if isEsRepeat { //es数据title判重
@@ -873,7 +872,7 @@ func (s *Spider) DownloadHighDetail() {
 						count := Es.Count(EsIndex, EsType, esQuery)
 						if count > 0 { //es中含本title数据，不再采集，更新list表数据状态
 							set := map[string]interface{}{"$set": map[string]interface{}{"state": 1, "exist": true, "updatetime": time.Now().Unix()}} //已存在state置为1
-							Mgo.Update("spider_highlistdata_test", query, set, false, false)
+							Mgo.Update("spider_highlistdata", query, set, false, false)
 							util.PutRedis("title_repeat_judgement", 0, "url_repeat_"+href, href, 3600*24*365)
 							continue
 						}
@@ -922,7 +921,7 @@ func (s *Spider) DownloadHighDetail() {
 							ss["state"] = -1
 						}
 						set := map[string]interface{}{"$set": ss}
-						Mgo.Update("spider_highlistdata_test", query, set, false, false)
+						Mgo.Update("spider_highlistdata", query, set, false, false)
 						continue
 					} else {
 						deleteData := FilterByDetail(href, query, data) //针对列表页无法过滤需要在详情页过滤的数据，进行过滤处理
@@ -952,7 +951,7 @@ func (s *Spider) DownloadHighDetail() {
 					data["iscompete"] = s.IsCompete //2021-11-01以后新增的爬虫不在展示原文链接(保存服务判断)
 					Store(s.StoreMode, s.StoreToMsgEvent, s.Collection, s.CoverAttr, data, true)
 					set := map[string]interface{}{"$set": map[string]interface{}{"state": 1, "updatetime": time.Now().Unix()}} //下载成功state置为1
-					Mgo.Update("spider_highlistdata_test", query, set, false, false)
+					Mgo.Update("spider_highlistdata", query, set, false, false)
 				}
 				//重载spider
 				s.LoadScript(&s.Name, &s.Channel, &s.MUserName, s.Code, s.ScriptFile, true)