修改redis相关操作

src/config.json

@@ -62,10 +62,6 @@
   "testdir": "res/test/spider_test.lua",
   "serveraddress": "127.0.0.1:8030",
   "jsserveraddress":  "127.0.0.1:8031",
-  "redisclusteraddrs": [
-    "192.168.3.207:2179",
-    "192.168.3.207:2279"
-  ],
   "word":{
     "keyword":"(抽签|中标|招标|成交|合同|中标候选人|资格预审|拟建|邀请|询价|比选|议价|竞价|磋商|采购|招投标|答疑|变更公告|更正公告|竞争性谈判|竞谈|意见征询|澄清|单一来源|流标|废标|验收公告|中止|终止|违规|处罚|征集公告|开标结果|评审结果|监理|招租|租赁|评判结果|项目|遴选|补遗|竞标|征求意见)",
     "notkeyword":"(招聘|拍卖|出租|出让|使用权|资产)"

src/front/data.go

@@ -634,7 +634,9 @@ func importWarnErrData(coll string, dr *DataRequest, isbidding bool) (ok bool, q
 		qu.Debug("导入数据完毕：", coll)
 	}()
 	//1、检索数据
-	query := map[string]interface{}{}
+	query := map[string]interface{}{
+		"ok": false,
+	}
 	if dr.From != "-1" { //来源
 		query["from"] = dr.From
 	}
@@ -689,7 +691,7 @@ func importWarnErrData(coll string, dr *DataRequest, isbidding bool) (ok bool, q
 			set := map[string]interface{}{}
 			if data, ok := tmp["data"].(map[string]interface{}); ok { //必要字段
 				for _, field := range util.ImportDataFieldsCheck {
-					if data[field] == nil { //必要字段不存在，视为异常数据
+					if field != "district" && data[field] == nil { //必要字段不存在，视为异常数据(校验district因为部分爬虫目前没有采district字段)
 						return
 					}
 					if field == "publishtime" { //

src/front/lua.go

@@ -8,7 +8,6 @@ import (
 	"io/ioutil"
 	"mime/multipart"
 	qu "qfw/util"
-	"spider"
 	. "spiderutil"
 	"strings"
 	. "task"
@@ -230,8 +229,6 @@ func (f *Front) LuaDataSend() {
 						msg = "保存成功；数据所在表：" + bidcoll + "；数据ID：" + bidid
 					} else {
 						msg = "推送成功，但数据存在异常或被判重！"
-						hashHref := spider.HexText(qu.ObjToString(data["href"])) //推送的异常数据放回全量redis
-						RedisClusterSet(hashHref, "", -1)
 					}
 				}
 			} else {
@@ -419,11 +416,11 @@ func (f *Front) FilterSave() {
 			msg = "保存成功"
 			success = true
 		}
-		shaid := ShaDetail(qu.ObjToString((*data)["detail"]))
+		shaid := Sha(qu.ObjToString((*data)["detail"]))
 		(*data)["s_sha"] = shaid
 		(*data)["extracttype"] = 0
-		RedisClusterSet(shaid, "", 86400*365)
 		MgoS.UpdateById("spider_filterdata", id, map[string]interface{}{"$set": map[string]interface{}{"biddingid": biddingId, "s_sha": shaid}})
+		//存redis（暂无）
 	}
 	f.ServeJson(map[string]interface{}{"msg": msg, "success": success})
 }

src/main.go

@@ -24,8 +24,6 @@ func init() {
 	util.InitMgoPool()        //初始化连接
 	util.InitOther()
 	InitUdp()
-	//redis集群
-	sp.InitRedisCluster(sp.Config.RedisClusterAddrs)
 	//xweb框架配置
 	xweb.Config.RecoverPanic = true
 	xweb.Config.Profiler = true

src/task/task.go

@@ -415,10 +415,7 @@ func AnalysisProjectInfo(data map[string]interface{}) bool {
 //调用消息保存
 func SaveObj(event int, checkAtrr string, data map[string]interface{}) (flagstr, biddingid, biddingcoll string) {
 	defer mu.Catch()
-	//删除redis
-	hashHref := spider.HexText(qu.ObjToString(data["href"]))
-	sputil.RedisClusterDelete(hashHref)
-
+	data["repeat"] = true //此标记可绕过保存服务判重
 	tmp, _ := json.Marshal([]interface{}{checkAtrr, []interface{}{data}})
 	flag := true //是否传输失败
 	idAndColl := ""