data_bak数据保存修改

src/spider/msgservice.go

@@ -340,6 +340,60 @@ func SaveObj(event int, checkAtrr string, data map[string]interface{}, saveredis
 		}
 	}
 }
+func SaveObjNew(event int, checkAtrr string, data map[string]interface{}, saveredis bool) {
+	bs, _ := json.Marshal(data)
+	size := len(bs) / (1024 * 1024)
+	if size > 10 { //超大数据过滤
+		href := fmt.Sprint(data["href"])
+		util.AddBloomRedis("href", href)
+		data["detail"] = ""      //字段太大
+		data["contenthtml"] = "" //字段太大
+		MgoS.Save("spider_filterdata", data)
+		//log.Println(event, checkAtrr, data["href"], data["title"], len(bs))
+		return
+	}
+	defer qu.Catch()
+	tmp, _ := json.Marshal([]interface{}{checkAtrr, []interface{}{data}})
+	switch event {
+	case mu.SERVICE_SPIDER_ECPS: //著作权等服务
+		Msclient.WriteObj("", "", mu.SERVICE_SPIDER_ECPS, mu.SENDTO_TYPE_ALL_RECIVER, data)
+	default:
+		fail := true
+		set := map[string]interface{}{}
+		for i := 1; i <= 3; i++ {
+			bs, err := Msclient.Call("", mu.UUID(8), event, mu.SENDTO_TYPE_ALL_RECIVER, tmp, 30)
+			msg := map[string]string{}
+			if err == nil && json.Unmarshal(bs, &msg) == nil {
+				isgetid := msg["isgetid"] //是否获取到id
+				if isgetid == "true" {    //1、数据被过滤；2、未过滤数据，正常保存
+					set["sendflag"] = "true"
+					set["biddingid"] = msg["id"]     //保存表id
+					set["biddingcoll"] = msg["coll"] //保存表名
+				} else { //获取id失败
+					set["repeat"] = "isgetid"
+					set["sendflag"] = "false"
+				}
+				fail = false
+				break
+			}
+			time.Sleep(time.Duration(5*i) * time.Second)
+		}
+		//qu.Debug("----------save-------")
+		if fail {
+			data["sendflag"] = "false"
+			logger.Error("未成功传送信息", event, data["title"])
+		} else {
+			for k, v := range set {
+				data[k] = v
+			}
+		}
+		href := fmt.Sprint(data["href"])
+		if len(href) > 5 && saveredis { //有效数据
+			MgoS.Save("data_bak", data)
+			//DataBakSaveCache <- data
+		}
+	}
+}
 
 //从微信端获取验证码
 func GetCodeByWx(img []byte) (string, error) {

src/spider/store.go

@@ -76,7 +76,7 @@ func Store(mode, event int, c, coverAttr string, data map[string]interface{}, fl
 		//}
 	} else if mode == 2 {
 		data["T"] = c
-		SaveObj(event, coverAttr, data, flag)
+		SaveObjNew(event, coverAttr, data, flag)
 	}
 	//公示数据，发送其他服务请求
 	if strings.HasPrefix(c, "ecps") {