package db import ( "log" "time" "KeyWebsiteMonitor/spider/config" spider_com "KeyWebsiteMonitor/util" qu "app.yhyue.com/moapp/jybase/common" mg "app.yhyue.com/moapp/jybase/mongodb" ) var Mgo mg.MongodbSim var MgoCommon mg.MongodbSim func initMgo() { Mgo = mg.MongodbSim{ MongodbAddr: config.Sysconfig.MongoSpiderAddress, Size: 5, DbName: config.Sysconfig.MongoSpiderDb, } Mgo.InitPool() log.Println("init mongodb") } func initMgoCommon() { MgoCommon = mg.MongodbSim{ MongodbAddr: config.Sysconfig.MongoCommonAddress, Size: 5, DbName: config.Sysconfig.MongoCommonDb, } MgoCommon.InitPool() log.Println("init mongodb") } func init() { initMgo() initMgoCommon() initRedis() } func initRedis() { //初始化redis // redis.InitRedisBySize(config.Redisaddrs, 50, 15, 150) //list=192.168.3.207:1679@jytopnet123,href=192.168.3.207:1679 //other=172.17.4.84:4679@jytopnet123 spider_com.InitRedisClient(config.Sysconfig.Redisaddrs) log.Println("init redis") } //DataCheck 数据检查是否异常 func DataCheck(data map[string]interface{}, spidercode string) bool { msg, ok := func() (string, bool) { if data == nil || len(data) <= 0 { log.Printf("%s spidercode no data", spidercode) return "no data", false } if data["title"] == nil || data["href"] == nil || data["content"] == nil || qu.ObjToString(data["title"]) == "" || qu.ObjToString(data["href"]) == "" || qu.ObjToString(data["content"]) == "" { log.Printf("%s spidercode miss fields", spidercode) return "miss fields", false } return "", true }() if msg != "" && !ok { saveErrData := map[string]interface{}{ "createtime": time.Now().Unix(), "spidercode": spidercode, "err": msg, } if data != nil && len(data) > 0 { saveErrData["data"] = data saveErrData["href"] = qu.ObjToString(data["href"]) saveErrData["title"] = qu.ObjToString(data["title"]) saveErrData["content"] = qu.ObjToString(data["content"]) } Mgo.Save("data_bak_common_err", saveErrData) } return ok }