websiteMonitor/spider/db/db.go

package db

import (
	"log"
	"time"

	"KeyWebsiteMonitor/spider/config"
	spider_com "KeyWebsiteMonitor/util"

	qu "app.yhyue.com/moapp/jybase/common"
	mg "app.yhyue.com/moapp/jybase/mongodb"
)

var Mgo mg.MongodbSim
var MgoCommon mg.MongodbSim

func initMgo() {
	Mgo = mg.MongodbSim{
		MongodbAddr: config.Sysconfig.MongoSpiderAddress,
		Size:        5,
		DbName:      config.Sysconfig.MongoSpiderDb,
	}
	Mgo.InitPool()
	log.Println("init mongodb")

}

func initMgoCommon() {
	MgoCommon = mg.MongodbSim{
		MongodbAddr: config.Sysconfig.MongoCommonAddress,
		Size:        5,
		DbName:      config.Sysconfig.MongoCommonDb,
	}
	MgoCommon.InitPool()
	log.Println("init mongodb")

}

func init() {

	initMgo()
	initMgoCommon()
	initRedis()
}

func initRedis() {
	//初始化redis
	// redis.InitRedisBySize(config.Redisaddrs, 50, 15, 150)
	//list=192.168.3.207:1679@jytopnet123,href=192.168.3.207:1679
	//other=172.17.4.84:4679@jytopnet123
	spider_com.InitRedisClient(config.Sysconfig.Redisaddrs)
	log.Println("init redis")

}

//DataCheck 数据检查是否异常
func DataCheck(data map[string]interface{}, spidercode string) bool {

	msg, ok := func() (string, bool) {
		if data == nil || len(data) <= 0 {
			log.Printf("%s spidercode no data", spidercode)
			return "no data", false
		}
		if data["title"] == nil || data["href"] == nil || data["content"] == nil ||
			qu.ObjToString(data["title"]) == "" ||
			qu.ObjToString(data["href"]) == "" ||
			qu.ObjToString(data["content"]) == "" {
			log.Printf("%s spidercode miss fields", spidercode)
			return "miss fields", false
		}
		return "", true
	}()

	if msg != "" && !ok {
		saveErrData := map[string]interface{}{
			"createtime": time.Now().Unix(),
			"spidercode": spidercode,
			"err":        msg,
		}
		if data != nil && len(data) > 0 {
			saveErrData["data"] = data
			saveErrData["href"] = qu.ObjToString(data["href"])
			saveErrData["title"] = qu.ObjToString(data["title"])
			saveErrData["content"] = qu.ObjToString(data["content"])
		}
		Mgo.Save("data_bak_common_err", saveErrData)
	}
	return ok
}