spider_task/src/util/config.go

package util

import (
	mgo "mongodb"
	qu "qfw/util"
	gm "qfw/util/mail"
	"regexp"
)

var (
	Config                     map[string]interface{}
	MgoE                       *mgo.MongodbSim //editor
	MgoEB                      *mgo.MongodbSim //editor
	MgoS                       *mgo.MongodbSim //spider
	MgoPy                      *mgo.MongodbSim //py_spider
	MgoB                       *mgo.MongodbSim //bidding
	CommServers                map[string]interface{}
	BidServers                 map[string]interface{}
	UploadEvents               map[int]string
	RandomDataPushCron         string
	QyworkRemindModifyuserCron string
	QyworkRemindAuditorCron    string
	StartTaskCron              string        //任务开始
	CodeSummaryCron            string        //每天统计爬虫信息
	ResetDataStateCron         string        //重置数据状态
	FileWarnCron               string        //每天统计附件异常数据
	MoveListDataCron           string        //迁移spider_highlistdata、spider_listdata数据
	CloseNum                   int           //关闭几天的任务
	DayNum                     int           //更新数据天数
	CodeEventModel             map[int]int   //节点对应的采集模式0：老模式；1：新模式
	CodeEventWorking           map[int]int   //节点对应的采集模式0：高性能模式；1：队列模式
	GMail                      *gm.GmailAuth //邮件信息
	To                         string        //邮件接收人

)
var TimeReg = regexp.MustCompile("[0-9]{4}-[0-9]{2}-[0-9]{2}")

func InitMgo() {
	spider := Config["spider"].(map[string]interface{})
	MgoS = &mgo.MongodbSim{
		MongodbAddr: qu.ObjToString(spider["addr"]),
		DbName:      qu.ObjToString(spider["db"]),
		Size:        qu.IntAll(spider["size"]),
	}
	MgoS.InitPool()
	pyspider := Config["pyspider"].(map[string]interface{})
	MgoPy = &mgo.MongodbSim{
		MongodbAddr: qu.ObjToString(pyspider["addr"]),
		DbName:      qu.ObjToString(pyspider["db"]),
		Size:        qu.IntAll(pyspider["size"]),
	}
	MgoPy.InitPool()
	editor := Config["editor"].(map[string]interface{})
	MgoE = &mgo.MongodbSim{
		MongodbAddr: qu.ObjToString(editor["addr"]),
		DbName:      qu.ObjToString(editor["db"]),
		Size:        qu.IntAll(editor["size"]),
	}
	MgoE.InitPool()
	bidding := Config["bidding"].(map[string]interface{})
	MgoB = &mgo.MongodbSim{
		MongodbAddr: qu.ObjToString(bidding["addr"]),
		DbName:      qu.ObjToString(bidding["db"]),
		Size:        qu.IntAll(bidding["size"]),
		UserName:    qu.ObjToString(bidding["username"]),
		Password:    qu.ObjToString(bidding["password"]),
	}
	MgoB.InitPool()
	bidEditor := Config["bideditor"].(map[string]interface{})
	MgoEB = &mgo.MongodbSim{
		MongodbAddr: qu.ObjToString(bidEditor["addr"]),
		DbName:      qu.ObjToString(bidEditor["db"]),
		Size:        qu.IntAll(bidEditor["size"]),
		UserName:    qu.ObjToString(bidEditor["username"]),
		Password:    qu.ObjToString(bidEditor["password"]),
	}
	MgoEB.InitPool()
}

func InitOther() {
	StartTaskCron = qu.ObjToString(Config["startaskcron"])
	CodeSummaryCron = qu.ObjToString(Config["codesummarycron"])
	ResetDataStateCron = qu.ObjToString(Config["resetdatastatecron"])
	RandomDataPushCron = qu.ObjToString(Config["randomdatapushcron"])
	QyworkRemindModifyuserCron = qu.ObjToString(Config["qyworkremindmodifyusercron"])
	QyworkRemindAuditorCron = qu.ObjToString(Config["qyworkremindauditorcron"])
	FileWarnCron = qu.ObjToString(Config["filewarncron"])
	MoveListDataCron = qu.ObjToString(Config["movelistdata"])
	CloseNum = qu.IntAll(Config["closenum"])
	DayNum = qu.IntAll(Config["daynum"])
	MsgServers := Config["msgservers"].(map[string]interface{})
	CommServers = MsgServers["comm"].(map[string]interface{})
	BidServers = MsgServers["bid"].(map[string]interface{})
	//event信息
	eventsinfo := Config["eventsinfo"].(map[string]interface{})
	UploadEvents = map[int]string{}
	CodeEventModel = map[int]int{}
	CodeEventWorking = map[int]int{}
	for event, info := range eventsinfo {
		eventTmp := qu.IntAll(event)
		infoMap := info.(map[string]interface{})
		UploadEvents[eventTmp] = qu.ObjToString(infoMap["server"])
		CodeEventModel[eventTmp] = qu.IntAll(infoMap["model"])
		CodeEventWorking[eventTmp] = qu.IntAll(infoMap["work"])
	}
	//qu.Debug(UploadEvents, CodeEventModel, CodeEventWorking)
	//mail
	mail := Config["mail"].(map[string]interface{})
	qu.Debug(mail)
	GMail = &gm.GmailAuth{
		SmtpHost: qu.ObjToString(mail["host"]),
		SmtpPort: qu.IntAll(mail["port"]),
		User:     qu.ObjToString(mail["user"]),
		Pwd:      qu.ObjToString(mail["pwd"]),
		ReTry:    qu.IntAll(mail["retry"]),
	}
	To = qu.ObjToString(mail["to"])
}