spider_task/src/util/config.go

package util

import (
	mgo "mongodb"
	qu "qfw/util"
	"regexp"
)

var (
	Config             map[string]interface{}
	MgoE               *mgo.MongodbSim //editor
	MgoS               *mgo.MongodbSim //spider
	MgoPy              *mgo.MongodbSim //py_spider
	CommServers        map[string]interface{}
	BidServers         map[string]interface{}
	UploadEvents       map[int]string
	RandomDataPushCron string
	QyworkRemindCron   string
	StartTaskCron      string      //任务开始
	CodeSummaryCron    string      //每天统计爬虫信息
	CloseNum           int         //关闭几天的任务
	DayNum             int         //更新数据天数
	CodeEventModel     map[int]int //节点对应的采集模式0：老模式；1：新模式
	CodeEventWorking   map[int]int //节点对应的采集模式0：高性能模式；1：队列模式

)
var TimeReg = regexp.MustCompile("[0-9]{4}-[0-9]{2}-[0-9]{2}")

func InitMgo() {
	spider := Config["spider"].(map[string]interface{})
	MgoS = &mgo.MongodbSim{
		MongodbAddr: qu.ObjToString(spider["addr"]),
		DbName:      qu.ObjToString(spider["db"]),
		Size:        qu.IntAll(spider["size"]),
	}
	MgoS.InitPool()
	pyspider := Config["spider"].(map[string]interface{})
	MgoPy = &mgo.MongodbSim{
		MongodbAddr: qu.ObjToString(pyspider["addr"]),
		DbName:      qu.ObjToString(pyspider["db"]),
		Size:        qu.IntAll(pyspider["size"]),
	}
	MgoPy.InitPool()
	editor := Config["editor"].(map[string]interface{})
	MgoE = &mgo.MongodbSim{
		MongodbAddr: qu.ObjToString(editor["addr"]),
		DbName:      qu.ObjToString(editor["db"]),
		Size:        qu.IntAll(editor["size"]),
	}
	MgoE.InitPool()
}

func InitOther() {
	StartTaskCron = qu.ObjToString(Config["startaskcron"])
	CodeSummaryCron = qu.ObjToString(Config["codesummarycron"])
	RandomDataPushCron = qu.ObjToString(Config["randomdatapushcron"])
	QyworkRemindCron = qu.ObjToString(Config["qyworkremindcron"])
	CloseNum = qu.IntAll(Config["closenum"])
	DayNum = qu.IntAll(Config["daynum"])
	MsgServers := Config["msgservers"].(map[string]interface{})
	CommServers = MsgServers["comm"].(map[string]interface{})
	BidServers = MsgServers["bid"].(map[string]interface{})
	//event信息
	eventsinfo := Config["eventsinfo"].(map[string]interface{})
	UploadEvents = map[int]string{}
	CodeEventModel = map[int]int{}
	CodeEventWorking = map[int]int{}
	for event, info := range eventsinfo {
		eventTmp := qu.IntAll(event)
		infoMap := info.(map[string]interface{})
		UploadEvents[eventTmp] = qu.ObjToString(infoMap["server"])
		CodeEventModel[eventTmp] = qu.IntAll(infoMap["model"])
		CodeEventWorking[eventTmp] = qu.IntAll(infoMap["work"])
	}
	//qu.Debug(UploadEvents, CodeEventModel, CodeEventWorking)
}