package main import ( "github.com/spf13/viper" //"jygit.jydev.jianyu360.cn/data_processing/common_utils/log" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb" "log" ) var ( GF GlobalConf MgoFm *mongodb.MongodbSim // 84 MgoTo *mongodb.MongodbSim // 166 MgoB *mongodb.MongodbSim //bidding 标讯数据库 ) func InitConfig() (err error) { viper.SetConfigFile("config.toml") // 指定配置文件路径 viper.SetConfigName("config") // 配置文件名称(无扩展名) viper.SetConfigType("toml") // 如果配置文件的名称中没有扩展名,则需要配置此项 viper.AddConfigPath("./") viper.AddConfigPath("./conf/") // 还可以在工作目录中查找配置 viper.AddConfigPath("../conf/") // 还可以在工作目录中查找配置 err = viper.ReadInConfig() // 查找并读取配置文件 if err != nil { // 处理读取配置文件的错误 return } err = viper.Unmarshal(&GF) return err } //func InitLog() { // err := log.InitLog( // //log.Path("./logs/log.out"), // log.Path(""), // log.Level("info"), // log.Compress(true), // log.MaxSize(10), // log.MaxBackups(10), // log.MaxAge(7), // log.Format("json"), // ) // if err != nil { // fmt.Printf("InitLog failed: %v\n", err) // } //} func InitMgo() { MgoB = &mongodb.MongodbSim{ MongodbAddr: GF.MongoB.Host, DbName: GF.MongoB.DB, Size: GF.MongoB.Size, UserName: GF.MongoB.Username, Password: GF.MongoB.Password, Direct: GF.MongoB.Direct, } MgoB.InitPool() MgoFm = &mongodb.MongodbSim{ MongodbAddr: GF.MongoF.Host, DbName: GF.MongoF.DB, Size: GF.MongoF.Size, UserName: GF.MongoF.Username, Password: GF.MongoF.Password, Direct: GF.MongoF.Direct, } MgoFm.InitPool() MgoTo = &mongodb.MongodbSim{ MongodbAddr: GF.MongoT.Host, DbName: GF.MongoT.DB, Size: GF.MongoT.Size, UserName: GF.MongoT.Username, Password: GF.MongoT.Password, Direct: GF.MongoT.Direct, } MgoTo.InitPool() } func main() { InitConfig() //InitLog() InitMgo() //f := GF //fmt.Println(f) dealData() log.Println("数据处理完毕") } func dealData() { sess := MgoFm.GetMgoConn() defer MgoFm.DestoryMongoConn(sess) where := map[string]interface{}{} if GF.Env.Appid != "" { where["appid"] = GF.Env.Appid } if GF.Env.Createtime > 0 { where["createtime"] = map[string]interface{}{ "$gt": GF.Env.Createtime, } } query := sess.DB(GF.MongoF.DB).C(GF.MongoF.Coll).Find(where).Select(nil).Iter() count := 0 //fields := []string{"area", "city", "projectname", "projectcode", "budget", "bidamount", "buyer", "s_winner"} for tmp := make(map[string]interface{}); query.Next(tmp); count++ { if count%100 == 0 { log.Println("current ---", count) } biddingId := mongodb.BsonIdToSId(tmp["_id"]) biddingH, _ := MgoB.FindById("bidding", biddingId, nil) if biddingH != nil && len(*biddingH) > 0 { tmp["is_high"] = true for _, field := range GF.Env.Fields { if val, ok := (*biddingH)[field]; ok && val != nil { tmp[field] = val } } MgoTo.SaveByOriID(GF.MongoT.Coll, tmp) } else { tmp["is_high"] = false MgoTo.SaveByOriID(GF.MongoT.Coll, tmp) } } }