jyps/operator/main.go

package main

import (
	"github.com/spf13/viper"
	//"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
	"log"
)

var (
	GF    GlobalConf
	MgoFm *mongodb.MongodbSim // 84
	MgoTo *mongodb.MongodbSim // 166
	MgoB  *mongodb.MongodbSim //bidding 标讯数据库

)

func InitConfig() (err error) {
	viper.SetConfigFile("config.toml") // 指定配置文件路径
	viper.SetConfigName("config")      // 配置文件名称(无扩展名)
	viper.SetConfigType("toml")        // 如果配置文件的名称中没有扩展名，则需要配置此项

	viper.AddConfigPath("./")
	viper.AddConfigPath("./conf/")  // 还可以在工作目录中查找配置
	viper.AddConfigPath("../conf/") // 还可以在工作目录中查找配置
	err = viper.ReadInConfig()      // 查找并读取配置文件
	if err != nil {                 // 处理读取配置文件的错误
		return
	}

	err = viper.Unmarshal(&GF)

	return err

}

//func InitLog() {
//	err := log.InitLog(
//		//log.Path("./logs/log.out"),
//		log.Path(""),
//		log.Level("info"),
//		log.Compress(true),
//		log.MaxSize(10),
//		log.MaxBackups(10),
//		log.MaxAge(7),
//		log.Format("json"),
//	)
//	if err != nil {
//		fmt.Printf("InitLog failed: %v\n", err)
//	}
//}

func InitMgo() {
	MgoB = &mongodb.MongodbSim{
		MongodbAddr: GF.MongoB.Host,
		DbName:      GF.MongoB.DB,
		Size:        GF.MongoB.Size,
		UserName:    GF.MongoB.Username,
		Password:    GF.MongoB.Password,
		Direct:      GF.MongoB.Direct,
	}
	MgoB.InitPool()

	MgoFm = &mongodb.MongodbSim{
		MongodbAddr: GF.MongoF.Host,
		DbName:      GF.MongoF.DB,
		Size:        GF.MongoF.Size,
		UserName:    GF.MongoF.Username,
		Password:    GF.MongoF.Password,
		Direct:      GF.MongoF.Direct,
	}
	MgoFm.InitPool()

	MgoTo = &mongodb.MongodbSim{
		MongodbAddr: GF.MongoT.Host,
		DbName:      GF.MongoT.DB,
		Size:        GF.MongoT.Size,
		UserName:    GF.MongoT.Username,
		Password:    GF.MongoT.Password,
		Direct:      GF.MongoT.Direct,
	}
	MgoTo.InitPool()

}

func main() {
	InitConfig()
	//InitLog()
	InitMgo()

	//f := GF
	//fmt.Println(f)
	dealData()
	log.Println("数据处理完毕")
}

func dealData() {
	sess := MgoFm.GetMgoConn()
	defer MgoFm.DestoryMongoConn(sess)

	where := map[string]interface{}{}
	if GF.Env.Appid != "" {
		where["appid"] = GF.Env.Appid
	}

	if GF.Env.Createtime > 0 {
		where["createtime"] = map[string]interface{}{
			"$gt": GF.Env.Createtime,
		}
	}

	query := sess.DB(GF.MongoF.DB).C(GF.MongoF.Coll).Find(where).Select(nil).Iter()
	count := 0

	//fields := []string{"area", "city", "projectname", "projectcode", "budget", "bidamount", "buyer", "s_winner"}

	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
		if count%100 == 0 {
			log.Println("current ---", count)
		}

		biddingId := mongodb.BsonIdToSId(tmp["_id"])
		biddingH, _ := MgoB.FindById("bidding", biddingId, nil)
		if biddingH != nil && len(*biddingH) > 0 {
			tmp["is_high"] = true
			for _, field := range GF.Env.Fields {
				if val, ok := (*biddingH)[field]; ok && val != nil {
					tmp[field] = val
				}
			}
			MgoTo.SaveByOriID(GF.MongoT.Coll, tmp)
		} else {
			tmp["is_high"] = false
			MgoTo.SaveByOriID(GF.MongoT.Coll, tmp)
		}
	}
}