package main import ( util "app.yhyue.com/data_processing/common_utils" "encoding/json" "fmt" "github.com/spf13/viper" "go.uber.org/zap" "jygit.jydev.jianyu360.cn/data_processing/common_utils/log" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb" ) var ( MongoTool *mongodb.MongodbSim QyStypeMap map[string]string CompanyStatusMap map[string]string AddressMap map[string]*City AddressOldMap map[string]*City CollArr []string // std 程序需要读取的数据表 ) func init() { InitLog() err := InitConfig() if err != nil { log.Info("init", zap.Any("InitConfig", err)) } InitMgo() InitQyStype() InitCompanyStatus() InitAddress() err = seg.LoadDictionary("dict.txt") if err != nil { log.Fatal("init", zap.Any("LoadDictionary err", err)) return } //qyxy_std 需要处理的数据表 CollArr = []string{"company_base", "company_employee", "company_history_name", "company_partner", "annual_report_base", "annual_report_website"} } func InitMgo() { MongoTool = &mongodb.MongodbSim{ MongodbAddr: GF.Env.Addr, Size: GF.Env.Dbsize, DbName: GF.Env.Dbname, UserName: GF.Env.Username, Password: GF.Env.Password, } MongoTool.InitPool() } func InitConfig() (err error) { viper.SetConfigFile("config.toml") // 指定配置文件路径 viper.SetConfigName("config") // 配置文件名称(无扩展名) viper.SetConfigType("toml") // 如果配置文件的名称中没有扩展名,则需要配置此项 viper.AddConfigPath("./") viper.AddConfigPath("./conf/") // 还可以在工作目录中查找配置 viper.AddConfigPath("../conf/") // 还可以在工作目录中查找配置 err = viper.ReadInConfig() // 查找并读取配置文件 if err != nil { // 处理读取配置文件的错误 return } err = viper.Unmarshal(&GF) return err } func InitLog() { err := log.InitLog( log.Path("./logs/log.out"), //log.Path(""), log.Level("info"), log.Compress(true), log.MaxSize(10), log.MaxBackups(10), log.MaxAge(7), log.Format("json"), ) if err != nil { fmt.Printf("InitLog failed: %v\n", err) } } func InitQyStype() { defer util.Catch() util.Debug("Init QyStype...") QyStypeMap = map[string]string{} qystype, _ := MongoTool.Find("qystype", nil, nil, nil, false, -1, -1) for _, tmp := range *qystype { name := util.ObjToString(tmp["name"]) prename := util.ObjToString(tmp["prename"]) QyStypeMap[name] = prename } } func InitCompanyStatus() { defer util.Catch() util.Debug("Init CompanyStatus...") CompanyStatusMap = map[string]string{} status, _ := MongoTool.Find("company_status", nil, nil, nil, false, -1, -1) for _, tmp := range *status { old_status := util.ObjToString(tmp["old"]) new_status := util.ObjToString(tmp["new"]) CompanyStatusMap[old_status] = new_status } } type City struct { Code string `json:"code"` Province string `json:"province"` City string `json:"city"` District string `json:"district"` } // CodeMap 区域code补全 var CodeMap = map[int]string{ 2: "0000", 4: "00", } func InitAddress() { defer util.Catch() util.Debug("Init Address...") AddressMap = map[string]*City{} AddressOldMap = map[string]*City{} sess := MongoTool.GetMgoConn() defer MongoTool.DestoryMongoConn(sess) result := sess.DB("mixdata").C("address_new_2020").Find(nil).Iter() count := 0 for tmp := make(map[string]interface{}); result.Next(&tmp); count++ { if count%50000 == 0 { log.Info("InitAddress", zap.Int("current", count)) } code := util.ObjToString(tmp["code"]) codeLen := len(code) if codeLen > 6 { continue } if t_code := CodeMap[codeLen]; t_code != "" { code = code + t_code } remark := fmt.Sprint(tmp["Remarks"]) city := &City{} tmpjson, err := json.Marshal(tmp) if err == nil { json.Unmarshal(tmpjson, city) } if remark == "已作废" { AddressOldMap[code] = city } else { AddressMap[code] = city } } log.Info("InitAddress", zap.Any("AddressMap", len(AddressMap)), zap.Int("AddressOldMap", len(AddressOldMap))) }