package main import ( util "app.yhyue.com/data_processing/common_utils" "app.yhyue.com/data_processing/common_utils/elastic" "app.yhyue.com/data_processing/common_utils/log" "app.yhyue.com/data_processing/common_utils/mongodb" "app.yhyue.com/data_processing/common_utils/mysqldb" "context" "esindex/config" "fmt" es7 "github.com/olivere/elastic/v7" "go.uber.org/zap" "os" "strings" ) var ( ProjectField = make(map[string]string, 500) //项目字段 ProjectListF = make(map[string]string, 200) BiddingField = make(map[string]string, 200) //bidding_processing_field, level=1 最外层字段, BiddingLevelField = make(map[string]map[string]string) //level=2 的第二层字段 ) // InitLog @Description // @Author J 2022/7/26 15:30 func InitLog() { logcfg := config.Conf.Log err := log.InitLog( log.Path(logcfg.LogPath), log.Level(logcfg.LogLevel), log.Compress(logcfg.Compress), log.MaxSize(logcfg.MaxSize), log.MaxBackups(logcfg.MaxBackups), log.MaxAge(logcfg.MaxAge), log.Format(logcfg.Format), ) if err != nil { fmt.Printf("InitLog failed: %v\n", err) os.Exit(1) } } func InitMgo() { MgoB = &mongodb.MongodbSim{ MongodbAddr: config.Conf.DB.MongoB.Addr, DbName: config.Conf.DB.MongoB.Dbname, Size: config.Conf.DB.MongoB.Size, UserName: config.Conf.DB.MongoB.User, Password: config.Conf.DB.MongoB.Password, } MgoB.InitPool() MgoP = &mongodb.MongodbSim{ MongodbAddr: config.Conf.DB.MongoP.Addr, DbName: config.Conf.DB.MongoP.Dbname, Size: config.Conf.DB.MongoP.Size, UserName: config.Conf.DB.MongoP.User, Password: config.Conf.DB.MongoP.Password, } MgoP.InitPool() MgoQ = &mongodb.MongodbSim{ MongodbAddr: config.Conf.DB.MongoQ.Addr, DbName: config.Conf.DB.MongoQ.Dbname, Size: config.Conf.DB.MongoQ.Size, UserName: config.Conf.DB.MongoQ.User, Password: config.Conf.DB.MongoQ.Password, } MgoQ.InitPool() MysqlB = &mysqldb.Mysql{ Address: config.Conf.DB.MysqlB.Addr, DBName: config.Conf.DB.MysqlB.Dbname, UserName: config.Conf.DB.MysqlB.Username, PassWord: config.Conf.DB.MysqlB.Password, } MysqlB.Init() } func InitEs() { Es = &elastic.Elastic{ S_esurl: config.Conf.DB.Es.Addr, I_size: config.Conf.DB.Es.Size, Username: config.Conf.DB.Es.Username, Password: config.Conf.DB.Es.Password, } Es.InitElasticSize() Es1 = &elastic.Elastic{ S_esurl: config.Conf.DB.Es.AddrP, I_size: config.Conf.DB.Es.Size, Username: config.Conf.DB.Es.Username, Password: config.Conf.DB.Es.Password, } Es1.InitElasticSize() } func InitField() { info, _ := MgoB.Find("bidding_processing_field", `{"stype": "project"}`, nil, nil, false, -1, -1) if len(*info) > 0 { for _, m := range *info { if util.IntAll(m["level"]) == 1 { ProjectField[util.ObjToString(m["field"])] = util.ObjToString(m["ftype"]) } else if util.IntAll(m["level"]) == 2 { ProjectListF[util.ObjToString(m["field"])] = util.ObjToString(m["ftype"]) } } } log.Info("InitField", zap.Int("ProjectField", len(ProjectField)), zap.Int("ProjectListF", len(ProjectListF))) } //InitEsBiddingField 初始化 bidding 索引字段 func InitEsBiddingField() { info, _ := MgoB.Find("bidding_processing_field", `{"stype": "bidding"}`, nil, nil, false, -1, -1) if len(*info) > 0 { for _, m := range *info { if util.IntAll(m["level"]) == 1 { BiddingField[util.ObjToString(m["field"])] = util.ObjToString(m["ftype"]) } else if util.IntAll(m["level"]) == 2 { pfield := util.ObjToString(m["pfield"]) pfieldMap := BiddingLevelField[pfield] if pfieldMap == nil { pfieldMap = make(map[string]string, 0) } pfieldMap[util.ObjToString(m["field"])] = util.ObjToString(m["ftype"]) BiddingLevelField[pfield] = pfieldMap } } } log.Info("InitEsBiddingField", zap.Int("BiddingField es 一级字段数量", len(BiddingField))) log.Info("InitEsBiddingField", zap.Int("BiddingLevelField es 二级字段数量", len(BiddingLevelField))) } //verifyESFields 验证es 定义字段类型和 MongoDB 数据字段 func verifyESFields() { log.Info("verifyESFields", zap.String("开始类型检测", "")) client, _ := es7.NewClient( es7.SetURL(config.Conf.DB.Es.Addr), es7.SetBasicAuth(config.Conf.DB.Es.Username, config.Conf.DB.Es.Password), es7.SetSniff(false), ) index := config.Conf.DB.Es.IndexB //索引表 bidding // 获取 Elasticsearch 索引的 mapping 信息 mapping, err := client.GetMapping().Index(index).Do(context.Background()) if err != nil { log.Info("verifyESFields", zap.Any("getting Elasticsearch mapping:", err)) } indexName, _ := GetIndexName(client, index) if indexName == "" { log.Info("verifyESFields", zap.String("索引不存在,请检查索引", index)) return } properties := mapping[indexName].(map[string]interface{})["mappings"].(map[string]interface{})["properties"].(map[string]interface{}) var errField = make([]string, 0) var okField = make([]string, 0) var analyzerMap = make(map[string]string) // 分词信息 var esMap = make(map[string]string) //存储es 字段类型 // for field, ftype := range BiddingField { eftypeMap, _ := properties[field].(map[string]interface{}) var etype string var analyzer string if fftype, ok := eftypeMap["type"]; ok { etype = fftype.(string) esMap[field] = etype } if ffanalyzer, ok := eftypeMap["analyzer"]; ok { analyzer = ffanalyzer.(string) analyzerMap[field] = analyzer } if ftype != "" { if chargeType(ftype, etype) { okField = append(okField, field) } else { errField = append(errField, field) } } else { if field == "_id" { continue } else if field == "purchasinglist" || field == "package" || field == "winnerorder" || field == "procurementlist" { if eproperties, ok := eftypeMap["properties"]; ok { if eproMap, ok := eproperties.(map[string]interface{}); ok { for k, v := range eproMap { if innerMap, ok := v.(map[string]interface{}); ok { if innerType, ok := innerMap["type"]; ok { innerLevel := BiddingLevelField[field] esMap[fmt.Sprintf("%s.%s", field, k)] = innerType.(string) if chargeType(innerLevel[k], innerType.(string)) { okField = append(okField, fmt.Sprintf("%s.%s", field, k)) } else { errField = append(errField, fmt.Sprintf("%s.%s", field, k)) } } } } } } } } } if len(errField) > 0 { log.Info("verifyESFields", zap.Int("错误字段数量", len(errField))) for _, field := range errField { if strings.Contains(field, ".") { fe := strings.Split(field, ".") log.Info(fmt.Sprintf("%s 字段类型错误", field), zap.String(fmt.Sprintf("数据库类型为:%s,但是es字段类型是:", BiddingLevelField[fe[0]][fe[1]]), esMap[field])) } else { log.Info(fmt.Sprintf("%s 字段类型错误", field), zap.String(fmt.Sprintf("数据库类型为:%s,但是es字段类型是:", BiddingField[field]), esMap[field])) } } } else { log.Info("es 字段类型检测结束,", zap.Int("所有字段都符合,检测字段数量为:", len(okField))) } } func GetIndexName(client *es7.Client, name string) (string, error) { // 判断 name 是否为一个别名 res, err := client.Aliases().Alias(name).Do(context.Background()) if err != nil { // 错误处理 if err.(*es7.Error).Status != 404 && err.(*es7.Error).Details != nil { return "", err } } if res != nil { for k, v := range res.Indices { for _, vv := range v.Aliases { if vv.AliasName == name { return k, nil } } } } // 判断 name 是否为一个正式索引名称 resa, err := client.IndexExists(name).Do(context.Background()) if err != nil { // 错误处理 return "", err } if resa { return name, nil } // 如果 name 既不是别名,也不是正式索引名称,则返回空字符串 return "", nil }