es_index_data/createEsIndex/es_test.go

package main

import (
	"context"
	"encoding/json"
	"esindex/config"
	"fmt"
	"github.com/olivere/elastic/v7"
	"go.uber.org/zap"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
	"strings"
	"testing"
)

func TestMatchService(t *testing.T) {
	data := `{

    "_id" : "6422d91e779467cff1a84885",
    "area" : "全国",
    "city" : "",
    "extracttype" : 0,
    "s_sha" : "d7cc66ac91dc6551991df0a37331b628de4c70973c6844f1ee6ef1c2d4e29e95",
    "jsondata" : {
        "area_city_district" : "福建",
        "buyer" : "莆田市第一医院",
        "item" : " 货物/医药品/医用材料/其他医用材料",
        "agency" : "福建省荔卫药械招标服务有限公司"
    },
    "channel" : "地方公告"
}`

	var obj map[string]interface{}
	if err := json.Unmarshal([]byte(data), &obj); err != nil {
		panic(err)
	}

	objectType := MatchService(obj)
	fmt.Println("objectType=>", objectType)
}

func TestGetMappting(t *testing.T) {
	client, _ := elastic.NewClient(
		elastic.SetURL(config.Conf.DB.Es.Addr),
		elastic.SetBasicAuth(config.Conf.DB.Es.Username, config.Conf.DB.Es.Password),
		elastic.SetSniff(false),
	)

	index := config.Conf.DB.Es.IndexB
	// 获取 Elasticsearch 索引的 mapping 信息
	mapping, err := client.GetMapping().Index(index).Do(context.Background())
	if err != nil {
		fmt.Println("Error getting Elasticsearch mapping:", err)
		return
	}

	indexName, _ := GetIndexName(client, index)
	properties := mapping[indexName].(map[string]interface{})["mappings"].(map[string]interface{})["properties"].(map[string]interface{})

	var errField = make([]string, 0)
	var okField = make([]string, 0)
	var analyzerMap = make(map[string]string) // 分词信息
	var esMap = make(map[string]string)       //存储es 字段类型
	//
	for field, ftype := range BiddingField {
		eftypeMap, _ := properties[field].(map[string]interface{})
		var etype string
		var analyzer string
		if fftype, ok := eftypeMap["type"]; ok {
			etype = fftype.(string)
			esMap[field] = etype
		}
		if ffanalyzer, ok := eftypeMap["analyzer"]; ok {
			analyzer = ffanalyzer.(string)
			analyzerMap[field] = analyzer
		}

		if ftype != "" {
			if chargeType(ftype, etype) {
				okField = append(okField, field)
			} else {
				errField = append(errField, field)
			}
		} else {
			if field == "_id" {
				continue
			} else if field == "purchasinglist" || field == "package" || field == "winnerorder" || field == "procurementlist" {
				if eproperties, ok := eftypeMap["properties"]; ok {
					if eproMap, ok := eproperties.(map[string]interface{}); ok {
						for k, v := range eproMap {
							if innerMap, ok := v.(map[string]interface{}); ok {
								if innerType, ok := innerMap["type"]; ok {
									innerLevel := BiddingLevelField[field]
									esMap[fmt.Sprintf("%s.%s", field, k)] = innerType.(string)
									if chargeType(innerLevel[k], innerType.(string)) {
										okField = append(okField, fmt.Sprintf("%s.%s", field, k))
									} else {
										errField = append(errField, fmt.Sprintf("%s.%s", field, k))
									}
								}
							}
						}
					}
				}
			}
		}
	}

	if len(errField) > 0 {
		log.Info("test", zap.Int("错误字段数量", len(errField)))
		for _, field := range errField {
			if strings.Contains(field, ".") {
				fe := strings.Split(field, ".")
				log.Info(fmt.Sprintf("%s 字段类型错误", field), zap.String(fmt.Sprintf("数据库类型为:%s,但是es字段类型是:", BiddingLevelField[fe[0]][fe[1]]), esMap[field]))
			} else {
				log.Info(fmt.Sprintf("%s 字段类型错误", field), zap.String(fmt.Sprintf("数据库类型为:%s,但是es字段类型是:", BiddingField[field]), esMap[field]))
			}
		}
	} else {
		log.Info("es 字段类型检测结束，", zap.Int("所有字段都符合,检测字段数量为:", len(okField)))
	}

}

func TestGetIndexName(t *testing.T) {
	client, _ := elastic.NewClient(
		elastic.SetURL(config.Conf.DB.Es.Addr),
		elastic.SetBasicAuth(config.Conf.DB.Es.Username, config.Conf.DB.Es.Password),
		elastic.SetSniff(false),
	)

	index := "bidding_v2"
	//index := config.Conf.DB.Es.IndexB

	name, _ := GetIndexName(client, index)
	fmt.Println("name ->", name)
	fmt.Println(name)
}

func TestBuyer(t *testing.T) {
	rowsPerPage := 1000
	currentPage := 1

	var total int
	for {
		fmt.Println("currentPage", currentPage)
		arrEs := make([]map[string]interface{}, 0)
		offset := (currentPage - 1) * rowsPerPage
		query := fmt.Sprintf(`
             SELECT * from goods
			
            LIMIT %d, %d;
        `, offset, rowsPerPage)

		result := Mysql.SelectBySql(query)

		if len(*result) > 0 {
			for _, re := range *result {
				arrEs = append(arrEs, re)
			}

		}

		total = total + len(*result)
		if len(*result) < rowsPerPage {
			break
		}

		// 继续查询下一页
		currentPage++
	}

	fmt.Println("over --------")
	fmt.Println("total --------", total)
}

func TestIsHanStart(t *testing.T) {
	name := "\\u001c 陈巴尔虎旗天顺矿业有限责任公司"
	fmt.Println("aa", IsCompanyName(name))

	//fmt.Println("uni", IsUnicodeStart(name))
	fmt.Println("name", getCompanyName(name))
	name = "RT农业发展（乌鲁木齐）有限责任公司"
	//fmt.Println("uni", IsUnicodeStart(name))

	fmt.Println("bb", IsCompanyName(name))
	fmt.Println("name", getCompanyName(name))
	name = "(宁波)综命能源服务有限公司"
	fmt.Println("name", getCompanyName(name))
	//fmt.Println("uni", IsUnicodeStart(name))
	fmt.Println("cc", IsCompanyName(name))

	a := 15
	b := 2
	fmt.Println((a / b) + 1)

}