jy-data-extract/udpdataclear/udpSensitiveWords/util/word.go

package util

import (
	"encoding/json"
	"log"
	"strings"
	"unicode"
	"unicode/utf8"
)

func dealWithScoreRules(name string) (string,bool) {
	new_name,isok :="",false
	for _, c := range name {
		if !(unicode.Is(unicode.Han, c) || unicode.IsNumber(c) || unicode.IsLetter(c)) {
			return "",false
		}
	}
	query:= `{"query":{"bool":{"must":[{"query_string":{"default_field":"`+es_index+`.name_word","query":"`+name+`"}}],"must_not":[],"should":[]}},"from":0,"size":1,"sort":[],"facets":{}}`
	//默认取最高分-分析多个分-遍历器查询
	tmp := make(map[string]interface{})
	json.Unmarshal([]byte(query),&tmp)
	searchResult, err := Client_Es.Search().Index(es_index).Type(es_type).Source(tmp).Do()
	if err != nil {
		log.Println("从ES查询出错", err.Error())
		return new_name,isok
	}
	data := make(map[string]interface{},0)
	if searchResult.Hits != nil {
		for _, hit := range searchResult.Hits.Hits {
			json.Unmarshal(*hit.Source, &data)
		}
	}
	if len(data)>0 && data != nil {
		new_name = objToString(data["name"])
	}
	if new_name!="" { //分析hit比例
		total,hit := dealWithWordsRules(name,new_name)
		if float64(hit)/float64(total)>=0.8 {
			isok = true
		}
	}
	return new_name,isok
}

//击中数量以及比例
func dealWithWordsRules(info_name string ,source_name string) (int,int){
	total,hit :=0,0
	nameArr,_ := calculateWordCount(info_name)
	_,total = calculateWordCount(source_name)
	for _,v1 := range nameArr {
		if strings.Contains(source_name,v1) {
			hit++
		}
	}
	return total,hit
}

//分词结果
func calculateWordCount(name string) ([]string,int) {

	arr, space := make([]string, 0), 2
	total := utf8.RuneCountInString(name) - (space - 1)
	if name == "" || total <= 0 {
		return arr, 0
	}
	nameRune := []rune(name)
	for i := 0; i < total; i++ {
		new_str := string(nameRune[i : space+i])
		arr = append(arr, new_str)
	}
	return arr, len(arr)
}

func objToString(old interface{}) string {
	if nil == old {
		return ""
	} else {
		r, _ := old.(string)
		return r
	}
}