data_field_dispose/field_sync/file.go

package main

import (
	"bufio"
	"fmt"
	"go.uber.org/zap"
	"io"
	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
	"os"
	"regexp"
	"strconv"
	"strings"
)

var (
	CmmonDFA        *DFA                                             //常用字
	NotCommonDFA    *DFA                                             //不常用字
	TimesLimit      int                                              //常用字界限
	CmmLmt, NcmmLmt float64                                          //更新界限
	HanReg          = regexp.MustCompile("[\u4e00-\u9fa5]+")         //中文正则
	SpaceReg        = regexp.MustCompile("[\\s\u3000\u2003\u00a0]+") //空格正则
	SpecialReg      = regexp.MustCompile("图片(\\d)+")                 //
)

func InitFileInfo() {
	TimesLimit = 500
	CmmLmt = 0.5
	NcmmLmt = 0.1
	CmmonDFA = &DFA{}
	NotCommonDFA = &DFA{}
	LoadDict("CommonDict.txt") //初始化常用字典
}

//DFA
type DFA struct {
	Link map[string]interface{}
}

func (d *DFA) AddWord(keys ...string) {
	d.AddWordAll(true, keys...)
}
func (d *DFA) AddWordAll(haskey bool, keys ...string) {
	if d.Link == nil {
		d.Link = make(map[string]interface{})
	}
	for _, key := range keys {
		nowMap := &d.Link
		for i := 0; i < len(key); i++ {
			kc := key[i : i+1]
			if v, ok := (*nowMap)[kc]; ok {
				nowMap, _ = v.(*map[string]interface{})
			} else {
				newMap := map[string]interface{}{}
				newMap["YN"] = "0"
				(*nowMap)[kc] = &newMap
				nowMap = &newMap
			}
			if i == len(key)-1 {
				(*nowMap)["YN"] = "1"
				if haskey {
					(*nowMap)["K"] = key
				}
			}
		}
	}
}

func (d *DFA) CheckSensitiveWord(src string) []string {
	res := make([]string, 0)
	for j := 0; j < len(src); j++ {
		nowMap := &d.Link
		for i := j; i < len(src); i++ {
			word := src[i : i+1]
			nowMap, _ = (*nowMap)[word].(*map[string]interface{})
			if nowMap != nil { // 存在，则判断是否为最后一个
				if "1" == util.ObjToString((*nowMap)["YN"]) {
					s := util.ObjToString((*nowMap)["K"])
					res = append(res, s)
				}
			} else {
				break
			}
		}
	}
	return res
}

//加载统计的常用词
func LoadDict(path string) {
	dictFile, err := os.Open(path)
	if err != nil {
		log.Error("Load Common.txt Error")
		os.Exit(-1)
	}
	defer dictFile.Close()
	reader := bufio.NewReader(dictFile)
	var (
		text      string
		frequency int
	)

	// 逐行读入分词
	line := 0
	for {
		line++
		size, fsErr := fmt.Fscanln(reader, &text, &frequency) //读每行赋值
		if fsErr == io.EOF {                                  //读取到结尾
			break
		}
		if size == 2 { //正确数据
			if frequency >= TimesLimit { //常用字
				CmmonDFA.AddWord(text)
			} else { //非常用字
				NotCommonDFA.AddWord(text)
			}
		} else {
			log.Error("Read Line Error:", zap.Int("line", line))
		}
	}
}

//解析附件
func AnalysisFile(filetext string) bool {
	defer util.Catch()
	//过滤空格
	filetextTmp := SpaceReg.ReplaceAllString(filetext, "")
	if filetextTmp == "" { //附件为空
		return false
	}
	//特殊情况：图片0 图片1
	filetextTmp = SpecialReg.ReplaceAllString(filetextTmp, "")
	if filetextTmp == "" { //附件为空
		return false
	}
	//中文匹配
	HanArr := HanReg.FindAllString(filetextTmp, -1)
	hanText := strings.Join(HanArr, "")
	hanTextLen := len([]rune(hanText))
	//长度过滤
	if hanTextLen <= 100 {
		return false
	}
	//qu.Debug(hanTextLen, hanText)
	commonArr := CmmonDFA.CheckSensitiveWord(hanText)
	commonLen := len(commonArr)
	//qu.Debug(commonLen, commonArr)
	//commonText := strings.Join(commonArr, "")
	notCommonArr := NotCommonDFA.CheckSensitiveWord(hanText)
	notCommonLen := len(notCommonArr)
	//qu.Debug(notCommonLen, notCommonArr)
	//解析常用字和非常用字占比(由于常用字或非常用字集不全，会导致比例相加不为100%)
	commonRatio := float64(commonLen) / float64(hanTextLen)
	commonRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", commonRatio), 64)
	//qu.Debug(commonRatio)
	notCommonRatio := float64(notCommonLen) / float64(hanTextLen)
	notCommonRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", notCommonRatio), 64)
	if commonRatio >= CmmLmt && notCommonRatio < NcmmLmt {
		return true
	}
	return false
}

//测试方法
func AnalysisFileTest(detail string) (bool, string, int, float64, float64) {
	//qu.Debug(detail)
	defer util.Catch()
	//过滤空格
	filetextTmp := SpaceReg.ReplaceAllString(detail, "")
	if filetextTmp == "" { //附件为空
		return false, "", 0, 0, 0
	}
	//特殊情况：图片0 图片1
	filetextTmp = SpecialReg.ReplaceAllString(filetextTmp, "")
	if filetextTmp == "" { //附件为空
		return false, "", 1, 0, 0
	}
	//中文匹配
	HanArr := HanReg.FindAllString(filetextTmp, -1)
	hanText := strings.Join(HanArr, "")
	hanTextLen := len([]rune(hanText))
	//长度过滤
	if hanTextLen <= 100 {
		return false, "", 2, 0, 0
	}
	//qu.Debug(textLen, text)
	commonArr := CmmonDFA.CheckSensitiveWord(hanText)
	commonLen := len(commonArr)
	//commonText := strings.Join(commonArr, "")
	notCommonArr := NotCommonDFA.CheckSensitiveWord(hanText)
	notCommonLen := len(notCommonArr)
	//notCommonText := strings.Join(notCommonArr, "")
	//解析常用字和非常用字占比(由于常用字或非常用字集不全，会导致比例相加不为100%)
	commonRatio := float64(commonLen) / float64(hanTextLen)
	commonRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", commonRatio), 64)
	notCommonRatio := float64(notCommonLen) / float64(hanTextLen)
	notCommonRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", notCommonRatio), 64)
	return true, filetextTmp, 10, commonRatio, notCommonRatio
}