package util import ( "fmt" . "jy/mongodbutil" qu "qfw/util" "regexp" "strconv" . "gopkg.in/mgo.v2/bson" ) //敏感词 type DFA struct { Link map[string]interface{} } //定义字典树 type Trie struct { y bool c map[rune]*Trie } var syncint chan bool //获取下标锁 var Config map[string]interface{} var Se = qu.SimpleEncrypt{Key: "topnet@extract"} var BrandRules map[string]map[string]string var GoodsConfig []string var BrandConfig []string var IsPriceNumber bool //是否开启价格和个数抽取 var PriceNumberConfig map[string]string var PriceNumberReg map[string]*regexp.Regexp var GoodsGet *DFA //商品 var BrandGet *DFA //品牌 var IsBrandGoods bool //是否开启品牌抽取 var SaveResult, FieldsFind, IsSaveTag, SaveBlock, QualityAudit, Ffield bool func init() { syncint = make(chan bool, 1) } func UtilInit() { initCap := qu.IntAll(Config["dbsize"]) addr := qu.ObjToString(Config["mgodb"]) dbname := qu.ObjToString(Config["dbname"]) Mgo = MgoFactory(initCap, initCap*3, 120, addr, dbname) SaveResult, _ = Config["saveresult"].(bool) FieldsFind, _ = Config["fieldsfind"].(bool) IsSaveTag, _ = Config["iscltlog"].(bool) SaveBlock, _ = Config["saveblock"].(bool) QualityAudit, _ = Config["qualityaudit"].(bool) Ffield, _ = Config["ffield"].(bool) PriceNumberReg = make(map[string]*regexp.Regexp) for k, v := range PriceNumberConfig { PriceNumberReg[k] = regexp.MustCompile(v) } } func GetSyncIndex(code string) string { tmp := "" syncint <- true Mgo.Update("rule_code", `{"code":"`+code+`"}`, M{"$inc": M{"index": 1}}, true, false) data, _ := Mgo.FindOne("rule_code", `{"code":"`+code+`"}`) <-syncint if len(*data) > 0 { tmp = code + "_" + fmt.Sprint((*data)["index"]) } return tmp } //nfields非复制字段集 func DeepCopy(value interface{}) interface{} { if valueMap, ok := value.(map[string]interface{}); ok { newMap := make(map[string]interface{}) for k, v := range valueMap { newMap[k] = DeepCopy(v) } return newMap } else if valueSlice, ok := value.([]interface{}); ok { newSlice := make([]interface{}, len(valueSlice)) for k, v := range valueSlice { newSlice[k] = DeepCopy(v) } return newSlice } return value } func (d *DFA) AddWord(keys ...string) { d.AddWordAll(true, keys...) } func (d *DFA) AddWordAll(haskey bool, keys ...string) { if d.Link == nil { d.Link = make(map[string]interface{}) } for _, key := range keys { nowMap := &d.Link for i := 0; i < len(key); i++ { kc := key[i : i+1] if v, ok := (*nowMap)[kc]; ok { nowMap, _ = v.(*map[string]interface{}) } else { newMap := map[string]interface{}{} newMap["YN"] = "0" (*nowMap)[kc] = &newMap nowMap = &newMap } if i == len(key)-1 { (*nowMap)["YN"] = "1" if haskey { (*nowMap)["K"] = key } } } } } func (d *DFA) CheckSensitiveWord(src string) string { pos := 0 nowMap := &d.Link res := "" for i := 0; i < len(src); i++ { word := src[i : i+1] nowMap, _ = (*nowMap)[word].(*map[string]interface{}) if nowMap != nil { // 存在,则判断是否为最后一个 if pos == 0 { pos = i } if "1" == qu.ObjToString((*nowMap)["YN"]) { // 如果为最后一个匹配规则,结束循环,返回匹配标识数 res = qu.ObjToString((*nowMap)["K"]) //pos = 0 //break } } else { if res != "" { break } else { nowMap = &d.Link if pos > 0 { i = pos pos = 0 } } } } return res } func (t *Trie) AddWords(words ...string) { cur := t for _, v := range words { for _, one := range v { if cur.c == nil { cur.c = map[rune]*Trie{} } n := cur.c[one] if n == nil { n = &Trie{} cur.c[one] = n } cur = n } cur.y = true cur = t } } func (t *Trie) Get(word string) bool { cur := t for _, one := range word { n := cur.c[one] if n == nil { return false } cur = n } return cur.y } //初始化商品 func InitGoods() { GoodsGet = &DFA{} GoodsGet.AddWord(GoodsConfig...) } //初始化品牌 func InitBrand() { BrandGet = &DFA{} BrandGet.AddWord(BrandConfig...) } func FloatFormat(val float64, length int) float64 { tmp, err := strconv.ParseFloat(strconv.FormatFloat(val, 'f', length, 64), 64) if err != nil { return 0 } else { return tmp } }