util.go 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204
  1. package util
  2. import (
  3. "fmt"
  4. . "jy/mongodbutil"
  5. qu "qfw/util"
  6. "regexp"
  7. "strconv"
  8. . "gopkg.in/mgo.v2/bson"
  9. )
  10. //敏感词
  11. type DFA struct {
  12. Link map[string]interface{}
  13. }
  14. //定义字典树
  15. type Trie struct {
  16. y bool
  17. c map[rune]*Trie
  18. }
  19. var syncint chan bool //获取下标锁
  20. var Config map[string]interface{}
  21. var Se = qu.SimpleEncrypt{Key: "topnet@extract"}
  22. var BrandRules map[string]map[string]string
  23. var GoodsConfig []string
  24. var BrandConfig []string
  25. var IsPriceNumber bool //是否开启价格和个数抽取
  26. var PriceNumberConfig map[string]string
  27. var PriceNumberReg map[string]*regexp.Regexp
  28. var GoodsGet *DFA //商品
  29. var BrandGet *DFA //品牌
  30. var IsBrandGoods bool //是否开启品牌抽取
  31. var SaveResult, FieldsFind, IsSaveTag, SaveBlock, QualityAudit, Ffield bool
  32. func init() {
  33. syncint = make(chan bool, 1)
  34. }
  35. func UtilInit() {
  36. initCap := qu.IntAll(Config["dbsize"])
  37. addr := qu.ObjToString(Config["mgodb"])
  38. dbname := qu.ObjToString(Config["dbname"])
  39. Mgo = MgoFactory(initCap, initCap*3, 120, addr, dbname)
  40. SaveResult, _ = Config["saveresult"].(bool)
  41. FieldsFind, _ = Config["fieldsfind"].(bool)
  42. IsSaveTag, _ = Config["iscltlog"].(bool)
  43. SaveBlock, _ = Config["saveblock"].(bool)
  44. QualityAudit, _ = Config["qualityaudit"].(bool)
  45. Ffield, _ = Config["ffield"].(bool)
  46. PriceNumberReg = make(map[string]*regexp.Regexp)
  47. for k, v := range PriceNumberConfig {
  48. PriceNumberReg[k] = regexp.MustCompile(v)
  49. }
  50. }
  51. func GetSyncIndex(code string) string {
  52. tmp := ""
  53. syncint <- true
  54. Mgo.Update("rule_code", `{"code":"`+code+`"}`, M{"$inc": M{"index": 1}}, true, false)
  55. data, _ := Mgo.FindOne("rule_code", `{"code":"`+code+`"}`)
  56. <-syncint
  57. if len(*data) > 0 {
  58. tmp = code + "_" + fmt.Sprint((*data)["index"])
  59. }
  60. return tmp
  61. }
  62. //nfields非复制字段集
  63. func DeepCopy(value interface{}) interface{} {
  64. if valueMap, ok := value.(map[string]interface{}); ok {
  65. newMap := make(map[string]interface{})
  66. for k, v := range valueMap {
  67. newMap[k] = DeepCopy(v)
  68. }
  69. return newMap
  70. } else if valueSlice, ok := value.([]interface{}); ok {
  71. newSlice := make([]interface{}, len(valueSlice))
  72. for k, v := range valueSlice {
  73. newSlice[k] = DeepCopy(v)
  74. }
  75. return newSlice
  76. }
  77. return value
  78. }
  79. func (d *DFA) AddWord(keys ...string) {
  80. d.AddWordAll(true, keys...)
  81. }
  82. func (d *DFA) AddWordAll(haskey bool, keys ...string) {
  83. if d.Link == nil {
  84. d.Link = make(map[string]interface{})
  85. }
  86. for _, key := range keys {
  87. nowMap := &d.Link
  88. for i := 0; i < len(key); i++ {
  89. kc := key[i : i+1]
  90. if v, ok := (*nowMap)[kc]; ok {
  91. nowMap, _ = v.(*map[string]interface{})
  92. } else {
  93. newMap := map[string]interface{}{}
  94. newMap["YN"] = "0"
  95. (*nowMap)[kc] = &newMap
  96. nowMap = &newMap
  97. }
  98. if i == len(key)-1 {
  99. (*nowMap)["YN"] = "1"
  100. if haskey {
  101. (*nowMap)["K"] = key
  102. }
  103. }
  104. }
  105. }
  106. }
  107. func (d *DFA) CheckSensitiveWord(src string) string {
  108. pos := 0
  109. nowMap := &d.Link
  110. res := ""
  111. for i := 0; i < len(src); i++ {
  112. word := src[i : i+1]
  113. nowMap, _ = (*nowMap)[word].(*map[string]interface{})
  114. if nowMap != nil { // 存在,则判断是否为最后一个
  115. if pos == 0 {
  116. pos = i
  117. }
  118. if "1" == qu.ObjToString((*nowMap)["YN"]) { // 如果为最后一个匹配规则,结束循环,返回匹配标识数
  119. res = qu.ObjToString((*nowMap)["K"])
  120. //pos = 0
  121. //break
  122. }
  123. } else {
  124. if res != "" {
  125. break
  126. } else {
  127. nowMap = &d.Link
  128. if pos > 0 {
  129. i = pos
  130. pos = 0
  131. }
  132. }
  133. }
  134. }
  135. return res
  136. }
  137. func (t *Trie) AddWords(words ...string) {
  138. cur := t
  139. for _, v := range words {
  140. for _, one := range v {
  141. if cur.c == nil {
  142. cur.c = map[rune]*Trie{}
  143. }
  144. n := cur.c[one]
  145. if n == nil {
  146. n = &Trie{}
  147. cur.c[one] = n
  148. }
  149. cur = n
  150. }
  151. cur.y = true
  152. cur = t
  153. }
  154. }
  155. func (t *Trie) Get(word string) bool {
  156. cur := t
  157. for _, one := range word {
  158. n := cur.c[one]
  159. if n == nil {
  160. return false
  161. }
  162. cur = n
  163. }
  164. return cur.y
  165. }
  166. //初始化商品
  167. func InitGoods() {
  168. GoodsGet = &DFA{}
  169. GoodsGet.AddWord(GoodsConfig...)
  170. }
  171. //初始化品牌
  172. func InitBrand() {
  173. BrandGet = &DFA{}
  174. BrandGet.AddWord(BrandConfig...)
  175. }
  176. func FloatFormat(val float64, length int) float64 {
  177. tmp, err := strconv.ParseFloat(strconv.FormatFloat(val, 'f', length, 64), 64)
  178. if err != nil {
  179. return 0
  180. } else {
  181. return tmp
  182. }
  183. }