|
|
@@ -2,12 +2,14 @@ package client
|
|
|
|
|
|
import (
|
|
|
"encoding/json"
|
|
|
+ "errors"
|
|
|
"fmt"
|
|
|
"io/ioutil"
|
|
|
"log"
|
|
|
mongoutil "qfw/mongodb"
|
|
|
qu "qfw/util"
|
|
|
"regexp"
|
|
|
+ sql "sqlmodel"
|
|
|
"strings"
|
|
|
"time"
|
|
|
. "util"
|
|
|
@@ -41,7 +43,9 @@ func (c *CustomerRule) CuserRuleCreate() {
|
|
|
delete(data, "id")
|
|
|
delete(data, "ids")
|
|
|
if qu.IntAll(data["i_esquerytype"]) == 1 { //自动生成es
|
|
|
- data["s_esquery"] = Utiltags(data)
|
|
|
+ esStr := Utiltags(data)
|
|
|
+ data["s_esquery"] = esStr
|
|
|
+ data["s_esquery_search"] = filter(esStr)
|
|
|
}
|
|
|
i_createtime := time.Now().Unix()
|
|
|
data["i_updatetime"] = i_createtime
|
|
|
@@ -108,6 +112,47 @@ func (c *CustomerRule) CuserRuleCreate() {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+func filter(sql string) string {
|
|
|
+ if !checkSingleWord(sql) {
|
|
|
+ replace_map := checkDetail(sql)
|
|
|
+ for k, v := range replace_map {
|
|
|
+ // sql = regexp.MustCompile(k).ReplaceAllString(sql, v)
|
|
|
+ sql = strings.ReplaceAll(sql, k, v)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return sql
|
|
|
+}
|
|
|
+
|
|
|
+//匹配包含detail的组
|
|
|
+func checkDetail(sql string) (arrMap map[string]string) {
|
|
|
+ arrMap = map[string]string{}
|
|
|
+ res := reg_detail.FindAllStringSubmatch(sql, -1)
|
|
|
+ for _, v := range res {
|
|
|
+ if len(v) == 2 {
|
|
|
+ if v[1] != makeReplace(v[1]) {
|
|
|
+ arrMap[v[1]] = makeReplace(v[1])
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+//校验是否有单字
|
|
|
+func checkSingleWord(sql string) bool {
|
|
|
+ return len(reg_single.FindStringSubmatch(sql)) > 0
|
|
|
+}
|
|
|
+
|
|
|
+//计算每组字段的替换值
|
|
|
+func makeReplace(s_match string) (s_replace string) {
|
|
|
+ arr := []string{}
|
|
|
+ for _, v := range strings.Split(s_match, ",") {
|
|
|
+ if !replaceField_detail[v] {
|
|
|
+ arr = append(arr, v)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return strings.Join(arr, ",")
|
|
|
+}
|
|
|
+
|
|
|
//导入关键词
|
|
|
func (r *CustomerRule) RuleImport() {
|
|
|
defer qu.Catch()
|
|
|
@@ -152,6 +197,9 @@ func (c *CustomerRule) ProductData() {
|
|
|
}
|
|
|
// 限制multi_match 个数小于1000个
|
|
|
s_esquery := qu.ObjToString((*tag)["s_esquery"])
|
|
|
+ if IsNewSql != 0 {
|
|
|
+ s_esquery = qu.ObjToString((*tag)["s_esquery_search"])
|
|
|
+ }
|
|
|
escount := Es.Count(Index, Itype, s_esquery)
|
|
|
log.Println("escount: ", escount)
|
|
|
if escount > 5000 {
|
|
|
@@ -242,7 +290,6 @@ func (c *CustomerRule) ProductData() {
|
|
|
func (r *CustomerRule) DemoData() {
|
|
|
defer qu.Catch()
|
|
|
if r.Method() == "POST" {
|
|
|
- esquery := r.GetString("esquery")
|
|
|
id := r.GetString("id")
|
|
|
draw, _ := r.GetInteger("draw")
|
|
|
tag, _ := Mgo.FindById("entniche_rule", id, `{}`)
|
|
|
@@ -264,6 +311,13 @@ func (r *CustomerRule) DemoData() {
|
|
|
})
|
|
|
}
|
|
|
}
|
|
|
+ esquery := qu.ObjToString((*tag)["s_esquery"])
|
|
|
+ if IsNewSql != 0 {
|
|
|
+ esquery = qu.ObjToString((*tag)["s_esquery_search"])
|
|
|
+ }
|
|
|
+ startTime := time.Now().Unix() - 86400*60
|
|
|
+ endTime := time.Now().Unix()
|
|
|
+ esquery, _ = FilterTimeSql(esquery, startTime, endTime)
|
|
|
esquery = esquery[:len(esquery)-1] + `,"size":` + fmt.Sprintf("%d", 100) + `}`
|
|
|
data := Es.Get(InterimIndex, Itype, esquery)
|
|
|
count := 0
|
|
|
@@ -323,6 +377,59 @@ func (r *CustomerRule) DemoData() {
|
|
|
})
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+func FilterTimeSql(esquery string, startTime, endTime int64) (string, error) {
|
|
|
+ query := map[string]*sql.QueryObjecct{}
|
|
|
+ if json.Unmarshal([]byte(esquery), &query) == nil {
|
|
|
+ qb := query["query"]
|
|
|
+ for i := 0; i <= 2; i++ {
|
|
|
+ filter := qb.Filtered.Filter
|
|
|
+ if filter != nil && filter.Bool != nil { //有filter
|
|
|
+ index := -1 //记录range的位置
|
|
|
+ for i, m := range filter.Bool.Must {
|
|
|
+ mMap := m.(map[string]interface{})
|
|
|
+ if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range
|
|
|
+ if esRange["publishtime"] != nil {
|
|
|
+ index = i
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 生成新的es语句
|
|
|
+ tmpRange_ := bson.M{
|
|
|
+ "range": bson.M{
|
|
|
+ "publishtime": bson.M{
|
|
|
+ "lte": endTime,
|
|
|
+ "gt": startTime,
|
|
|
+ },
|
|
|
+ },
|
|
|
+ }
|
|
|
+ if index > -1 {
|
|
|
+ filter.Bool.Must[index] = tmpRange_
|
|
|
+ } else {
|
|
|
+ filter.Bool.Must = append(filter.Bool.Must, tmpRange_)
|
|
|
+ }
|
|
|
+ tmpQuery := query
|
|
|
+ strQuery, err := json.Marshal(tmpQuery)
|
|
|
+ if err == nil {
|
|
|
+ return string(strQuery), nil
|
|
|
+ } else {
|
|
|
+ log.Println("失败", err)
|
|
|
+ return esquery, err
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ qb.Filtered.Filter = &sql.Filter{}
|
|
|
+ qb.Filtered.Filter.Bool = &sql.BoolObject{}
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ log.Println(json.Unmarshal([]byte(esquery), &query), "err")
|
|
|
+ return esquery, errors.New("err")
|
|
|
+ }
|
|
|
+ return esquery, nil
|
|
|
+}
|
|
|
+
|
|
|
func (this *CustomerRule) DataTest(id string) {
|
|
|
//获取数据
|
|
|
title := this.GetString("title")
|
