package client import ( "cmplatform/util" "encoding/json" "fmt" "io" "os" "regexp" "strings" "time" "app.yhyue.com/moapp/jybase/common" "app.yhyue.com/moapp/jybase/encrypt" "app.yhyue.com/moapp/jybase/go-xweb/xweb" "app.yhyue.com/moapp/jybase/log" "app.yhyue.com/moapp/jybase/mongodb" "github.com/antonmedv/expr" "github.com/lauyoume/gopinyin" "go.mongodb.org/mongo-driver/bson" "go.uber.org/zap" ) type CustomerRule struct { *xweb.Action cuserRuleCreate xweb.Mapper `xweb:"/customerRule/cuser/rule/create"` //新建规则 ruleImport xweb.Mapper `xweb:"/customerRule/rule/import"` //导入关键词 导入文件excel productData xweb.Mapper `xweb:"/customerRule/cuser/produce"` //生成预览数据 demoData xweb.Mapper `xweb:"/customerRule/rule/preview"` //预览数据 dataTest xweb.Mapper `xweb:"/customerRule/rule/dataTest/(.*)"` //规则测试 downloadRule xweb.Mapper `xweb:"/customerRule/rule/downloadrule"` //下载规则 } var ( replaceField_detail = map[string]bool{`"title"`: true, `"projectname.pname"`: true} reg_detail = regexp.MustCompile("\"fields\":\\[([^]]*detail[^]]*)\\]") reg_single = regexp.MustCompile("\"query\":\"[^\"{:]\"") ) func (c *CustomerRule) CuserRuleCreate() { defer common.Catch() ids := strings.Split(c.GetString("ids"), ",") if c.Method() == "POST" { user := c.GetSession("user").(map[string]interface{}) data := util.GetPostForm(c.Request) o_rules := []map[string]interface{}{} o_rulesStr := data["o_rules"].(string) json.Unmarshal([]byte(o_rulesStr), &o_rules) data["o_rules"] = o_rules id := common.ObjToString(data["id"]) delete(data, "id") delete(data, "ids") if common.IntAll(data["i_esquerytype"]) == 1 { //自动生成es esStr := util.Utiltags(data, id) data["s_esquery"] = esStr data["s_esquery_search"] = filter(esStr) } i_createtime := time.Now().Unix() data["i_updatetime"] = i_createtime data["s_updateuser"] = user["name"] var rep = false if id == "" { //新建 data["s_userid"] = ids[1] data["s_departid"] = ids[0] data["i_createtime"] = i_createtime data["s_createuser"] = user["name"] s_namekey := gopinyin.Convert(common.ObjToString(data["s_name"]), false) data["s_namekey"] = s_namekey data["b_delete"] = false data["s_dataid"] = encrypt.SE.EncodeString(fmt.Sprintf("%v", i_createtime) + s_namekey + ids[0]) id = util.Mgo.Save("cuserdepartrule", data) if id != "" { rep = true } else { rep = false } } else { query := bson.M{ "_id": mongodb.StringTOBsonId(id), } rep = util.Mgo.Update("cuserdepartrule", query, bson.M{"$set": data}, false, false) } c.ServeJson(map[string]interface{}{ "id": id, "rep": rep, "s_esquery": data["s_esquery"], "s_dataid": data["s_dataid"], }) } else { c.T["did"] = ids[0] //部门id c.T["cid"] = ids[1] //客户id c.T["ids"] = c.GetString("ids") c.T["province"] = util.Province c.T["city"] = util.ProvinceCitys c.T["district"] = util.CityDistricts c.T["topTypeArr"] = util.TopTypeArr c.T["subTypeArr"] = util.SubTypeArr c.T["matchTypeMap"] = util.MatchTypeMap c.T["matchTypeMap2"] = util.MatchTypeMap2 c.T["existField"] = util.ExistFiled c.T["buyerClass"] = util.BuyerClass c.T["buyerClassMap"] = util.BuyerClassMap c.T["scopeClass"] = util.ScopeClassMap c.T["siteArr"] = util.SiteArr /*userId := ids[1] cuser, _ := util.Mgo.FindById("cuser", userId, `{"s_appid":1}`) appid := common.ObjToString((*cuser)["s_appid"]) user, _ := MgoCus.FindOneByField("user", map[string]interface{}{"appid": appid}, `{"plan":1}`) _plan := (*user)["plan"] if _plan != nil { plan := _plan.(map[string]interface{}) fieldstype := common.ObjToString(plan["name"]) if strings.Contains(fieldstype, "B") || strings.Contains(fieldstype, "D") { c.T["i_extfieldstype"] = 2 } else { c.T["i_extfieldstype"] = 1 } }*/ c.Render("client/cuser_rule_create.html", &c.T) } } func filter(sql string) string { if !checkSingleWord(sql) { replace_map := checkDetail(sql) for k, v := range replace_map { // sql = regexp.MustCompile(k).ReplaceAllString(sql, v) sql = strings.ReplaceAll(sql, k, v) } } return sql } // 匹配包含detail的组 func checkDetail(sql string) (arrMap map[string]string) { arrMap = map[string]string{} res := reg_detail.FindAllStringSubmatch(sql, -1) for _, v := range res { if len(v) == 2 { if v[1] != makeReplace(v[1]) { arrMap[v[1]] = makeReplace(v[1]) } } } return } // 校验是否有单字 func checkSingleWord(sql string) bool { return len(reg_single.FindStringSubmatch(sql)) > 0 } // 计算每组字段的替换值 func makeReplace(s_match string) (s_replace string) { arr := []string{} for _, v := range strings.Split(s_match, ",") { if !replaceField_detail[v] { arr = append(arr, v) } } return strings.Join(arr, ",") } // 导入关键词 func (r *CustomerRule) RuleImport() { defer common.Catch() if r.Method() == "POST" { mf, _, err := r.GetFile("xlsx") if err == nil { binary, err := io.ReadAll(mf) if err == nil { rdata, err := util.Parsxlsx(binary) if err == nil { //id, rep := updateDbXf("", rdata) //common.Debug("import data:", rdata) r.ServeJson(map[string]interface{}{ "rdata": rdata, "id": "", "rep": true, }) return } } } r.ServeJson(map[string]interface{}{ "rep": false, }) } } // 生成预览数据 func (c *CustomerRule) ProductData() { defer common.Catch() if c.Method() == "POST" { rep := false id := c.GetString("id") dataType := c.GetString("dataType") user := c.GetSession("user").(map[string]interface{}) userId := common.ObjToString(user["id"]) tag, ok := util.Mgo.FindById("cuserdepartrule", id, `{}`) if !ok { c.ServeJson(map[string]interface{}{ "rep": false, "msg": "规则获取失败", }) return } s_esquery := common.ObjToString((*tag)["s_esquery"]) if util.IsNewSql != 0 { s_esquery = common.ObjToString((*tag)["s_esquery_search"]) } s_startTime := common.IntAll((*tag)["i_starttime"]) s_endTime := common.IntAll((*tag)["i_endtime"]) totalCount := 0 multiMatchcount := strings.Count(s_esquery, "multi_match") termsCount := strings.Count(s_esquery, "terms") rangeCount := strings.Count(s_esquery, "range") if multiMatchcount != -1 { totalCount += multiMatchcount } if termsCount != -1 { totalCount += termsCount } if rangeCount != -1 { totalCount += rangeCount } log.Debug("multi_match", zap.Int("multiMatchcount", multiMatchcount), zap.String("s_esquery", s_esquery)) if totalCount > 1000 && (s_startTime == 0 || s_endTime == 0) { c.ServeJson(map[string]interface{}{ "rep": false, "msg": "查询过多,请限制开始时间及结束时间", }) return } if totalCount > 8000 { c.ServeJson(map[string]interface{}{ "rep": false, "msg": "查询过多,无法执行", }) return } // 其他的分隔 // 1000<=总个数<4000,将时间(publishtime)按照每3个月进行分割,然后查询到的数据量进行求和。 // 4000<=总个数<8000,将时间(publishtime)按照每1个月进行分割,然后查询到的数据量进行求和 var err error var count int64 var dataTypes string // var n int // 小于1000的直接查 // if totalCount < 1000 { err, dataTypes, count = util.UtilEsFind1(*tag, dataType, userId) // } else { // if totalCount >= 1000 && totalCount < 4000 { // n = 3 // } // if totalCount >= 4000 && totalCount < 8000 { // n = 1 // } // err, count = UtilEsFind2(*tag, n) // } var msg string if err == nil { rep = true msg = "数据生成成功" } else { rep = false msg = "数据生成失败,请稍后再试" if err.Error() == "请设置开始结束时间" { msg = "请设置开始结束时间" } } c.ServeJson(map[string]interface{}{ "rep": rep, "count": count, "msg": msg, "dataType": dataTypes, }) } } func (r *CustomerRule) DemoData() { defer common.Catch() if r.Method() == "POST" { sDataid := r.GetString("s_dataid") start, _ := r.GetInt("start") limit, _ := r.GetInt("length") draw, _ := r.GetInt("draw") dataType := r.GetString("dataType") index := util.InterimIndex if dataType != "1" { index = util.TotalIndex } query := bson.M{ "s_dataid": sDataid, "esIndex": index, } data, _ := util.Mgo.Find("tagsdata", query, `{"publishtime":-1}`, nil, false, int(start), int(limit)) count := util.Mgo.Count("tagsdata", query) for _, v := range *data { if v["budget"] != nil { v["budget"] = common.Float64All(fmt.Sprintf("%f", common.Float64All(v["budget"])/10000)) } if v["bidamount"] != nil { v["bidamount"] = common.Float64All(fmt.Sprintf("%f", common.Float64All(v["bidamount"])/10000)) } } r.ServeJson(map[string]interface{}{ "data": data, "draw": draw, "recordsFiltered": count, "recordsTotal": count, }) } } func (this *CustomerRule) DataTest(id string) { //获取数据 title := this.GetString("title") detail := this.GetString("detail") exactRule, exactResult := "", false if title == "" && detail == "" { this.ServeJson(map[string]interface{}{ "status": false, "data": "", "message": "标题和正文至少要有一个", "exactResult": exactResult, "exactRule": exactRule, }) return } article := map[string]interface{}{ "title": title, "detail": detail, } log.Debug("测试数据是否匹配开始...") //加载一个客户 customer, _ := util.Mgo.Find("cuserdepartrule", map[string]interface{}{"_id": mongodb.StringTOBsonId(id)}, nil, nil, false, -1, -1) if len(*customer) == 1 { c := (*customer)[0] s_globaladdkey := common.ObjToString(c["s_globaladdkey"]) s_globaladdkeymatch := common.ObjToString(c["s_globaladdkeymatch"]) s_globalnotkey := common.ObjToString(c["s_globalnotkey"]) s_globalnotkeymatch := common.ObjToString(c["s_globalnotkeymatch"]) s_globalclearkey := common.ObjToString(c["s_globalclearkey"]) s_globalclearkeymatch := common.ObjToString(c["s_globalclearkeymatch"]) exactRule = common.ObjToString(c["s_exactRule"]) //清理词匹配方式 cwmArr := []string{} for _, mv := range strings.Split(s_globalclearkeymatch, ",") { if field := common.ObjToString(MatchType[mv]); field != "" { cwmArr = append(cwmArr, field) } } //清理词正则 cwkArr := []*regexp.Regexp{} for _, kv := range strings.Split(s_globalclearkey, ",") { if LetterCase.MatchString(kv) { //字母转大写 kv = strings.ToUpper(kv) } reg := regexp.MustCompile(kv) cwkArr = append(cwkArr, reg) } //清理清理词 for _, cwm := range cwmArr { if text := common.ObjToString(article[cwm]); text != "" { for _, gcw_reg := range cwkArr { text = gcw_reg.ReplaceAllString(text, "") } article[cwm] = text } } o_rule, ok := c["o_rules"].([]interface{}) if !ok { o_rule = []interface{}{} } o_rules := common.ObjArrToMapArr(o_rule) matchKey := "" if s_globaladdkey != "" { gKey := TestMactchKeys(s_globaladdkeymatch, s_globaladdkey, article) if gKey == "" { this.ServeJson(map[string]interface{}{ "status": true, "data": map[string]interface{}{ "result": false, "info": "全局附加词没有匹配成功", "data": s_globaladdkey, }, "message": "测试成功", "exactResult": exactResult, "exactRule": exactRule, }) return } else { matchKey = gKey } } if s_globalnotkey != "" { gNKey := TestMactchKeys(s_globalnotkeymatch, s_globalnotkey, article) if gNKey != "" { this.ServeJson(map[string]interface{}{ "status": true, "data": map[string]interface{}{ "result": false, "info": "被全局排除词排除", "data": gNKey, }, "message": "测试成功", "exactResult": exactResult, "exactRule": exactRule, }) return } } var resultList []map[string]interface{} for _, v := range o_rules { matchKey = "" key := common.ObjToString(v["s_matchkey"]) keymatch := common.ObjToString(v["s_keymatch"]) addkey := common.ObjToString(v["s_addkey"]) addkeymatch := common.ObjToString(v["s_addkeymatch"]) notkey := common.ObjToString(v["s_notkey"]) notkeymatch := common.ObjToString(v["s_notkeymatch"]) s_group := common.ObjToString(v["s_group"]) var tempData = map[string]interface{}{ "key": key, "keymatch": keymatch, "addkey": addkey, "addkeymatch": addkeymatch, "notkey": notkey, "notkeymatch": notkeymatch, "s_group": s_group, } if notkey != "" { nKey := TestMactchKeys(notkeymatch, notkey, article) if nKey != "" { tempData["status"] = false tempData["info"] = "排除词排除" tempData["data"] = nKey resultList = append(resultList, tempData) continue } } if key != "" { sKey := TestMactchKeys(keymatch, key, article) aKey := "" if addkey != "" { aKey = TestMactchKeys(addkeymatch, addkey, article) if aKey != "" && sKey != "" { matchKey = aKey + "," + sKey tempData["status"] = true tempData["info"] = "匹配成功" tempData["data"] = matchKey resultList = append(resultList, tempData) } else if aKey == "" { tempData["status"] = false tempData["info"] = "附加词匹配失败" tempData["data"] = "" resultList = append(resultList, tempData) } else if sKey == "" { tempData["status"] = false tempData["info"] = "关键词匹配失败" tempData["data"] = "" resultList = append(resultList, tempData) } } else { if sKey != "" { matchKey = sKey tempData["status"] = true tempData["info"] = "匹配成功" tempData["data"] = matchKey resultList = append(resultList, tempData) } else { matchKey = sKey tempData["status"] = false tempData["info"] = "关键词匹配失败" tempData["data"] = matchKey resultList = append(resultList, tempData) } } } } if resultList == nil { this.ServeJson(map[string]interface{}{ "status": true, "data": map[string]interface{}{ "result": true, "info": "全局附加词匹配成功" + matchKey, "data": "", }, "message": "测试成功", "exactResult": exactResult, "exactRule": exactRule, }) return } else { nameArr := []string{} data, ok := util.Mgo.Find("groups", map[string]interface{}{"ruleId": id}, nil, nil, false, -1, -1) if ok && data != nil && len(*data) > 0 { for _, v := range *data { nameArr = append(nameArr, common.ObjToString(v["name"])) } } exactResult = exactMatch(exactRule, resultList, nameArr) this.ServeJson(map[string]interface{}{ "status": true, "data": resultList, "message": "测试成功", "exactResult": exactResult, "exactRule": exactRule, }) } return } else { this.ServeJson(map[string]interface{}{ "status": true, "data": "", "message": "初始化客户信息失败", "exactResult": exactResult, "exactRule": exactRule, }) return } log.Debug("测试数据是否匹配结束...") } func exactMatch(rule string, data []map[string]interface{}, nameArr []string) bool { realdata := map[string]float64{} for _, v := range nameArr { realdata["title_"+v] = 0 realdata["content_"+v] = 0 } mapping := map[string]string{ " and ": " && ", " or ": " || ", " not ": " ! ", } for k, v := range mapping { rule = strings.ReplaceAll(rule, k, v) } //可以将编译后的表达式,存放在缓存中 program, err := expr.Compile(rule, expr.Env(realdata)) if err != nil { log.Error("表达式错误 ", zap.Error(err)) return false } for _, v := range data { if v["status"].(bool) { keymatch := common.ObjToString(v["keymatch"]) group := common.ObjToString(v["s_group"]) markkey := strings.Split(common.ObjToString(v["markkey"]), ",") if group != "" { if strings.Contains(keymatch, "1") { realdata["title_"+group] = realdata["title_"+group] + float64(len(markkey)) } if strings.Contains(keymatch, "2") { realdata["content_"+group] = realdata["content_"+group] + float64(len(markkey)) } } } } log.Debug("匹配结果 ", zap.Any("realdata", realdata)) output, err := expr.Run(program, realdata) if err != nil { log.Error("表达式执行错误 ", zap.Error(err)) return false } return output.(bool) } func (this *CustomerRule) DownloadRule() { defer common.Catch() id := this.GetString("id") // 验证一下规则 user, ok := this.GetSession("user").(map[string]interface{}) if !ok { this.ServeJson("身份异常") return } uID := "" users, ok := util.Mgo.FindOne("cuser", map[string]interface{}{"s_name": user["name"], "b_delete": false}) if users != nil && ok { uID = mongodb.BsonIdToSId((*users)["_id"]) } if uID == "" { this.ServeJson("身份异常") return } count := util.Mgo.Count("cuserdepartrule", map[string]interface{}{"_id": mongodb.StringTOBsonId(id), "s_userid": uID, "b_delete": false}) if count <= 0 { this.ServeJson("规则异常") return } path := util.ResponseXlsx_Rule(id) if path == "" { this.ServeJson("没有数据") } else { arr := strings.Split(path, "/") this.ResponseWriter.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=%s", arr[len(arr)-1])) this.ServeFile(path) go func(path string) { time.Sleep(time.Minute * 3) os.Remove(path) }(path) } }