package client import ( "app.yhyue.com/moapp/jybase/common" "app.yhyue.com/moapp/jybase/encrypt" "app.yhyue.com/moapp/jybase/go-xweb/xweb" "app.yhyue.com/moapp/jybase/log" "app.yhyue.com/moapp/jybase/mongodb" "cmplatform/util" "encoding/json" "fmt" "github.com/lauyoume/gopinyin" "go.mongodb.org/mongo-driver/bson" "go.uber.org/zap" "io" "regexp" "strings" "time" ) type CustomerRule struct { *xweb.Action cuserRuleCreate xweb.Mapper `xweb:"/customerRule/cuser/rule/create"` //新建规则 ruleImport xweb.Mapper `xweb:"/customerRule/rule/import"` //导入关键词 导入文件excel productData xweb.Mapper `xweb:"/customerRule/cuser/produce"` //生成预览数据 demoData xweb.Mapper `xweb:"/customerRule/rule/preview"` //预览数据 dataTest xweb.Mapper `xweb:"/customerRule/rule/dataTest/(.*)"` //规则测试 } var ( replaceField_detail = map[string]bool{`"title"`: true, `"projectname.pname"`: true} reg_detail = regexp.MustCompile("\"fields\":\\[([^]]*detail[^]]*)\\]") reg_single = regexp.MustCompile("\"query\":\"[^\"{:]\"") ) func (c *CustomerRule) CuserRuleCreate() { defer common.Catch() ids := strings.Split(c.GetString("ids"), ",") if c.Method() == "POST" { user := c.GetSession("user").(map[string]interface{}) data := util.GetPostForm(c.Request) o_rules := []map[string]interface{}{} o_rulesStr := data["o_rules"].(string) json.Unmarshal([]byte(o_rulesStr), &o_rules) data["o_rules"] = o_rules id := common.ObjToString(data["id"]) delete(data, "id") delete(data, "ids") if common.IntAll(data["i_esquerytype"]) == 1 { //自动生成es esStr := util.Utiltags(data) data["s_esquery"] = esStr data["s_esquery_search"] = filter(esStr) } i_createtime := time.Now().Unix() data["i_updatetime"] = i_createtime data["s_updateuser"] = user["name"] var rep = false if id == "" { //新建 data["s_userid"] = ids[1] data["s_departid"] = ids[0] data["i_createtime"] = i_createtime data["s_createuser"] = user["name"] s_namekey := gopinyin.Convert(common.ObjToString(data["s_name"]), false) data["s_namekey"] = s_namekey data["b_delete"] = false data["s_dataid"] = encrypt.SE.EncodeString(fmt.Sprintf("%v", i_createtime) + s_namekey + ids[0]) id = util.Mgo.Save("cuserdepartrule", data) if id != "" { rep = true } else { rep = false } } else { query := bson.M{ "_id": mongodb.StringTOBsonId(id), } rep = util.Mgo.Update("cuserdepartrule", query, bson.M{"$set": data}, false, false) } c.ServeJson(map[string]interface{}{ "id": id, "rep": rep, "s_esquery": data["s_esquery"], "s_dataid": data["s_dataid"], }) } else { c.T["did"] = ids[0] //部门id c.T["cid"] = ids[1] //客户id c.T["ids"] = c.GetString("ids") c.T["province"] = util.Province c.T["city"] = util.ProvinceCitys c.T["district"] = util.CityDistricts c.T["topTypeArr"] = util.TopTypeArr c.T["subTypeArr"] = util.SubTypeArr c.T["matchTypeMap"] = util.MatchTypeMap c.T["matchTypeMap2"] = util.MatchTypeMap2 c.T["existField"] = util.ExistFiled c.T["buyerClass"] = util.BuyerClass c.T["buyerClassMap"] = util.BuyerClassMap c.T["scopeClass"] = util.ScopeClassMap c.T["siteArr"] = util.SiteArr /*userId := ids[1] cuser, _ := util.Mgo.FindById("cuser", userId, `{"s_appid":1}`) appid := common.ObjToString((*cuser)["s_appid"]) user, _ := MgoCus.FindOneByField("user", map[string]interface{}{"appid": appid}, `{"plan":1}`) _plan := (*user)["plan"] if _plan != nil { plan := _plan.(map[string]interface{}) fieldstype := common.ObjToString(plan["name"]) if strings.Contains(fieldstype, "B") || strings.Contains(fieldstype, "D") { c.T["i_extfieldstype"] = 2 } else { c.T["i_extfieldstype"] = 1 } }*/ c.Render("client/cuser_rule_create.html", &c.T) } } func filter(sql string) string { if !checkSingleWord(sql) { replace_map := checkDetail(sql) for k, v := range replace_map { // sql = regexp.MustCompile(k).ReplaceAllString(sql, v) sql = strings.ReplaceAll(sql, k, v) } } return sql } // 匹配包含detail的组 func checkDetail(sql string) (arrMap map[string]string) { arrMap = map[string]string{} res := reg_detail.FindAllStringSubmatch(sql, -1) for _, v := range res { if len(v) == 2 { if v[1] != makeReplace(v[1]) { arrMap[v[1]] = makeReplace(v[1]) } } } return } // 校验是否有单字 func checkSingleWord(sql string) bool { return len(reg_single.FindStringSubmatch(sql)) > 0 } // 计算每组字段的替换值 func makeReplace(s_match string) (s_replace string) { arr := []string{} for _, v := range strings.Split(s_match, ",") { if !replaceField_detail[v] { arr = append(arr, v) } } return strings.Join(arr, ",") } // 导入关键词 func (r *CustomerRule) RuleImport() { defer common.Catch() if r.Method() == "POST" { mf, _, err := r.GetFile("xlsx") if err == nil { binary, err := io.ReadAll(mf) if err == nil { rdata, err := util.Parsxlsx(binary) if err == nil { //id, rep := updateDbXf("", rdata) //common.Debug("import data:", rdata) r.ServeJson(map[string]interface{}{ "rdata": rdata, "id": "", "rep": true, }) return } } } r.ServeJson(map[string]interface{}{ "rep": false, }) } } // 生成预览数据 func (c *CustomerRule) ProductData() { defer common.Catch() if c.Method() == "POST" { rep := false id := c.GetString("id") dataType := c.GetString("dataType") user := c.GetSession("user").(map[string]interface{}) userId := common.ObjToString(user["id"]) tag, ok := util.Mgo.FindById("cuserdepartrule", id, `{}`) if !ok { c.ServeJson(map[string]interface{}{ "rep": false, "msg": "规则获取失败", }) return } s_esquery := common.ObjToString((*tag)["s_esquery"]) if util.IsNewSql != 0 { s_esquery = common.ObjToString((*tag)["s_esquery_search"]) } s_startTime := common.IntAll((*tag)["i_starttime"]) s_endTime := common.IntAll((*tag)["i_endtime"]) totalCount := 0 multiMatchcount := strings.Count(s_esquery, "multi_match") termsCount := strings.Count(s_esquery, "terms") rangeCount := strings.Count(s_esquery, "range") if multiMatchcount != -1 { totalCount += multiMatchcount } if termsCount != -1 { totalCount += termsCount } if rangeCount != -1 { totalCount += rangeCount } log.Debug("multi_match", zap.Int("multiMatchcount", multiMatchcount), zap.String("s_esquery", s_esquery)) if totalCount > 1000 && (s_startTime == 0 || s_endTime == 0) { c.ServeJson(map[string]interface{}{ "rep": false, "msg": "查询过多,请限制开始时间及结束时间", }) return } if totalCount > 8000 { c.ServeJson(map[string]interface{}{ "rep": false, "msg": "查询过多,无法执行", }) return } // 其他的分隔 // 1000<=总个数<4000,将时间(publishtime)按照每3个月进行分割,然后查询到的数据量进行求和。 // 4000<=总个数<8000,将时间(publishtime)按照每1个月进行分割,然后查询到的数据量进行求和 var err error var count int64 var dataTypes string // var n int // 小于1000的直接查 // if totalCount < 1000 { err, dataTypes, count = util.UtilEsFind1(*tag, dataType, userId) // } else { // if totalCount >= 1000 && totalCount < 4000 { // n = 3 // } // if totalCount >= 4000 && totalCount < 8000 { // n = 1 // } // err, count = UtilEsFind2(*tag, n) // } var msg string if err == nil { rep = true msg = "数据生成成功" } else { rep = false msg = "数据生成失败,请稍后再试" if err.Error() == "请设置开始结束时间" { msg = "请设置开始结束时间" } } c.ServeJson(map[string]interface{}{ "rep": rep, "count": count, "msg": msg, "dataType": dataTypes, }) } } func (r *CustomerRule) DemoData() { defer common.Catch() if r.Method() == "POST" { sDataid := r.GetString("s_dataid") start, _ := r.GetInt("start") limit, _ := r.GetInt("length") draw, _ := r.GetInt("draw") dataType := r.GetString("dataType") index := util.InterimIndex if dataType != "1" { index = util.TotalIndex } query := bson.M{ "s_dataid": sDataid, "esIndex": index, } data, _ := util.Mgo.Find("tagsdata", query, `{"publishtime":-1}`, nil, false, int(start), int(limit)) count := util.Mgo.Count("tagsdata", query) for _, v := range *data { if v["budget"] != nil { v["budget"] = common.Float64All(fmt.Sprintf("%f", common.Float64All(v["budget"])/10000)) } if v["bidamount"] != nil { v["bidamount"] = common.Float64All(fmt.Sprintf("%f", common.Float64All(v["bidamount"])/10000)) } } r.ServeJson(map[string]interface{}{ "data": data, "draw": draw, "recordsFiltered": count, "recordsTotal": count, }) } } func (this *CustomerRule) DataTest(id string) { //获取数据 title := this.GetString("title") detail := this.GetString("detail") fmt.Println(title, detail) if title == "" && detail == "" { this.ServeJson(map[string]interface{}{ "status": false, "data": "", "message": "标题和正文至少要有一个", }) return } article := map[string]interface{}{ "title": title, "detail": detail, } log.Debug("测试数据是否匹配开始...") //加载一个客户 customer, _ := util.Mgo.Find("cuserdepartrule", map[string]interface{}{"_id": mongodb.StringTOBsonId(id)}, nil, nil, false, -1, -1) if len(*customer) == 1 { c := (*customer)[0] s_globaladdkey := common.ObjToString(c["s_globaladdkey"]) s_globaladdkeymatch := common.ObjToString(c["s_globaladdkeymatch"]) s_globalnotkey := common.ObjToString(c["s_globalnotkey"]) s_globalnotkeymatch := common.ObjToString(c["s_globalnotkeymatch"]) s_globalclearkey := common.ObjToString(c["s_globalclearkey"]) s_globalclearkeymatch := common.ObjToString(c["s_globalclearkeymatch"]) //清理词匹配方式 cwmArr := []string{} for _, mv := range strings.Split(s_globalclearkeymatch, ",") { if field := common.ObjToString(MatchType[mv]); field != "" { cwmArr = append(cwmArr, field) } } //清理词正则 cwkArr := []*regexp.Regexp{} for _, kv := range strings.Split(s_globalclearkey, ",") { if LetterCase.MatchString(kv) { //字母转大写 kv = strings.ToUpper(kv) } reg := regexp.MustCompile(kv) cwkArr = append(cwkArr, reg) } //清理清理词 for _, cwm := range cwmArr { if text := common.ObjToString(article[cwm]); text != "" { for _, gcw_reg := range cwkArr { text = gcw_reg.ReplaceAllString(text, "") } article[cwm] = text } } o_rule, ok := c["o_rules"].([]interface{}) if !ok { o_rule = []interface{}{} } o_rules := common.ObjArrToMapArr(o_rule) matchKey := "" if s_globaladdkey != "" { gKey := TestMactchKeys(s_globaladdkeymatch, s_globaladdkey, article) if gKey == "" { this.ServeJson(map[string]interface{}{ "status": true, "data": map[string]interface{}{ "result": false, "info": "全局附加词没有匹配成功", "data": s_globaladdkey, }, "message": "测试成功", }) return } else { matchKey = gKey } } if s_globalnotkey != "" { gNKey := TestMactchKeys(s_globalnotkeymatch, s_globalnotkey, article) if gNKey != "" { this.ServeJson(map[string]interface{}{ "status": true, "data": map[string]interface{}{ "result": false, "info": "被全局排除词排除", "data": gNKey, }, "message": "测试成功", }) return } } var resultList []interface{} for _, v := range o_rules { matchKey = "" key := common.ObjToString(v["s_matchkey"]) keymatch := common.ObjToString(v["s_keymatch"]) addkey := common.ObjToString(v["s_addkey"]) addkeymatch := common.ObjToString(v["s_addkeymatch"]) notkey := common.ObjToString(v["s_notkey"]) notkeymatch := common.ObjToString(v["s_notkeymatch"]) var tempData = map[string]interface{}{ "key": key, "keymatch": keymatch, "addkey": addkey, "addkeymatch": addkeymatch, "notkey": notkey, "notkeymatch": notkeymatch, } if notkey != "" { nKey := TestMactchKeys(notkeymatch, notkey, article) if nKey != "" { tempData["status"] = false tempData["info"] = "排除词排除" tempData["data"] = nKey resultList = append(resultList, tempData) continue } } if key != "" { sKey := TestMactchKeys(keymatch, key, article) aKey := "" if addkey != "" { aKey = TestMactchKeys(addkeymatch, addkey, article) if aKey != "" && sKey != "" { matchKey = aKey + "," + sKey tempData["status"] = true tempData["info"] = "匹配成功" tempData["data"] = matchKey resultList = append(resultList, tempData) } else if aKey == "" { tempData["status"] = false tempData["info"] = "附加词匹配失败" tempData["data"] = "" resultList = append(resultList, tempData) } else if sKey == "" { tempData["status"] = false tempData["info"] = "关键词匹配失败" tempData["data"] = "" resultList = append(resultList, tempData) } } else { if sKey != "" { matchKey = sKey tempData["status"] = true tempData["info"] = "匹配成功" tempData["data"] = matchKey resultList = append(resultList, tempData) } else { matchKey = sKey tempData["status"] = false tempData["info"] = "关键词匹配失败" tempData["data"] = matchKey resultList = append(resultList, tempData) } } } } if resultList == nil { this.ServeJson(map[string]interface{}{ "status": true, "data": map[string]interface{}{ "result": true, "info": "全局附加词匹配成功" + matchKey, "data": "", }, "message": "测试成功", }) return } else { this.ServeJson(map[string]interface{}{ "status": true, "data": resultList, "message": "测试成功", }) } return } else { this.ServeJson(map[string]interface{}{ "status": true, "data": "", "message": "初始化客户信息失败", }) return } log.Debug("测试数据是否匹配结束...") }