datatag/CMPlatform/client/customerRule.go

package client

import (
	"cmplatform/util"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
	"time"

	"app.yhyue.com/moapp/jybase/common"
	"app.yhyue.com/moapp/jybase/encrypt"
	"app.yhyue.com/moapp/jybase/go-xweb/xweb"
	"app.yhyue.com/moapp/jybase/log"
	"app.yhyue.com/moapp/jybase/mongodb"
	"github.com/antonmedv/expr"
	"github.com/lauyoume/gopinyin"
	"go.mongodb.org/mongo-driver/bson"
	"go.uber.org/zap"
)

type CustomerRule struct {
	*xweb.Action
	cuserRuleCreate xweb.Mapper `xweb:"/customerRule/cuser/rule/create"`  //新建规则
	ruleImport      xweb.Mapper `xweb:"/customerRule/rule/import"`        //导入关键词  导入文件excel
	productData     xweb.Mapper `xweb:"/customerRule/cuser/produce"`      //生成预览数据
	demoData        xweb.Mapper `xweb:"/customerRule/rule/preview"`       //预览数据
	dataTest        xweb.Mapper `xweb:"/customerRule/rule/dataTest/(.*)"` //规则测试
	downloadRule    xweb.Mapper `xweb:"/customerRule/rule/downloadrule"`  //下载规则
}

var (
	replaceField_detail = map[string]bool{`"title"`: true, `"projectname.pname"`: true}
	reg_detail          = regexp.MustCompile("\"fields\":\\[([^]]*detail[^]]*)\\]")
	reg_single          = regexp.MustCompile("\"query\":\"[^\"{:]\"")
)

func (c *CustomerRule) CuserRuleCreate() {
	defer common.Catch()
	ids := strings.Split(c.GetString("ids"), ",")
	if c.Method() == "POST" {
		user := c.GetSession("user").(map[string]interface{})
		data := util.GetPostForm(c.Request)
		o_rules := []map[string]interface{}{}
		o_rulesStr := data["o_rules"].(string)
		json.Unmarshal([]byte(o_rulesStr), &o_rules)
		data["o_rules"] = o_rules
		id := common.ObjToString(data["id"])
		delete(data, "id")
		delete(data, "ids")
		if common.IntAll(data["i_esquerytype"]) == 1 { //自动生成es
			esStr := util.Utiltags(data, id)
			data["s_esquery"] = esStr
			data["s_esquery_search"] = filter(esStr)
		}
		i_createtime := time.Now().Unix()
		data["i_updatetime"] = i_createtime
		data["s_updateuser"] = user["name"]
		var rep = false
		if id == "" { //新建
			data["s_userid"] = ids[1]
			data["s_departid"] = ids[0]
			data["i_createtime"] = i_createtime
			data["s_createuser"] = user["name"]
			s_namekey := gopinyin.Convert(common.ObjToString(data["s_name"]), false)
			data["s_namekey"] = s_namekey
			data["b_delete"] = false
			data["s_dataid"] = encrypt.SE.EncodeString(fmt.Sprintf("%v", i_createtime) + s_namekey + ids[0])
			id = util.Mgo.Save("cuserdepartrule", data)
			if id != "" {
				rep = true
			} else {
				rep = false
			}
		} else {
			query := bson.M{
				"_id": mongodb.StringTOBsonId(id),
			}
			rep = util.Mgo.Update("cuserdepartrule", query, bson.M{"$set": data}, false, false)
		}

		c.ServeJson(map[string]interface{}{
			"id":        id,
			"rep":       rep,
			"s_esquery": data["s_esquery"],
			"s_dataid":  data["s_dataid"],
		})
	} else {
		c.T["did"] = ids[0] //部门id
		c.T["cid"] = ids[1] //客户id
		c.T["ids"] = c.GetString("ids")
		c.T["province"] = util.Province
		c.T["city"] = util.ProvinceCitys
		c.T["district"] = util.CityDistricts
		c.T["topTypeArr"] = util.TopTypeArr
		c.T["subTypeArr"] = util.SubTypeArr
		c.T["matchTypeMap"] = util.MatchTypeMap
		c.T["matchTypeMap2"] = util.MatchTypeMap2
		c.T["existField"] = util.ExistFiled
		c.T["buyerClass"] = util.BuyerClass
		c.T["buyerClassMap"] = util.BuyerClassMap
		c.T["scopeClass"] = util.ScopeClassMap
		c.T["siteArr"] = util.SiteArr
		/*userId := ids[1]
		cuser, _ := util.Mgo.FindById("cuser", userId, `{"s_appid":1}`)
		appid := common.ObjToString((*cuser)["s_appid"])
		user, _ := MgoCus.FindOneByField("user", map[string]interface{}{"appid": appid}, `{"plan":1}`)
		_plan := (*user)["plan"]
		if _plan != nil {
			plan := _plan.(map[string]interface{})
			fieldstype := common.ObjToString(plan["name"])
			if strings.Contains(fieldstype, "B") || strings.Contains(fieldstype, "D") {
				c.T["i_extfieldstype"] = 2
			} else {
				c.T["i_extfieldstype"] = 1
			}
		}*/
		c.Render("client/cuser_rule_create.html", &c.T)
	}
}

func filter(sql string) string {
	if !checkSingleWord(sql) {
		replace_map := checkDetail(sql)
		for k, v := range replace_map {
			// sql = regexp.MustCompile(k).ReplaceAllString(sql, v)
			sql = strings.ReplaceAll(sql, k, v)
		}
	}
	return sql
}

// 匹配包含detail的组
func checkDetail(sql string) (arrMap map[string]string) {
	arrMap = map[string]string{}
	res := reg_detail.FindAllStringSubmatch(sql, -1)
	for _, v := range res {
		if len(v) == 2 {
			if v[1] != makeReplace(v[1]) {
				arrMap[v[1]] = makeReplace(v[1])
			}
		}
	}
	return
}

// 校验是否有单字
func checkSingleWord(sql string) bool {
	return len(reg_single.FindStringSubmatch(sql)) > 0
}

// 计算每组字段的替换值
func makeReplace(s_match string) (s_replace string) {
	arr := []string{}
	for _, v := range strings.Split(s_match, ",") {
		if !replaceField_detail[v] {
			arr = append(arr, v)
		}
	}
	return strings.Join(arr, ",")
}

// 导入关键词
func (r *CustomerRule) RuleImport() {
	defer common.Catch()
	if r.Method() == "POST" {
		mf, _, err := r.GetFile("xlsx")
		if err == nil {
			binary, err := io.ReadAll(mf)
			if err == nil {
				rdata, err := util.Parsxlsx(binary)
				if err == nil {
					//id, rep := updateDbXf("", rdata)
					//common.Debug("import data:", rdata)
					r.ServeJson(map[string]interface{}{
						"rdata": rdata,
						"id":    "",
						"rep":   true,
					})
					return
				}
			}
		}
		r.ServeJson(map[string]interface{}{
			"rep": false,
		})
	}
}

// 生成预览数据
func (c *CustomerRule) ProductData() {
	defer common.Catch()
	if c.Method() == "POST" {
		rep := false
		id := c.GetString("id")
		dataType := c.GetString("dataType")
		user := c.GetSession("user").(map[string]interface{})
		userId := common.ObjToString(user["id"])
		tag, ok := util.Mgo.FindById("cuserdepartrule", id, `{}`)
		if !ok {
			c.ServeJson(map[string]interface{}{
				"rep": false,
				"msg": "规则获取失败",
			})
			return
		}
		s_esquery := common.ObjToString((*tag)["s_esquery"])
		if util.IsNewSql != 0 {
			s_esquery = common.ObjToString((*tag)["s_esquery_search"])
		}
		s_startTime := common.IntAll((*tag)["i_starttime"])
		s_endTime := common.IntAll((*tag)["i_endtime"])
		totalCount := 0
		multiMatchcount := strings.Count(s_esquery, "multi_match")
		termsCount := strings.Count(s_esquery, "terms")
		rangeCount := strings.Count(s_esquery, "range")
		if multiMatchcount != -1 {
			totalCount += multiMatchcount
		}
		if termsCount != -1 {
			totalCount += termsCount
		}
		if rangeCount != -1 {
			totalCount += rangeCount
		}

		log.Debug("multi_match", zap.Int("multiMatchcount", multiMatchcount), zap.String("s_esquery", s_esquery))
		if totalCount > 1000 && (s_startTime == 0 || s_endTime == 0) {
			c.ServeJson(map[string]interface{}{
				"rep": false,
				"msg": "查询过多,请限制开始时间及结束时间",
			})
			return
		}
		if totalCount > 8000 {
			c.ServeJson(map[string]interface{}{
				"rep": false,
				"msg": "查询过多，无法执行",
			})
			return
		}

		// 其他的分隔
		// 1000<=总个数<4000，将时间（publishtime）按照每3个月进行分割，然后查询到的数据量进行求和。
		// 4000<=总个数<8000，将时间（publishtime）按照每1个月进行分割，然后查询到的数据量进行求和
		var err error
		var count int64
		var dataTypes string
		// var n int
		// 小于1000的直接查
		// if totalCount < 1000 {
		err, dataTypes, count = util.UtilEsFind1(*tag, dataType, userId)
		// } else {
		// 	if totalCount >= 1000 && totalCount < 4000 {
		// 		n = 3
		// 	}
		// 	if totalCount >= 4000 && totalCount < 8000 {
		// 		n = 1
		// 	}
		// 	err, count = UtilEsFind2(*tag, n)
		// }
		var msg string
		if err == nil {
			rep = true
			msg = "数据生成成功"
		} else {
			rep = false
			msg = "数据生成失败，请稍后再试"
			if err.Error() == "请设置开始结束时间" {
				msg = "请设置开始结束时间"
			}
		}
		c.ServeJson(map[string]interface{}{
			"rep":      rep,
			"count":    count,
			"msg":      msg,
			"dataType": dataTypes,
		})
	}
}

func (r *CustomerRule) DemoData() {
	defer common.Catch()
	if r.Method() == "POST" {
		sDataid := r.GetString("s_dataid")
		start, _ := r.GetInt("start")
		limit, _ := r.GetInt("length")
		draw, _ := r.GetInt("draw")
		dataType := r.GetString("dataType")
		index := util.InterimIndex
		if dataType != "1" {
			index = util.TotalIndex
		}
		query := bson.M{
			"s_dataid": sDataid,
			"esIndex":  index,
		}
		data, _ := util.Mgo.Find("tagsdata", query, `{"publishtime":-1}`, nil, false, int(start), int(limit))
		count := util.Mgo.Count("tagsdata", query)
		for _, v := range *data {
			if v["budget"] != nil {
				v["budget"] = common.Float64All(fmt.Sprintf("%f", common.Float64All(v["budget"])/10000))
			}
			if v["bidamount"] != nil {
				v["bidamount"] = common.Float64All(fmt.Sprintf("%f", common.Float64All(v["bidamount"])/10000))
			}
		}
		r.ServeJson(map[string]interface{}{
			"data":            data,
			"draw":            draw,
			"recordsFiltered": count,
			"recordsTotal":    count,
		})
	}
}
func (this *CustomerRule) DataTest(id string) {
	//获取数据
	title := this.GetString("title")
	detail := this.GetString("detail")
	exactRule, exactResult := "", false
	if title == "" && detail == "" {
		this.ServeJson(map[string]interface{}{
			"status":      false,
			"data":        "",
			"message":     "标题和正文至少要有一个",
			"exactResult": exactResult,
			"exactRule":   exactRule,
		})
		return
	}
	article := map[string]interface{}{
		"title":  title,
		"detail": detail,
	}
	log.Debug("测试数据是否匹配开始...")
	//加载一个客户
	customer, _ := util.Mgo.Find("cuserdepartrule", map[string]interface{}{"_id": mongodb.StringTOBsonId(id)}, nil, nil, false, -1, -1)
	if len(*customer) == 1 {
		c := (*customer)[0]

		s_globaladdkey := common.ObjToString(c["s_globaladdkey"])
		s_globaladdkeymatch := common.ObjToString(c["s_globaladdkeymatch"])
		s_globalnotkey := common.ObjToString(c["s_globalnotkey"])
		s_globalnotkeymatch := common.ObjToString(c["s_globalnotkeymatch"])
		s_globalclearkey := common.ObjToString(c["s_globalclearkey"])
		s_globalclearkeymatch := common.ObjToString(c["s_globalclearkeymatch"])
		exactRule = common.ObjToString(c["s_exactRule"])

		//清理词匹配方式
		cwmArr := []string{}
		for _, mv := range strings.Split(s_globalclearkeymatch, ",") {
			if field := common.ObjToString(MatchType[mv]); field != "" {
				cwmArr = append(cwmArr, field)
			}
		}
		//清理词正则
		cwkArr := []*regexp.Regexp{}
		for _, kv := range strings.Split(s_globalclearkey, ",") {
			if LetterCase.MatchString(kv) { //字母转大写
				kv = strings.ToUpper(kv)
			}
			reg := regexp.MustCompile(kv)
			cwkArr = append(cwkArr, reg)
		}
		//清理清理词
		for _, cwm := range cwmArr {
			if text := common.ObjToString(article[cwm]); text != "" {
				for _, gcw_reg := range cwkArr {
					text = gcw_reg.ReplaceAllString(text, "")
				}
				article[cwm] = text
			}
		}

		o_rule, ok := c["o_rules"].([]interface{})
		if !ok {
			o_rule = []interface{}{}
		}
		o_rules := common.ObjArrToMapArr(o_rule)
		matchKey := ""

		if s_globaladdkey != "" {
			gKey := TestMactchKeys(s_globaladdkeymatch, s_globaladdkey, article)
			if gKey == "" {
				this.ServeJson(map[string]interface{}{
					"status": true,
					"data": map[string]interface{}{
						"result": false,
						"info":   "全局附加词没有匹配成功",
						"data":   s_globaladdkey,
					},
					"message":     "测试成功",
					"exactResult": exactResult,
					"exactRule":   exactRule,
				})
				return
			} else {
				matchKey = gKey
			}
		}
		if s_globalnotkey != "" {
			gNKey := TestMactchKeys(s_globalnotkeymatch, s_globalnotkey, article)
			if gNKey != "" {
				this.ServeJson(map[string]interface{}{
					"status": true,
					"data": map[string]interface{}{
						"result": false,
						"info":   "被全局排除词排除",
						"data":   gNKey,
					},
					"message":     "测试成功",
					"exactResult": exactResult,
					"exactRule":   exactRule,
				})
				return
			}
		}

		var resultList []map[string]interface{}
		for _, v := range o_rules {
			matchKey = ""
			key := common.ObjToString(v["s_matchkey"])
			keymatch := common.ObjToString(v["s_keymatch"])
			addkey := common.ObjToString(v["s_addkey"])
			addkeymatch := common.ObjToString(v["s_addkeymatch"])
			notkey := common.ObjToString(v["s_notkey"])
			notkeymatch := common.ObjToString(v["s_notkeymatch"])
			s_group := common.ObjToString(v["s_group"])

			var tempData = map[string]interface{}{
				"key":         key,
				"keymatch":    keymatch,
				"addkey":      addkey,
				"addkeymatch": addkeymatch,
				"notkey":      notkey,
				"notkeymatch": notkeymatch,
				"s_group":     s_group,
			}
			if notkey != "" {
				nKey := TestMactchKeys(notkeymatch, notkey, article)
				if nKey != "" {
					tempData["status"] = false
					tempData["info"] = "排除词排除"
					tempData["data"] = nKey
					resultList = append(resultList, tempData)
					continue
				}
			}
			if key != "" {
				sKey := TestMactchKeys(keymatch, key, article)
				aKey := ""
				if addkey != "" {
					aKey = TestMactchKeys(addkeymatch, addkey, article)
					if aKey != "" && sKey != "" {
						matchKey = aKey + "," + sKey
						tempData["status"] = true
						tempData["info"] = "匹配成功"
						tempData["data"] = matchKey
						resultList = append(resultList, tempData)

					} else if aKey == "" {
						tempData["status"] = false
						tempData["info"] = "附加词匹配失败"
						tempData["data"] = ""
						resultList = append(resultList, tempData)

					} else if sKey == "" {
						tempData["status"] = false
						tempData["info"] = "关键词匹配失败"
						tempData["data"] = ""
						resultList = append(resultList, tempData)

					}
				} else {
					if sKey != "" {
						matchKey = sKey
						tempData["status"] = true
						tempData["info"] = "匹配成功"
						tempData["data"] = matchKey
						resultList = append(resultList, tempData)

					} else {

						matchKey = sKey
						tempData["status"] = false
						tempData["info"] = "关键词匹配失败"
						tempData["data"] = matchKey
						resultList = append(resultList, tempData)

					}
				}
			}
		}
		if resultList == nil {
			this.ServeJson(map[string]interface{}{
				"status": true,
				"data": map[string]interface{}{
					"result": true,
					"info":   "全局附加词匹配成功" + matchKey,
					"data":   "",
				},
				"message":     "测试成功",
				"exactResult": exactResult,
				"exactRule":   exactRule,
			})
			return

		} else {
			nameArr := []string{}
			data, ok := util.Mgo.Find("groups", map[string]interface{}{"ruleId": id}, nil, nil, false, -1, -1)
			if ok && data != nil && len(*data) > 0 {
				for _, v := range *data {
					nameArr = append(nameArr, common.ObjToString(v["name"]))
				}
			}
			exactResult = exactMatch(exactRule, resultList, nameArr)
			this.ServeJson(map[string]interface{}{
				"status":      true,
				"data":        resultList,
				"message":     "测试成功",
				"exactResult": exactResult,
				"exactRule":   exactRule,
			})
		}
		return
	} else {
		this.ServeJson(map[string]interface{}{
			"status":      true,
			"data":        "",
			"message":     "初始化客户信息失败",
			"exactResult": exactResult,
			"exactRule":   exactRule,
		})
		return
	}
	log.Debug("测试数据是否匹配结束...")
}
func exactMatch(rule string, data []map[string]interface{}, nameArr []string) bool {
	realdata := map[string]float64{}
	for _, v := range nameArr {
		realdata["title_"+v] = 0
		realdata["content_"+v] = 0
	}

	mapping := map[string]string{
		" and ": " && ",
		" or ":  " || ",
		" not ": " ! ",
	}

	for k, v := range mapping {
		rule = strings.ReplaceAll(rule, k, v)
	}
	//可以将编译后的表达式，存放在缓存中
	program, err := expr.Compile(rule, expr.Env(realdata))
	if err != nil {
		log.Error("表达式错误 ", zap.Error(err))
		return false
	}

	for _, v := range data {
		if v["status"].(bool) {
			keymatch := common.ObjToString(v["keymatch"])
			group := common.ObjToString(v["s_group"])
			markkey := strings.Split(common.ObjToString(v["markkey"]), ",")
			if group != "" {
				if strings.Contains(keymatch, "1") {
					realdata["title_"+group] = realdata["title_"+group] + float64(len(markkey))
				}
				if strings.Contains(keymatch, "2") {
					realdata["content_"+group] = realdata["content_"+group] + float64(len(markkey))
				}
			}
		}
	}
	log.Debug("匹配结果 ", zap.Any("realdata", realdata))
	output, err := expr.Run(program, realdata)
	if err != nil {
		log.Error("表达式执行错误 ", zap.Error(err))
		return false
	}
	return output.(bool)
}

func (this *CustomerRule) DownloadRule() {
	defer common.Catch()
	id := this.GetString("id")
	// 验证一下规则
	user, ok := this.GetSession("user").(map[string]interface{})
	if !ok {
		this.ServeJson("身份异常")
		return
	}
	uID := ""
	users, ok := util.Mgo.FindOne("cuser", map[string]interface{}{"s_name": user["name"], "b_delete": false})
	if users != nil && ok {
		uID = mongodb.BsonIdToSId((*users)["_id"])
	}
	if uID == "" {
		this.ServeJson("身份异常")
		return
	}
	count := util.Mgo.Count("cuserdepartrule", map[string]interface{}{"_id": mongodb.StringTOBsonId(id), "s_userid": uID, "b_delete": false})
	if count <= 0 {
		this.ServeJson("规则异常")
		return
	}
	path := util.ResponseXlsx_Rule(id)
	if path == "" {
		this.ServeJson("没有数据")
	} else {
		arr := strings.Split(path, "/")
		this.ResponseWriter.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=%s", arr[len(arr)-1]))
		this.ServeFile(path)
		go func(path string) {
			time.Sleep(time.Minute * 3)
			os.Remove(path)
		}(path)
	}
}