jyseo/internal/service/queryStruct.go

package service

import (
	"app.yhyue.com/moapp/jybase/encrypt"
	"context"
	"fmt"
	"github.com/gogf/gf/v2/container/gvar"
	"github.com/gogf/gf/v2/frame/g"
	"github.com/gogf/gf/v2/util/gconv"
	"jyseo/internal/consts"
	"jyseo/utility"
	"strings"
)

type (
	SeoBiddingQuery struct {
		keys       string
		area       string
		city       string
		district   string
		topType    string
		subType    string
		topClass   string
		subClass   string
		industry   string
		SIGN       int
		Similarity string
		Other      map[string]interface{}
	}
	InfoList struct {
		Id          string
		Title       string
		Url         string
		Area        string
		Industry    string
		Subtype     string
		Price       string
		Site        string
		Detail      string
		Keyword     string
		FileExists  bool
		PublishTime int64
		ExpandMap   map[string]interface{}
	}

	ListResp struct {
		Total int
		List  []*InfoList
	}
)

func NewBiddingQuery() *SeoBiddingQuery {
	return &SeoBiddingQuery{}
}

// EquipArea 装备地区查询
func (query *SeoBiddingQuery) EquipArea(areaNode *AreaNode) *SeoBiddingQuery {
	if areaNode != nil {
		switch areaNode.Type {
		case 1, 3:
			query.area = areaNode.Name
		case 2:
			query.city = areaNode.Name
		case 4:
			// 区县会重名，所以带上上级名字
			if areaNode.PNode != nil {
				if areaNode.PNode.Type == 3 {
					query.area = areaNode.PNode.Name
				} else {
					query.city = areaNode.PNode.Name
				}
			}
			query.district = areaNode.Name
		}
	}
	return query
}

func (query *SeoBiddingQuery) EquipSType(sTypeNode *STypeNode) *SeoBiddingQuery {
	if sTypeNode.PNode != nil {
		query.subType = sTypeNode.Name
	} else {
		query.topType = sTypeNode.Name
	}
	return query
}

func (query *SeoBiddingQuery) EquipIndustryNode(node *IndustryNode) *SeoBiddingQuery {
	if node != nil {
		query.industry = node.Name
	}
	return query
}

func (query *SeoBiddingQuery) EquipSIGN(sign int) *SeoBiddingQuery {
	if sign > 0 {
		query.SIGN = sign
	}
	return query
}

func (query *SeoBiddingQuery) EquipKeyWord(keyword string) *SeoBiddingQuery {
	query.keys = keyword
	return query
}

// EquipEsSimilarity es查询相似度
func (query *SeoBiddingQuery) EquipEsSimilarity(similarity string) *SeoBiddingQuery {
	query.Similarity = similarity
	return query
}

func (query *SeoBiddingQuery) EquipIndustry(topClass, subClass string) *SeoBiddingQuery {
	if topClass != "" {
		query.topClass = topClass
	}
	if subClass != "" {
		query.subClass = subClass
	}
	return query
}

func (query *SeoBiddingQuery) EquipOtherMap(m map[string]interface{}) *SeoBiddingQuery {
	if m != nil && len(m) > 0 {
		query.Other = m
	}
	return query
}

// defaultDataFormat 默认格式化数据
func (query *SeoBiddingQuery) defaultDataFormat(data []map[string]interface{}) (bList []*InfoList) {
	if len(data) > 0 {
		titleReduction := map[string]bool{}
		for _, v := range data {
			bl := &InfoList{
				Id:      gconv.String(v["bid_id"]),
				Title:   gconv.String(v["title"]),
				Url:     fmt.Sprintf("/nologin/content/%s.html", encrypt.EncodeArticleId2ByCheck(gconv.String(v["bid_id"]))),
				Area:    gconv.String(v["area"]),
				Subtype: gconv.String(v["subtype"]),
				Detail:  gconv.String(v["desc"]),
				Keyword: gconv.String(v["keyword"]),
			}
			//重复标题加数字后缀
			if titleReduction[bl.Title] {
				continue
			}
			titleReduction[bl.Title] = true

			bl.PublishTime = gconv.Int64(v["publish_time"])

			if gconv.String(v["area"]) == "剑鱼信息发布平台" {
				bl.Site = "用户发布"
			}

			if industry := gconv.String(v["industry"]); industry != "" {
				bl.Industry = industry
			}
			if isValidFile, _ := v["isValidFile"].(bool); isValidFile {
				bl.FileExists = true
			}
			if budget := gconv.Float64(v["budget"]); budget > 0 {
				bl.Price = utility.ConversionMoney(v["budget"])
			} else if bidamount := gconv.Float64(v["bidamount"]); bidamount > 0 {
				bl.Price = utility.ConversionMoney(v["bidamount"])
			}
			bList = append(bList, bl)
		}
	}
	return
}

// getBidListCacheKey 获取列表缓存
func (query *SeoBiddingQuery) getDataPageListCacheKey(pageNum, pageSize, maxTotal int, flag string) string {
	return fmt.Sprintf("JySeoBidDataPageList_%s_%d_%d_%d_%s_%s_%s_%s_%s_%s_%s_%s_%s_%d", flag, pageNum, pageSize, maxTotal, query.keys, query.area, query.city, query.district, query.topType, query.subType, query.topClass, query.subClass, query.industry, query.SIGN)
}

// GetDataPageList 翻页列表页查询
// 一次性查询全部信息，生成每页cache
func (query *SeoBiddingQuery) GetDataPageList(ctx context.Context, pageNum, pageSize, maxTotal int, flag string, getData func(context.Context, int, *SeoBiddingQuery) []map[string]interface{}, formatFunc ...func([]map[string]interface{}) []*InfoList) (res *ListResp, err error) {
	var vars *gvar.Var
	res = &ListResp{}
	if pageSize == 0 {
		pageSize = consts.SettingPageSize
	}
	vars, err = g.Redis().Get(ctx, query.getDataPageListCacheKey(pageNum, pageSize, maxTotal, flag))
	if err != nil || vars.IsNil() {
		//并发限制
		if ok := JySeoQueryListLimit.Limit(); !ok {
			err = fmt.Errorf("请求超时")
			return
		}
		defer JySeoQueryListLimit.Reset()

		data := getData(ctx, maxTotal, query)

		formatExec := query.defaultDataFormat
		if len(formatFunc) > 0 && formatFunc[0] != nil {
			formatExec = formatFunc[0]
		}
		formatData := formatExec(data)

		count := len(formatData)
		if count > maxTotal {
			count = maxTotal
		}
		if data != nil && len(data) > 0 {
			totalPage := count / pageSize
			if count%pageSize != 0 {
				totalPage++
			}
			for i := 1; i <= gconv.Int(totalPage); i++ {
				start, end := (i-1)*pageSize, (i)*pageSize
				if end > count {
					end = count
				}
				pageTmp := &ListResp{
					Total: totalPage,
					List:  formatData[start:end],
				}
				if i == pageNum {
					res = pageTmp
				}
				if err := g.Redis().SetEX(ctx, query.getDataPageListCacheKey(i, pageSize, maxTotal, flag), pageTmp, consts.GetListCacheTime(ctx, flag)); err != nil {
					g.Log().Errorf(ctx, "第%d页数据 存储redis err:%v", err)
				}
			}
		}
	} else {
		err = vars.Struct(res)
	}
	return
}

// getTabDataCacheKey 获取列表缓存
func (query *SeoBiddingQuery) getOnceDataCacheKey(total int, flag string) string {
	return fmt.Sprintf("JySeoBidDataOnceList_%s_%d_%s_%s_%s_%s_%s_%s_%s_%s_%s_%d_%v", flag, total, query.keys, query.area, query.city, query.district, query.topType, query.subType, query.topClass, query.subClass, query.industry, query.SIGN, query.Other)
}

// GetOnceData 获取数据
func (query *SeoBiddingQuery) GetOnceData(ctx context.Context, total int, flag string, getData func(context.Context, int, *SeoBiddingQuery) []map[string]interface{}, formatFunc ...func([]map[string]interface{}) []*InfoList) (res []*InfoList, err error) {
	var vars *gvar.Var
	cacheKey := query.getOnceDataCacheKey(total, flag)
	vars, err = g.Redis().Get(ctx, cacheKey)
	if err != nil || vars.IsNil() {
		//并发限制
		if ok := JySeoQueryTabLimit.Limit(); !ok {
			err = fmt.Errorf("请求超时")
			return
		}
		defer JySeoQueryTabLimit.Reset()

		data := getData(ctx, total*2, query)
		if data != nil && len(data) > 0 {
			formatExec := query.defaultDataFormat
			if len(formatFunc) > 0 && formatFunc[0] != nil {
				formatExec = formatFunc[0]
			}
			res = formatExec(data)
		}
		if len(res) > total {
			res = res[:total]
		}
		fmt.Println(cacheKey, consts.GetListCacheTime(ctx, flag))
		if res != nil && len(res) > 0 {
			if err := g.Redis().SetEX(ctx, cacheKey, res, consts.GetListCacheTime(ctx, flag)); err != nil {
				g.Log().Errorf(ctx, "GetOnceData 存储redis err:%v", err)
			}
		}
	} else {
		err = vars.Struct(&res)
	}
	return
}

func FillingBiddingBaseFields(ctx context.Context, res []map[string]interface{}, filterNj ...bool) []map[string]interface{} {
	var batchSize = g.Cfg().MustGet(context.Background(), "listPageSetting.batchSize", 200).Int()
	var (
		queryIdBatch = [][]string{}
		tmpArr       = make([]string, 0, batchSize)
		total        = len(res)
		index        = 0
	)

	for _, m := range res {
		bidId := gconv.String(m["bid_id"])
		if bidId == "" {
			continue
		}
		tmpArr = append(tmpArr, bidId)
		if len(tmpArr) == batchSize {
			queryIdBatch = append(queryIdBatch, tmpArr)
			tmpArr = make([]string, 0, batchSize)
		}
		index++
		if index == total {
			queryIdBatch = append(queryIdBatch, tmpArr)
		}
	}
	var (
		bidMap  = map[string]map[string]interface{}{}
		execSql = `SELECT * FROM new_bidList WHERE bid_id IN ('%s')`
	)
	if len(filterNj) > 0 {
		execSql = `SELECT * FROM new_bidList WHERE toptype !='拟建' AND toptype !='采购意向' AND bid_id IN ('%s')`
	}
	for _, ids := range queryIdBatch {
		bidRes, _ := g.DB().Query(ctx, fmt.Sprintf(execSql, strings.Join(ids, "','")))
		if bidRes.IsEmpty() {
			continue
		}
		for _, m := range bidRes.List() {
			if bidIdStr := gconv.String(m["bid_id"]); bidIdStr != "" {
				bidMap[bidIdStr] = m
			}
		}
	}
	var finalArr []map[string]interface{}
	for i := 0; i < len(res); i++ {
		if tBid := gconv.String(res[i]["bid_id"]); tBid != "" && bidMap[tBid] != nil && len(bidMap[tBid]) > 0 {
			for k, v := range bidMap[tBid] {
				res[i][k] = v
			}
			finalArr = append(finalArr, res[i])
		}
	}
	return finalArr
}

func FillingEsBiddingBaseFields(res []map[string]interface{}, key string) []map[string]interface{} {
	bidMap := map[string]map[string]interface{}{}
	for _, m := range res {
		if bidIdStr := gconv.String(m["_id"]); bidIdStr != "" {
			mv := make(map[string]interface{})
			mv["keyword"] = key
			if len([]rune(gconv.String(m["detail"]))) > 100 {
				mv["desc"] = string([]rune(gconv.String(m["detail"]))[:100])
			} else {
				mv["desc"] = gconv.String(m["detail"])
			}
			mv["bid_id"] = bidIdStr
			mv["title"] = gconv.String(m["title"])
			mv["area"] = gconv.String(m["area"])
			mv["city"] = gconv.String(m["city"])
			mv["district"] = gconv.String(m["district"])
			mv["toptype"] = gconv.String(m["toptype"])
			mv["subtype"] = gconv.String(m["subtype"])
			mv["industry"] = ""
			if subs := gconv.String(m["s_subscopeclass"]); subs != "" {
				classArr := strings.Split(strings.Split(subs, ",")[0], "_")
				if len(classArr) == 2 {
					mv["industry"] = classArr[0]
				}
			}
			mv["bidamount"] = gconv.Float64(m["bidamount"])
			mv["budget"] = gconv.Float64(m["budget"])
			mv["size"] = gconv.String(m["size"])
			mv["isValidFile"] = gconv.Bool(m["isValidFile"])
			mv["publish_time"] = gconv.Int64(m["publishtime"])
			bidMap[bidIdStr] = mv
		}
	}

	var finalArr []map[string]interface{}
	for i := 0; i < len(res); i++ {
		if tBid := gconv.String(res[i]["_id"]); tBid != "" && bidMap[tBid] != nil && len(bidMap[tBid]) > 0 {
			for k, v := range bidMap[tBid] {
				res[i][k] = v
			}
			finalArr = append(finalArr, res[i])
		}
	}
	return finalArr
}

func GetEsQuery(maxTotal int, query *SeoBiddingQuery) string {
	var (
		musts  []string
		should []string
	)

	qu := `{
  "query": {
    "bool": {
      "filter": [
        %s
      ],
     %s
      "must_not": [
        {
          "terms": {
            "toptype": [
              "拟建",
              "采购意向"
            ]
          }
        }
      ]
    }
  },
  "_source": [
    "title",
    "publishtime",
    "_id",
    "area",
    "city",
    "district",
    "toptype",
    "subtype",
    "s_subscopeclass",
    "bidamount",
    "budget",
    "site",
    "detail",
    "isValidFile"
  ],
  "sort": {
    "publishtime": "desc"
  },
  "from": 0,
  "size": %d
}`
	if query.keys != "" {
		if query.Similarity != "" {
			musts = append(musts, fmt.Sprintf(`{"multi_match": {"query": "%s","minimum_should_match": "%s","fields": ["title"]}}`, query.keys, query.Similarity))
		} else {
			if strings.Index(query.keys, ",") > -1 {
				for _, s := range strings.Split(query.keys, ",") {
					k := strings.TrimSpace(s)
					if k != "" {
						should = append(should, fmt.Sprintf(`{"multi_match": {"query": "%s","type": "phrase","fields": ["title"]}}`, s))
					}
				}
			} else {
				should = append(should, fmt.Sprintf(`{"multi_match": {"query": "%s","type": "phrase","fields": ["title"]}}`, query.keys))
			}
		}
	}
	if query.district != "" {
		musts = append(musts, fmt.Sprintf(`{"term": {"district": "%s" }}`, query.district))
	}
	if query.city != "" {
		musts = append(musts, fmt.Sprintf(`{"term": {"city": "%s" }}`, query.city))
	}
	if query.area != "" {
		musts = append(musts, fmt.Sprintf(`{"term": {"area": "%s" }}`, query.area))
	}

	if query.topType != "" {
		if val, _ := consts.TopTypeMap[query.topType]; val != "" {
			musts = append(musts, fmt.Sprintf(`{"term": {"topType": "%s" }}`, val))
		} else {
			musts = append(musts, fmt.Sprintf(`{"term": {"topType": "%s" }}`, query.topType))
		}
	}

	if query.topClass != "" {
		musts = append(musts, fmt.Sprintf(`{"term": {"s_topscopeclass": "%s" }}`, query.topClass))
	}
	if query.subClass != "" && query.topClass != "" {
		musts = append(musts, fmt.Sprintf(`{"term": {"s_subscopeclass": "%s" }}`, fmt.Sprintf("%s_%s", query.topClass, query.subClass)))
	}
	var shouldStr string
	if len(should) > 0 {
		shouldStr = fmt.Sprintf(`"should":[%s],"minimum_should_match": 1,`, strings.Join(should, ","))
	}
	return fmt.Sprintf(qu, strings.Join(musts, ","), shouldStr, maxTotal)
}