jyMicroservices/jyBXCore/rpc/util/search.go

package util

import (
	MC "app.yhyue.com/moapp/jybase/common"
	ME "app.yhyue.com/moapp/jybase/encrypt"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"github.com/zeromicro/go-zero/core/logx"
	"io/ioutil"
	IC "jyBXCore/rpc/init"
	"jyBXCore/rpc/internal/config"
	"jyBXCore/rpc/type/bxcore"
	"math/big"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode"
)

var (
	ClearHtml  = regexp.MustCompile("<[^>]*>")
	MatchSpace = regexp.MustCompile("\\s+")
	filterReg3 = regexp.MustCompile("(项目|公告|公示)$")
	filterReg2 = regexp.MustCompile("^[）\\)>》】\\]}｝〕,，;；:：'\"“”。.\\?？、/+=\\_—*&……\\^%$￥@!！`~·（\\(<《【\\[{｛〔]+$")
	filterReg1 = regexp.MustCompile("^([0-9]{1,3}|[零一二三四五六七八九十]{1,2}|联系人?|电话|地址|编号|采购|政府采购|成交|更正|招标|中标|变更|结果)$")
	filterReg  = regexp.MustCompile("^[的人号时元万公告项目地址电话邮编日期联系招标中结果成交项目项目采购采购项目政府采购公告更正公告]+$")
	//PhoneReg     = regexp.MustCompile("^[1][3-9][0-9]{9}$")
	//EmailPattern = regexp.MustCompile("^([a-zA-Z0-9_\\-\\.]+)@((\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.)|(([a-zA-Z0-9\\-]+\\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\\]?)$")
)

// SearchHistory 格式化 关键词搜索历史记录
func SearchHistory(history, searchValue, additionalWords string) (arrS []string) {
	//主关键词
	var searchKeys = strings.Split(searchValue, IC.C.JYKeyMark)
	//附加词
	if additionalWords != "" {
		for _, aws := range strings.Split(additionalWords, ",") {
			for _, as := range strings.Split(aws, IC.C.JYKeyMark) {
				searchKeys = append(searchKeys, as)
			}
		}
	}
	//关键词 和 附加词 合并，作为新的关键词历史搜索记录
	if len(searchKeys) > 0 {
		arrS = strings.Split(history, ",")
		//新增历史记录
		if history == "" {
			arrS = make([]string, 0)
		}
		for _, sv := range searchKeys {
			for k, v := range arrS {
				if v == strings.TrimSpace(sv) {
					arrS = append(arrS[:k], arrS[k+1:]...)
					break
				}
			}
		}
		arrS = append(arrS, searchKeys...)
		if len(arrS) > 10 {
			arrS = arrS[len(arrS)-10:]
		}
	}
	return arrS
}

func FilterKey(k string) string {
	k = strings.TrimSpace(k)
	k = filterReg3.ReplaceAllString(k, "")
	k = filterReg2.ReplaceAllString(k, "")
	k = filterReg1.ReplaceAllString(k, "")
	k = filterReg.ReplaceAllString(k, "")
	return k
}

// InterceptSearchKW 超过keywordsLimit个字，截断
// 返回截取后的字符串和截取掉中的前3个字
// b_word:截取后的关键词；a_word：截取后 后面三个字；s_word：已截取 处理过的关键词
func InterceptSearchKW(word string, keywordsLimit int, isFilter bool) (bWord, aWord, sWord string) {
	if isFilter {
		word = FilterKey(word)
	}
	word = MatchSpace.ReplaceAllString(strings.TrimSpace(word), " ")
	words := []rune(word)
	if len(words) > keywordsLimit {
		bWord = string(words[:keywordsLimit])
		bWord = strings.TrimSpace(bWord)
		if len(words) > keywordsLimit+3 {
			aWord = string(words[keywordsLimit : keywordsLimit+3])
		} else {
			aWord = string(words[keywordsLimit:])
		}
	} else {
		bWord = word
	}
	aWord = strings.TrimSpace(aWord)
	sWord = MatchSpace.ReplaceAllString(bWord, IC.C.JYKeyMark)
	return
}

func HttpEs(ques, analyzer, esAddress string) (res string) {
	var adders []string
	curl := ""
	for _, s := range strings.Split(esAddress, ",") {
		adders = append(adders, s)
	}
	i, _ := rand.Int(rand.Reader, big.NewInt(int64(len(adders)))) //随机
	curl = adders[int(i.Int64())] + "/bidding/_analyze"
	URL, _ := url.Parse(curl)
	Q := URL.Query()
	Q.Add("text", ques)
	Q.Add("analyzer", analyzer)
	URL.RawQuery = Q.Encode()
	resp, err := http.Get(URL.String())
	if err != nil {
		logx.Info("es连接失败 err1:", err)
		resp, err = getESResp(ques, analyzer, adders)
		if err != nil {
			return
		}
	}
	result, err1 := ioutil.ReadAll(resp.Body)
	if err1 == nil {
		defer resp.Body.Close()
		var resMap map[string]interface{}
		json.Unmarshal(result, &resMap)
		if resMap != nil && resMap["tokens"] != nil {
			tokens := MC.ObjArrToMapArr(resMap["tokens"].([]interface{}))
			for _, v := range tokens {
				token := MC.ObjToString(v["token"])
				if len([]rune(token)) == 1 && !unicode.Is(unicode.Scripts["Han"], []rune(token)[0]) { //(P260保留单个汉字)
					continue
				}
				if res != "" && token != "" {
					res += IC.C.JYKeyMark
				}
				res += token
			}
		}
	}
	return
}

func getESResp(ques, analyzer string, adds []string) (resp *http.Response, err error) {
	for _, v := range adds {
		curl := v + "/bidding/_analyze"
		URL, _ := url.Parse(curl)
		Q := URL.Query()
		Q.Add("text", ques)
		Q.Add("analyzer", analyzer)
		URL.RawQuery = Q.Encode()
		resp, err = http.Get(URL.String())
		if err == nil {
			break
		}
	}
	return resp, err
}

const (
	RedisName      = "other"
	RedisNameNew   = "newother"
	SearchPageSize = 50 //招标搜索分页--每页显示数量
	//招标搜索分页--最大页数
	SearchMaxPageNum       = 10  //免费用户500条记录
	SearchMaxPageNum_PAYED = 100 //付费用户5000条记录

)

// MakeCollection 是否收藏
func MakeCollection(userId string, list []*bxcore.SearchList) {
	if userId == "" {
		return
	}
	if list == nil || len(list) == 0 {
		return
	}
	param := []interface{}{userId}
	var wh []string
	for _, v := range list {
		logx.Info(v.Title, "---v.id---:", v.Id)
		array := ME.DecodeArticleId2ByCheck(v.Id)
		if len(array) == 1 && array[0] != "" {
			param = append(param, array[0])
			wh = append(wh, "?")
		}
	}
	if len(wh) > 0 {
		result := IC.MainMysql.SelectBySql(`select bid from bdcollection where userid=? and bid in (`+strings.Join(wh, ",")+`)`, param...)
		bidMap := map[string]bool{}
		if result != nil {
			for _, v := range *result {
				bidMap[ME.EncodeArticleId2ByCheck(MC.ObjToString(v["bid"]))] = true
			}
		}
		for _, v := range list {
			if bidMap[v.Id] {
				v.IsCollected = true
			}
		}
	}
}

// IndustryFormat 行业处理
func IndustryFormat(industry, subScopeClass string) (newIndustry string) {
	commonSubstring := func(v string) (value string) {
		bcs := strings.Split(v, "_")
		if len(bcs) == 1 {
			value = bcs[0]
		} else if len(bcs) == 2 {
			value = bcs[0]
			if strings.TrimSpace(value) == "" {
				value = bcs[0]
			}
		}
		return
	}
	bct := strings.Split(subScopeClass, ",")
	if bct == nil || len(bct) == 0 {
		return
	}
	//搜索条件中没有行业的话，取查询结果中第一个行业
	if industry == "" {
		newIndustry = commonSubstring(bct[0])
	} else { //搜索条件中有行业的话，取行业中和搜索条件相对应的第一个
		industryArr := strings.Split(industry, ",")
	L:
		for _, bc := range bct {
			for _, is := range industryArr {
				if bc == is {
					newIndustry = strings.TrimSpace(commonSubstring(bc))
					break L
				}
			}
		}
	}
	return
}

// SearchListFormat  格式化数据
func SearchListFormat(userid, industry string, repl *[]map[string]interface{}, b bool) (list []*bxcore.SearchList) {
	for _, v := range *repl {
		var searchList = &bxcore.SearchList{}
		//正文
		if b {
			//正文匹配检索关键词
			highlight, _ := v["highlight"].(map[string][]string)
			detail := ""
			for _, val := range highlight["detail"] {
				detail += ClearHtml.ReplaceAllString(val, "")
			}
			searchList.Detail = detail
		}
		searchList.Id = ME.EncodeArticleId2ByCheck(MC.ObjToString(v["_id"]))                                    //ME.EncodeArticleId2ByCheck(MC.ObjToString(v["_id"]))                                         //加密信息id
		searchList.Area = MC.ObjToString(v["area"])                                                             //地区
		searchList.AreaUrl = IC.LabelMap[searchList.Area].Url                                                   //地区分类链接
		searchList.BuyerClass = MC.ObjToString(v["buyerclass"])                                                 //采购单位类型
		searchList.City = MC.ObjToString(v["city"])                                                             //城市
		searchList.Industry = IndustryFormat(industry, strings.Trim(MC.ObjToString(v["s_subscopeclass"]), ",")) //行业
		searchList.IndustryUrl = IC.LabelMap[searchList.Industry].Url                                           //行业分类地址
		searchList.PublishTime = MC.Int64All(v["publishtime"])                                                  //发布时间
		searchList.FileExists, _ = v["isValidFile"].(bool)                                                      //是否有附件
		searchList.Subtype = MC.ObjToString(v["subtype"])                                                       //信息类型
		searchList.SubtypeUrl = IC.LabelMap[searchList.Subtype].Url                                             //信息类型分类链接
		searchList.Title = MC.ObjToString(v["title"])                                                           //标题
		searchList.ProjectName = MC.ObjToString(v["projectname"])                                               //项目名称
		searchList.ProjectCode = MC.ObjToString(v["projectcode"])                                               //项目代码
		if budget, ok := v["budget"].(float64); ok && budget > 0 {                                              //预算
			searchList.Budget = int64(budget)
		}
		if bidAmount, ok := v["bidamount"].(float64); ok && bidAmount > 0 { //中标金额
			searchList.BidAmount = int64(bidAmount)
		}
		searchList.Buyer = MC.ObjToString(v["buyer"])               //采购单位
		searchList.BuyerTel = MC.ObjToString(v["buyertel"])         //采购单位联系方式
		searchList.BuyerPerson = MC.ObjToString(v["buyerperson"])   //采购单位联系人
		searchList.Agency = MC.ObjToString(v["agency"])             //代理机构
		searchList.AgencyTel = MC.ObjToString(v["agencytel"])       //代理机构联系电话
		searchList.AgencyPerson = MC.ObjToString(v["agencyperson"]) //代理机构联系人
		searchList.BidOpenTime = MC.Int64All(v["bidopentime"])      //开标时间
		searchList.BidEndTime = MC.Int64All(v["bidendtime"])        //发布时间
		searchList.SignEndTime = MC.Int64All(v["signendtime"])      //投标截止日期
		searchList.Site = MC.ObjToString(v["site"])                 //网站来源名称
		searchList.SpiderCode = MC.ObjToString(v["spidercode"])     //网站来源代码
		searchList.Winner = MC.ObjToString(v["winner"])             //中标企业
		winnerList := MC.ObjToString(v["s_winner"])                 //中标企业名称集合
		if winnerList != "" && len(strings.Split(winnerList, ",")) > 0 {
			for wk, wv := range strings.Split(winnerList, ",") {
				var (
					winnerId = ""
				)
				if v["entidlist"] != nil {
					if entIdList := MC.ObjArrToStringArr(v["entidlist"].([]interface{})); len(entIdList) > wk { //中标企业id集合
						winnerId = entIdList[wk]
					}
				}
				searchList.WinnerInfo = append(searchList.WinnerInfo, &bxcore.WinnerInfo{
					Winner:       wv,                                                                                                     //中标企业 需要单独处理
					WinnerTel:    MC.ObjToString(v["winnertel"]),                                                                         //中标企业联系电话
					WinnerPerson: MC.ObjToString(v["winnerperson"]),                                                                      //中标企业联系人
					WinnerId:     MC.If(winnerId != "" && len([]rune(winnerId)) > 12, ME.EncodeArticleId2ByCheck(winnerId), "").(string), //中标企业加密id  存在winnerId 异常的情况
				})
			}
		}
		searchList.ProjectInfo = &bxcore.PInfo{} //拟建项目信息
		if v["projectinfo"] != nil {
			pInfo := MC.ObjToMap(v["projectinfo"])
			searchList.ProjectInfo.ApproveCode = MC.ObjToString((*pInfo)["approvecode"])
			searchList.ProjectInfo.ApproveContent = MC.ObjToString((*pInfo)["approvecontent"])
			searchList.ProjectInfo.ApproveDept = MC.ObjToString((*pInfo)["approvedept"])
			searchList.ProjectInfo.ApproveStatus = MC.ObjToString((*pInfo)["approvestatus"])
			searchList.ProjectInfo.ProjectType = MC.ObjToString((*pInfo)["projecttype"])
			searchList.ProjectInfo.ApproveNumber = MC.ObjToString((*pInfo)["approvenumber"])
			searchList.ProjectInfo.ApproveTime = MC.ObjToString((*pInfo)["approvetime"])
		}
		list = append(list, searchList)
	}
	return
}

// IsOptimize 付费用户搜索优化
// 需求来源：付费用户 默认查询 五年内数据，数据查询耗时，
// 付费用户 且开关打开，针对前两页数据，满足关键词（< 7个字），查询时间范围一年以上，缩短查询时间
func IsOptimize(cc config.Config, in *bxcore.SearchReq) bool {
	if cc.PaySearchLimit.Switch && in.UserType != "fType" {
		//首页----字数（<7）
		if in.PageNum <= cc.PaySearchLimit.PageNum && len([]rune(in.KeyWords)) < cc.PaySearchLimit.WordSize {
			//时间超过一年----
			if pTime := GetPublishTime(cc.PaySearchLimit.Year, cc.PaySearchLimit.Month, in.PublishTime); pTime != "" {
				in.PublishTime = pTime
				return true
			}
		}
	}
	return false
}

// GetPublishTime 查询时间调整
func GetPublishTime(y, m int, publishTime string) string {
	//发布时间
	timeArray := strings.Split(publishTime, "-")
	if len(timeArray) == 2 {
		startTime, err1 := strconv.ParseInt(timeArray[0], 10, 64)
		endTime, err2 := strconv.ParseInt(timeArray[1], 10, 64)
		if err1 == nil && err2 == nil {
			if endTime == 0 {
				endTime = time.Now().Unix()
			}
			//重新计算数据查询 开始时间
			pTime := time.Unix(endTime, 0).AddDate(y, m, 0).Unix()
			//从新定义搜索时间跨度
			if endTime-startTime > pTime {
				return fmt.Sprintf("%d-%d", pTime, endTime)
			}
		}
	}
	return ""
}

// GetQueryItems 免费 标题(title)  正文(content) 老用户【中标企业(winner)】
// 付费用户 全部(all)、标题(title)  正文(content)  会员: 采购单位(buyer) 中标企业(winner) 招标代理机构(agency) 附件(file)
// 项目名称projectname和标的物purchasing(ppa)
func GetQueryItems(selectType string, limitOldTime, registerData int64, isPay bool) (items []string) {
	if isPay {
		for _, t := range strings.Split(selectType, ",") {
			if t == "content" {
				items = append(items, "detail")
			} else if t == "buyer" {
				items = append(items, "mbuyer")
			} else if t == "winner" {
				items = append(items, "mwinner")
			} else if t == "agency" {
				items = append(items, "magency")
			} else if t == "title" {
				items = append(items, "title")
			} else if t == "ppa" {
				items = append(items, []string{"purchasing", "projectname.pname"}...)
			} else if t == "file" { //dev4.7.8 标讯优化：搜索范围附件-》全部用户可用
				items = append(items, "filetext")
			}
		}
		return
	}
	//老用户 使用付费功能
	isOldUser := registerData != 0 && registerData < limitOldTime
	for _, t := range strings.Split(selectType, ",") {
		if t == "winner" && isOldUser {
			items = append(items, "mwinner")
		} else if t == "title" {
			items = append(items, "title")
		} else if t == "content" {
			items = append(items, "detail")
		} else if t == "file" { //dev4.7.8 标讯优化：搜索范围附件-》全部用户可用
			items = append(items, "filetext")
		}
	}
	return
}