jyMicroservices/jyBXCore/rpc/util/search.go

package util

import (
	MC "app.yhyue.com/moapp/jybase/common"
	ME "app.yhyue.com/moapp/jybase/encrypt"
	elastic "app.yhyue.com/moapp/jybase/esv1"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"github.com/zeromicro/go-zero/core/logx"
	"io/ioutil"
	IC "jyBXCore/rpc/init"
	"jyBXCore/rpc/type/bxcore"
	"math/big"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode"
)

var ClearHtml = regexp.MustCompile("<[^>]*>")
var MatchSpace = regexp.MustCompile("\\s+")
var filterReg_3 = regexp.MustCompile("(项目|公告|公示)$")
var filterReg_2 = regexp.MustCompile("^[）\\)>》】\\]}｝〕,，;；:：'\"“”。.\\?？、/+=\\_—*&……\\^%$￥@!！`~·（\\(<《【\\[{｛〔]+$")
var filterReg_1 = regexp.MustCompile("^([0-9]{1,3}|[零一二三四五六七八九十]{1,2}|联系人?|电话|地址|编号|采购|政府采购|成交|更正|招标|中标|变更|结果)$")
var filterReg = regexp.MustCompile("^[的人号时元万公告项目地址电话邮编日期联系招标中结果成交项目项目采购采购项目政府采购公告更正公告]+$")
var PhoneReg = regexp.MustCompile("^[1][3-9][0-9]{9}$")
var EmailPattern = regexp.MustCompile("^([a-zA-Z0-9_\\-\\.]+)@((\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.)|(([a-zA-Z0-9\\-]+\\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\\]?)$")

func SearchHistory(history, searchvalue string) (arrs []string) {
	if searchvalue != "" {
		arrs = strings.Split(history, ",")
		//新增历史记录
		if history == "" {
			arrs = make([]string, 0)
		}
		for k, v := range arrs {
			if v == strings.TrimSpace(searchvalue) {
				arrs = append(arrs[:k], arrs[k+1:]...)
				break
			}
		}
		arrs = append(arrs, searchvalue)
		if len(arrs) > 10 {
			arrs = arrs[1:11]
		}
	}
	return arrs
}

func FilteKey(k string) string {
	k = strings.TrimSpace(k)
	k = filterReg_3.ReplaceAllString(k, "")
	k = filterReg_2.ReplaceAllString(k, "")
	k = filterReg_1.ReplaceAllString(k, "")
	k = filterReg.ReplaceAllString(k, "")
	return k
}

//超过20个字，截断
//返回截取后的字符串和截取掉中的前3个字
func InterceptSearchKW(word string, isIntercept, isFilter bool) (b_word, a_word, s_word string) {
	if isFilter {
		word = FilteKey(word)
	}
	word = MatchSpace.ReplaceAllString(strings.TrimSpace(word), " ")
	words := []rune(word)
	if len(words) > 20 && isIntercept {
		b_word = string(words[:20])
		b_word = strings.TrimSpace(b_word)
		if len(words) > 23 {
			a_word = string(words[20:23])
		} else {
			a_word = string(words[20:])
		}
	} else {
		b_word = word
	}
	a_word = strings.TrimSpace(a_word)
	s_word = MatchSpace.ReplaceAllString(b_word, "+")
	return
}

func HttpEs(ques, analyzer, esAddress string) (res string) {
	var addrs []string
	surl := ""
	for _, s := range strings.Split(esAddress, ",") {
		addrs = append(addrs, s)
	}
	i, _ := rand.Int(rand.Reader, big.NewInt(int64(len(addrs)))) //随机
	surl = addrs[int(i.Int64())] + "/bidding/_analyze"
	URL, _ := url.Parse(surl)
	Q := URL.Query()
	Q.Add("text", ques)
	Q.Add("analyzer", analyzer)
	URL.RawQuery = Q.Encode()
	resp, err := http.Get(URL.String())
	if err != nil {
		logx.Info("es连接失败 err1:", err)
		resp, err = getesResp(ques, analyzer, addrs)
		if err != nil {
			return
		}
	}
	result, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		defer resp.Body.Close()
		var resmap map[string]interface{}
		json.Unmarshal(result, &resmap)
		if resmap != nil && resmap["tokens"] != nil {
			tokens := MC.ObjArrToMapArr(resmap["tokens"].([]interface{}))
			for _, v := range tokens {
				token := MC.ObjToString(v["token"])
				if len([]rune(token)) == 1 && unicode.IsLetter([]rune(token)[0]) {
					continue
				}
				if res != "" {
					res += "+"
				}
				res += token
			}
		}
	}
	return
}

//
func getesResp(ques, analyzer string, addrs []string) (resp *http.Response, err error) {
	for _, v := range addrs {
		surl := v + "/bidding/_analyze"
		URL, _ := url.Parse(surl)
		Q := URL.Query()
		Q.Add("text", ques)
		Q.Add("analyzer", analyzer)
		URL.RawQuery = Q.Encode()
		resp, err = http.Get(URL.String())
		if err == nil {
			break
		}
	}
	return resp, err
}

//pc、微信、app 招标信息搜索

const (
	INDEX          = "bidding"
	TYPE           = "bidding"
	bidSearch_sort = `{"publishtime":-1}`
	RedisName      = "other"
	//招标搜索分页--每页显示数量
	SearchPageSize = 50
	//招标搜索分页--最大页数
	SearchMaxPageNum       = 10  //免费用户500条记录
	SearchMaxPageNum_PAYED = 100 //付费用户5000条记录

	bidSearch_field_1    = `"_id","title","publishtime","toptype","subtype","type","area","city","s_subscopeclass","bidamount","budget","buyerclass","spidercode","site"` //,"filetext"
	bidSearch_field      = bidSearch_field_1 + `,"bidopentime","winner","buyer","projectname","projectcode","projectinfo"`
	bidSearch_field_file = `,"filetext"`
	query_bool_should    = `{"bool":{"should":[%s],"minimum_should_match": 1}}`
)

//GetBidSearchData 标信息搜索
func GetBidSearchData(in *bxcore.SearchReq) (count int64, list []*bxcore.SearchList) {
	t := time.Now()
	var hightlightContent bool = false //是否高亮正文
	var selectTypeArr = strings.Split(in.SelectType, ",")
	for _, v := range selectTypeArr {
		if v == "detail" {
			hightlightContent = true
			break
		}
	}
	qstr := GetSearchQuery(in, GetBidSearchQuery(in))
	var start = int((in.PageNum - 1) * in.PageSize)
	//首页
	if qstr != "" { //&& start == 0
		count = elastic.Count(INDEX, TYPE, qstr)
	}
	if count > 0 || start > 1 {
		field := bidSearch_field_1
		if start == 0 {
			field = bidSearch_field
		}
		if IC.C.FileSignBool {
			field = field + bidSearch_field_file
		}
		var repl *[]map[string]interface{}
		if hightlightContent {
			repl = elastic.GetAllByNgram(INDEX, TYPE, qstr, `"detail"`, bidSearch_sort, field, start, int(in.PageSize), 115, true)
		} else {
			repl = elastic.GetAllByNgram(INDEX, TYPE, qstr, ``, bidSearch_sort, field, start, int(in.PageSize), 0, false)
		}
		if repl != nil && *repl != nil && len(*repl) > 0 {
			BidListConvert(in.Industry, repl)
			list = searchListFormart(repl, true)
		}
		logx.Info("关键词 -1- 查询耗时：", time.Since(t).Seconds())
		MakeCollection(in.UserId, list)
	}
	logx.Info("关键词 查询耗时：", time.Since(t).Seconds())
	return
}

//
var topTypeMap = map[string]string{
	"招标预告":   "预告",
	"招标公告":   "招标",
	"招标结果":   "结果",
	"招标信用信息": "其它",
	"拟建项目":   "拟建",
	"采购意向":   "采购意向",
}

//
func GetBidSearchQuery(in *bxcore.SearchReq) string {
	query := ``
	//省份
	area := in.Province
	if area != "" {
		query += `{"terms":{"area":[`
		for k, v := range strings.Split(area, ",") {
			if k > 0 {
				query += `,`
			}
			query += `"` + v + `"`
		}
		query += `]}}`
	}
	//
	city := in.City
	if city != "" {
		if len(query) > 0 {
			query += ","
		}
		query += `{"terms":{"city":[`
		for k, v := range strings.Split(city, ",") {
			if k > 0 {
				query += `,`
			}
			query += `"` + v + `"`
		}
		query += `]}}`
	}
	if query != "" {
		query = fmt.Sprintf(query_bool_should, query)
	}
	//发布时间
	publishtime := in.PublishTime
	if publishtime != "" && len(strings.Split(publishtime, "-")) > 1 {
		if len(query) > 0 {
			query += ","
		}
		starttime := strings.Split(publishtime, "-")[0]
		endtime := strings.Split(publishtime, "-")[1]
		query += `{"range":{"publishtime":{`
		if starttime != "" {
			query += `"gte":` + starttime
		}
		if starttime != "" && endtime != "" {
			query += `,`
		}
		if endtime != "" {
			query += `"lt":` + endtime
		}
		query += `}}}`
	}
	//信息类型-二级
	subtype := in.Subtype
	toptype := MC.If(in.Toptype != "", strings.Split(in.Toptype, ","), []string{}).([]string)
	alltype := ``
	//二级分类
	if subtype != "" {
		var typeInt = 0
		alltype += `{"terms":{"subtype":[`
		for k, v := range strings.Split(subtype, ",") {
			if ttype := MC.If(topTypeMap[v] != "" && in.Toptype == "", topTypeMap[v], "").(string); ttype != "" {
				toptype = append(toptype, ttype)
				typeInt += 1
				continue
			}
			if k > typeInt {
				alltype += `,`
			}
			alltype += `"` + v + `"`
		}
		alltype += `]}}`
	}
	//信息类型  一级分类
	logx.Info("toptype:", toptype)
	if len(toptype) > 0 {
		if alltype != "" {
			alltype += ","
		}
		alltype += `{"terms":{"toptype":[`
		for k, v := range toptype {
			if k > 0 {
				alltype += `,`
			}
			alltype += `"` + v + `"`
		}
		alltype += `]}}`
	}

	if alltype != "" {
		if query != "" {
			query += ","
		}
		query += fmt.Sprintf(query_bool_should, alltype)
	}
	//采购单位类型
	buyerclass := in.BuyerClass
	if buyerclass != "" {
		if len(query) > 0 {
			query += ","
		}
		query += `{"terms":{"buyerclass":[`
		for k, v := range strings.Split(buyerclass, ",") {
			if k > 0 {
				query += `,`
			}
			query += `"` + v + `"`
		}
		query += `]}}`
	}
	return query
}

func GetSearchQuery(in *bxcore.SearchReq, mustquery string) (qstr string) {
	multi_match := `{"multi_match": {"query": "%s","type": "phrase", "fields": [%s]}}`
	query := `{"query":{"bool":{"must":[%s],"must_not":[%s]}}}`
	query_bool_should := `{"bool":{"should":[%s],"minimum_should_match": 1}}`
	query_bools_must := `{"bool":{"must":[{"range":{"bidamount":{%s}}}]}},{"bool":{"must":[{"range":{"budget":{%s}}}],"must_not":[{"range":{"bidamount":{"gte":-1}}}]}}`
	query_bool_must := `{"bool":{"must":[{"terms":{"s_subscopeclass":[%s]}}]}}`
	query_missing := `{"constant_score":{"filter":{"missing":{"field":"%s"}}}}`
	gte := `"gte": %s`
	lte := `"lte": %s`
	musts, must_not := []string{}, []string{}
	if mustquery != "" {
		musts = append(musts, mustquery)
	}
	//搜索范围是否只有附件
	//搜索范围只选择附件，是否有附件条件无效；
	var isFileSearch bool = in.SelectType == "filetext"
	selectTypeArr := strings.Split(in.SelectType, ",")
	var findfields string
	if selectTypeArr == nil || len(selectTypeArr) == 0 {
		findfields = `"title"`
	} else {
		findfields = fmt.Sprintf(`"%s"`, strings.Join(selectTypeArr, "\",\""))
	}
	//此时关键词中间有+进行隔离
	if in.KeyWords != "" {
		shoulds := []string{}
		var switchBool = strings.Contains(findfields, "detail") && strings.Contains(findfields, "title") && IC.C.SearchTypeSwitch
		for _, v := range strings.Split(in.KeyWords, "+") {
			//标题 全文搜索 搜索类型开关打开 默认搜索全文；（全文包含标题）(单字排除)
			if switchBool && len([]rune(elastic.ReplaceYH(v))) > 1 {
				if strings.Contains(findfields, `"title",`) {
					findfields = strings.Replace(findfields, `"title",`, ``, -1)
				} else if strings.Contains(findfields, `,"title"`) {
					findfields = strings.Replace(findfields, `,"title"`, ``, -1)
				}
			}
			keyword_multi_match := fmt.Sprintf(multi_match, "%s", findfields)
			shoulds = append(shoulds, fmt.Sprintf(keyword_multi_match, elastic.ReplaceYH(v)))
		}
		musts = append(musts, fmt.Sprintf(elastic.NgramMust, strings.Join(shoulds, ",")))
	}
	if in.Industry != "" {
		industrys := strings.Split(in.Industry, ",")
		musts = append(musts, fmt.Sprintf(query_bool_must, `"`+strings.Join(industrys, `","`)+`"`))
	}
	//价格
	if in.Price != "" && len(strings.Split(in.Price, "-")) > 1 {
		minprice, maxprice := strings.Split(in.Price, "-")[0], strings.Split(in.Price, "-")[1]
		if minprice != "" || maxprice != "" {
			sq := ``
			if minprice != "" {
				min, _ := strconv.ParseFloat(minprice, 64)
				minprice = fmt.Sprintf("%.0f", min*10000)
				if minprice == "0" {
					minprice = ""
				}
			}
			if maxprice != "" {
				max, _ := strconv.ParseFloat(maxprice, 64)
				maxprice = fmt.Sprintf("%.0f", max*10000)
				if maxprice == "0" {
					maxprice = ""
				}
			}
			if minprice != "" {
				sq += fmt.Sprintf(gte, minprice)
			}
			if minprice != "" && maxprice != "" {
				sq += `,`
			}
			if maxprice != "" {
				sq += fmt.Sprintf(lte, maxprice)
			}
			if minprice != "" || maxprice != "" {
				query_price := fmt.Sprintf(query_bool_should, fmt.Sprintf(query_bools_must, sq, sq))
				musts = append(musts, query_price)
			}
		}
	}
	hasBuyerTel := in.BuyerTel
	if hasBuyerTel != "" {
		if hasBuyerTel == "y" {
			must_not = append(must_not, fmt.Sprintf(query_missing, "buyertel"))
		} else {
			musts = append(musts, fmt.Sprintf(query_missing, "buyertel"))
		}
	}
	hasWinnerTel := in.WinnerTel
	if hasWinnerTel != "" {
		if hasWinnerTel == "y" {
			must_not = append(must_not, fmt.Sprintf(query_missing, "winnertel"))
		} else {
			musts = append(musts, fmt.Sprintf(query_missing, "winnertel"))
		}
	}
	//排除词
	notkey := in.ExclusionWords
	if notkey = strings.TrimSpace(notkey); notkey != "" {
		notkey_multi_match := fmt.Sprintf(multi_match, "%s", findfields)
		notkey_must_not := []string{}
		for _, v := range strings.Split(notkey, " ") {
			v = strings.TrimSpace(v)
			if v == "" {
				continue
			}
			notkey_must_not = append(notkey_must_not, fmt.Sprintf(notkey_multi_match, elastic.ReplaceYH(v)))
		}
		must_not = append(must_not, fmt.Sprintf(query_bool_should, strings.Join(notkey_must_not, ",")))
	}
	//
	fileExists := in.FileExists
	if !isFileSearch && fileExists != "" {
		if fileExists == "1" { //有附件
			must_not = append(must_not, fmt.Sprintf(query_missing, "filetext"))
		} else if fileExists == "-1" { //无附件
			musts = append(musts, fmt.Sprintf(query_missing, "filetext"))
		}
	}
	qstr = fmt.Sprintf(query, strings.Join(musts, ","), strings.Join(must_not, ","))
	logx.Info(qstr)
	return
}

/*
 * 结果列表转换，目前只换行行业字段
 * 所有的招标搜索都要调用此方法，列表中有展示行业的也可以用
 * industry 搜索条件中的行业，默认为空
 */
func BidListConvert(industry string, res *[]map[string]interface{}) {
	if res == nil {
		return
	}
	commonSubstring := func(v string) (value string) {
		bcs := strings.Split(v, "_")
		if len(bcs) == 1 {
			value = bcs[0]
		} else if len(bcs) == 2 {
			value = bcs[0]
			if strings.TrimSpace(value) == "" {
				value = bcs[0]
			}
		}
		return
	}
	for _, v := range *res {
		v["id"] = v["_id"]
		delete(v, "_id")
		budget, _ := v["budget"].(float64)
		bidamount, _ := v["bidamount"].(float64)
		if budget == 0 || strings.TrimSpace(fmt.Sprint(v["budget"])) == "" {
			delete(v, "budget")
		}
		if bidamount == 0 || strings.TrimSpace(fmt.Sprint(v["bidamount"])) == "" {
			delete(v, "bidamount")
		}
		value := ""
		subscopeclass, _ := v["s_subscopeclass"].(string)
		subscopeclass = strings.Trim(subscopeclass, ",")
		bct := strings.Split(subscopeclass, ",")
		if bct == nil || len(bct) == 0 {
			continue
		}
		//搜索条件中没有行业的话，取查询结果中第一个行业
		if industry == "" {
			value = commonSubstring(bct[0])
		} else { //搜索条件中有行业的话，取行业中和搜索条件相对应的第一个
			industrys := strings.Split(industry, ",")
		L:
			for _, bc := range bct {
				for _, is := range industrys {
					if bc == is {
						value = commonSubstring(bc)
						break L
					}
				}
			}
		}
		if strings.TrimSpace(value) == "" {
			continue
		}
		v["industry"] = value
	}
}

//是否收藏
func MakeCollection(userId string, list []*bxcore.SearchList) {
	if list == nil || len(list) == 0 {
		return
	}
	param := []interface{}{userId}
	wh := []string{}
	for _, v := range list {
		array := ME.DecodeArticleId2ByCheck(v.Id)
		if len(array) == 1 && array[0] != "" {
			param = append(param, array[0])
			wh = append(wh, "?")
		}
	}
	if len(wh) > 0 {
		result := IC.MainMysql.SelectBySql(`select bid from bdcollection where userid=? and bid in (`+strings.Join(wh, ",")+`)`, param...)
		bid_map := map[string]bool{}
		if result != nil {
			for _, v := range *result {
				bid_map[ME.EncodeArticleId2ByCheck(MC.ObjToString(v["bid"]))] = true
			}
		}
		for _, v := range list {
			if bid_map[v.Id] {
				v.IsCollected = true
			}
		}
	}
}

//默认查询缓存数据
func SearchCahcheData(in *bxcore.SearchReq) (count int64, list []*bxcore.SearchList) {
	//最新招标信息
	findfields := `"title"`
	qstr := GetSearchQuery(in, GetBidSearchQuery(in))
	//首页
	if in.PageNum == 1 {
		count = elastic.Count(INDEX, TYPE, qstr)
	}
	if count > 0 || in.PageNum > 1 {
		repl := elastic.GetAllByNgram(INDEX, TYPE, qstr, findfields, bidSearch_sort, bidSearch_field, int(in.PageNum), int(in.PageSize), 0, false)
		if repl != nil && *repl != nil && len(*repl) > 0 {
			BidListConvert(in.Industry, repl)
			list = searchListFormart(repl, false)
		}
	}
	return
}

//格式化数据
func searchListFormart(repl *[]map[string]interface{}, b bool) (list []*bxcore.SearchList) {
	for _, v := range *repl {
		var searchList = &bxcore.SearchList{}
		if b {
			//正文匹配检索关键词
			highlight, _ := v["highlight"].(map[string][]string)
			detail := ""
			for _, val := range highlight["detail"] {
				detail += ClearHtml.ReplaceAllString(val, "")
			}
			searchList.Detail = detail
		}
		searchList.Area = MC.ObjToString(v["area"])
		searchList.AreaUrl = IC.LabelMap[searchList.Area].Url
		searchList.BuyerClass = MC.ObjToString(v["buyerclass"])
		searchList.City = MC.ObjToString(v["city"])
		searchList.FileExists = v["filetext"] != nil
		searchList.Id = ME.EncodeArticleId2ByCheck(MC.ObjToString(v["id"]))
		searchList.Industry = MC.ObjToString(v["industry"])
		searchList.IndustryUrl = IC.LabelMap[searchList.Industry].Url
		searchList.PublishTime = MC.Int64All(v["publishtime"])
		searchList.Subtype = MC.ObjToString(v["subtype"])
		searchList.SubtypeUrl = IC.LabelMap[searchList.Subtype].Url
		searchList.Title = MC.ObjToString(v["title"])
		searchList.Budget = MC.Int64All(v["budget"])
		searchList.Bidamount = MC.Int64All(v["bidamount"])
		searchList.ProjectName = MC.ObjToString(v["projectname"])
		searchList.ProjectCode = MC.ObjToString(v["projectcode"])
		searchList.ProjectInfo = &bxcore.PInfo{}
		if v["projectinfo"] != nil {
			pinfo := MC.ObjToMap(v["projectinfo"])
			searchList.ProjectInfo.ApproveCode = MC.ObjToString((*pinfo)["approvecode"])
			searchList.ProjectInfo.ApproveContent = MC.ObjToString((*pinfo)["approvecontent"])
			searchList.ProjectInfo.ApproveDept = MC.ObjToString((*pinfo)["approvedept"])
			searchList.ProjectInfo.ApproveStatus = MC.ObjToString((*pinfo)["approvestatus"])
			searchList.ProjectInfo.ProjectType = MC.ObjToString((*pinfo)["projecttype"])
			searchList.ProjectInfo.ApproveNumber = MC.ObjToString((*pinfo)["approvenumber"])
			searchList.ProjectInfo.ApproveTime = MC.ObjToString((*pinfo)["approvetime"])
		}
		searchList.Winner = MC.ObjToString(v["winner"])
		searchList.Buyer = MC.ObjToString(v["buyer"])
		searchList.BidopenTime = MC.Int64All(v["bidopentime"])
		searchList.Site = MC.ObjToString(v["site"])
		searchList.SpiderCode = MC.ObjToString(v["spidercode"])
		list = append(list, searchList)
	}
	return
}

//合并map数据,去重,排序
func DelRepeatSearchData(resOne, resTwo []*bxcore.SearchList) []*bxcore.SearchList {
	if len(resOne) > 0 && len(resTwo) > 0 {
		for _, v := range resOne {
			for n, m := range resTwo {
				if v.Id == m.Id {
					resTwo = append(resTwo[0:n], resTwo[n+1:]...)
					break
				}
			}
		}
		resOne = append(resOne, resTwo...)
	} else {
		resOne = append(resOne, resTwo...)
	}
	//sort.Slice(resOne, func(i, j int) bool {
	//	return resOne[i].PublishTime > resOne[j].PublishTime
	//})
	return resOne
}