jyMicroservices/jyBXCore/rpc/util/search.go

package util

import (
	MC "app.yhyue.com/moapp/jybase/common"
	elastic "app.yhyue.com/moapp/jybase/esv1"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math/big"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode"
)

var MatchSpace = regexp.MustCompile("\\s+")
var filterReg_3 = regexp.MustCompile("(项目|公告|公示)$")
var filterReg_2 = regexp.MustCompile("^[）\\)>》】\\]}｝〕,，;；:：'\"“”。.\\?？、/+=\\_—*&……\\^%$￥@!！`~·（\\(<《【\\[{｛〔]+$")
var filterReg_1 = regexp.MustCompile("^([0-9]{1,3}|[零一二三四五六七八九十]{1,2}|联系人?|电话|地址|编号|采购|政府采购|成交|更正|招标|中标|变更|结果)$")
var filterReg = regexp.MustCompile("^[的人号时元万公告项目地址电话邮编日期联系招标中结果成交项目项目采购采购项目政府采购公告更正公告]+$")
var PhoneReg = regexp.MustCompile("^[1][3-9][0-9]{9}$")
var EmailPattern = regexp.MustCompile("^([a-zA-Z0-9_\\-\\.]+)@((\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.)|(([a-zA-Z0-9\\-]+\\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\\]?)$")

func SearchHistory(history, searchvalue string) []string {
	arrs := strings.Split(history, ",")
	//新增历史记录
	if history == "" {
		arrs = make([]string, 0)
	}
	for k, v := range arrs {
		if v == strings.TrimSpace(searchvalue) {
			arrs = append(arrs[:k], arrs[k+1:]...)
			break
		}
	}
	arrs = append(arrs, searchvalue)
	if len(arrs) > 10 {
		arrs = arrs[1:11]
	}
	return arrs
}

func FilteKey(k string) string {
	k = strings.TrimSpace(k)
	k = filterReg_3.ReplaceAllString(k, "")
	k = filterReg_2.ReplaceAllString(k, "")
	k = filterReg_1.ReplaceAllString(k, "")
	k = filterReg.ReplaceAllString(k, "")
	return k
}

//超过20个字，截断
//返回截取后的字符串和截取掉中的前3个字
func InterceptSearchKW(word string, isIntercept, isFilter bool) (b_word, a_word, s_word string) {
	if isFilter {
		word = FilteKey(word)
	}
	word = MatchSpace.ReplaceAllString(strings.TrimSpace(word), " ")
	words := []rune(word)
	if len(words) > 20 && isIntercept {
		b_word = string(words[:20])
		b_word = strings.TrimSpace(b_word)
		if len(words) > 23 {
			a_word = string(words[20:23])
		} else {
			a_word = string(words[20:])
		}
	} else {
		b_word = word
	}
	a_word = strings.TrimSpace(a_word)
	s_word = MatchSpace.ReplaceAllString(b_word, "+")
	return
}

func HttpEs(ques, analyzer, esAddress string) (res string) {
	var addrs []string
	surl := ""
	for _, s := range strings.Split(esAddress, ",") {
		addrs = append(addrs, s)
	}
	i, _ := rand.Int(rand.Reader, big.NewInt(int64(len(addrs)))) //随机
	surl = addrs[int(i.Int64())] + "/bidding/_analyze"
	URL, _ := url.Parse(surl)
	Q := URL.Query()
	Q.Add("text", ques)
	Q.Add("analyzer", analyzer)
	URL.RawQuery = Q.Encode()
	resp, err := http.Get(URL.String())
	if err != nil {
		log.Println("es连接失败 err1:", err)
		resp, err = getesResp(ques, analyzer, addrs)
		if err != nil {
			return
		}
	}
	result, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		defer resp.Body.Close()
		var resmap map[string]interface{}
		json.Unmarshal(result, &resmap)
		if resmap != nil && resmap["tokens"] != nil {
			tokens := MC.ObjArrToMapArr(resmap["tokens"].([]interface{}))
			for _, v := range tokens {
				token := MC.ObjToString(v["token"])
				if len([]rune(token)) == 1 && unicode.IsLetter([]rune(token)[0]) {
					continue
				}
				if res != "" {
					res += "+"
				}
				res += token
			}
		}
	}
	return
}

//
func getesResp(ques, analyzer string, addrs []string) (resp *http.Response, err error) {
	for _, v := range addrs {
		surl := v + "/bidding/_analyze"
		URL, _ := url.Parse(surl)
		Q := URL.Query()
		Q.Add("text", ques)
		Q.Add("analyzer", analyzer)
		URL.RawQuery = Q.Encode()
		resp, err = http.Get(URL.String())
		if err == nil {
			break
		}
	}
	return resp, err
}

//pc、微信、app 招标信息搜索

const (
	INDEX          = "bidding"
	TYPE           = "bidding"
	bidSearch_sort = `{"publishtime":-1}`

	//招标搜索分页--每页显示数量
	SearchPageSize_APP = 50
	SearchPageSize_WX  = 50
	SearchPageSize_PC  = 50
	//招标搜索分页--最大页数
	SearchMaxPageNum_APP   = 10
	SearchMaxPageNum_WX    = 10
	SearchMaxPageNum_PC    = 10  //免费用户500条记录
	SearchMaxPageNum_PAYED = 100 //付费用户5000条记录

	bidSearch_field_1 = `"_id","title","publishtime","toptype","subtype","type","area","city","s_subscopeclass","bidamount","budget","buyerclass"`
	bidSearch_field   = bidSearch_field_1 + `,"bidopentime","winner","buyer","projectname","projectcode","projectinfo"`
)

func SearchData(platform string, request *http.Request, currentPage int, userId, secondKWS, s_word, area, city, publishtime, subtype, industry, minprice, maxprice, winner, buyerclass, hasBuyerTel, hasWinnerTel, fileExists string, start, pageSize int, isGetCount bool, queryItems []string, field, notkey string, isPayedUser bool) (second, b_word, a_word, pcAjaxFlag, secondFlag string, count, totalPage int64, list *[]map[string]interface{}) {
	count, totalPage, list = GetPcBidSearchData(s_word, area, city, publishtime, subtype, industry, minprice, maxprice, winner, buyerclass, hasBuyerTel, hasWinnerTel, fileExists, start, pageSize, isGetCount, queryItems, field, notkey, isPayedUser)
	//if len([]rune(s_word)) > 3 && int(count) < SearchPageSize_PC && start == 0 {
	//	var paramList = list
	//	s_word, pcAjaxFlag, secondFlag, second, list = IntegratedData(platform, s_word, secondKWS, industry, minprice, maxprice, hasBuyerTel, hasWinnerTel, fileExists, secondFlag, area, city, publishtime, subtype, buyerclass, notkey, queryItems, paramList)
	//}

	listSize := 0
	if list != nil {
		listSize = len(*list)
	}
	//public.SaveUserSearchLog(request, userId, -1, platform, "超级搜索", map[string]interface{}{
	//	"search_word":        MC.If(platform == "app" || platform == "wx", searchvalue, s_word),
	//	"search_area":        area,
	//	"search_city":        city,
	//	"search_price":       []string{minprice, maxprice},
	//	"search_publishtime": publishtime,
	//	"search_type":        subtype,
	//	"search_industry":    industry,
	//	"pagenum":            currentPage,
	//	"pagesize":           listSize,
	//	"fileExists":         fileExists,
	//})
	return
}

//GetPcBidSearchData pc端招标信息搜索
func GetPcBidSearchData(searchvalue, area, city, publishtime, subtype, industry, minprice, maxprice, winner, buyerclass, hasBuyerTel, hasWinnerTel, fileExists, selectType, field, notkey string, start, pageSize int, isGetCount bool, ispayed bool) (count, totalPage int64, list *[]map[string]interface{}) {
	var findfields string
	var hightlightContent bool = false //是否高亮正文
	for _, v := range selectTypeArr {
		if v == "detail" {
			hightlightContent = true
			break
		}
	}
	if selectTypeArr == nil || len(selectTypeArr) == 0 {
		findfields = `"title"`
	} else {
		findfields = fmt.Sprintf(`"%s"`, strings.Join(selectTypeArr, "\",\""))
	}
	qstr := GetSearchQuery(searchvalue, industry, minprice, maxprice, hasBuyerTel, hasWinnerTel, fileExists, findfields, GetBidSearchQuery(area, city, publishtime, subtype, winner, buyerclass), notkey)
	if isGetCount && qstr != "" && start == 0 {
		count = elastic.Count(INDEX, TYPE, qstr)
	}
	if !isGetCount || count > 0 || start > 0 {
		var repl *[]map[string]interface{}
		if hightlightContent {
			repl = elastic.GetAllByNgram(INDEX, TYPE, qstr, `"detail"`, bidSearch_sort, field, start, pageSize, 115, true)
		} else {
			repl = elastic.GetAllByNgram(INDEX, TYPE, qstr, ``, bidSearch_sort, field, start, pageSize, 0, false)
		}
		if repl != nil && *repl != nil && len(*repl) > 0 {
			BidListConvert(industry, repl)
			list = repl
		}
	}
	limitCount := MC.If(ispayed, int64(SearchPageSize_PC*SearchMaxPageNum_PAYED), int64(SearchPageSize_PC*SearchMaxPageNum_PC)).(int64)
	if count > limitCount {
		count = limitCount
	}
	totalPage = (count + int64(SearchPageSize_PC) - 1) / int64(SearchPageSize_PC)
	return
}

func GetBidSearchQuery(area, city, publishtime, subtype, winner, buyerclass string) string {
	query := ``
	if area != "" {
		query += `{"terms":{"area":[`
		for k, v := range strings.Split(area, ",") {
			if k > 0 {
				query += `,`
			}
			query += `"` + v + `"`
		}
		query += `]}}`
	}
	//
	if city != "" {
		if len(query) > 0 {
			query += ","
		}
		query += `{"terms":{"city":[`
		for k, v := range strings.Split(city, ",") {
			if k > 0 {
				query += `,`
			}
			query += `"` + v + `"`
		}
		query += `]}}`
	}
	if publishtime != "" {
		if len(query) > 0 {
			query += ","
		}
		starttime, endtime := "", ""
		now := time.Now()
		if publishtime == "lately-7" { //最近7天
			starttime = fmt.Sprint(time.Date(now.Year(), now.Month(), now.Day()-7, 0, 0, 0, 0, time.Local).Unix())
		} else if publishtime == "lately-30" { //最近30天
			starttime = fmt.Sprint(time.Date(now.Year(), now.Month(), now.Day()-30, 0, 0, 0, 0, time.Local).Unix())
		} else if publishtime == "thisyear" { //最近一年
			starttime = fmt.Sprint(time.Date(now.Year()-1, now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), 0, time.Local).Unix())
			endtime = fmt.Sprint(now.Unix())
		} else if publishtime == "threeyear" { //最近三年
			starttime = fmt.Sprint(time.Date(now.Year()-3, now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), 0, time.Local).Unix())
			endtime = fmt.Sprint(now.Unix())
		} else if publishtime == "fiveyear" { //最近五年
			starttime = fmt.Sprint(time.Date(now.Year()-5, now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), 0, time.Local).Unix())
			endtime = fmt.Sprint(now.Unix())
		} else {
			starttime = strings.Split(publishtime, "_")[0]
			endtime = strings.Split(publishtime, "_")[1]
			etTime := time.Now()
			if endtime != "" {
				et, _ := strconv.ParseInt(endtime, 0, 64)
				etTime = time.Unix(et, 0)
			}
			endtime = fmt.Sprint(time.Date(etTime.Year(), etTime.Month(), etTime.Day()+1, 0, 0, 0, 0, time.Local).Unix())
		}
		query += `{"range":{"publishtime":{`
		if starttime != "" {
			query += `"gte":` + starttime
		}
		if starttime != "" && endtime != "" {
			query += `,`
		}
		if endtime != "" {
			query += `"lt":` + endtime
		}
		query += `}}}`
	}
	if subtype != "" {
		if len(query) > 0 {
			query += ","
		}
		query += `{"terms":{"subtype":[`
		for k, v := range strings.Split(subtype, ",") {
			if k > 0 {
				query += `,`
			}
			query += `"` + v + `"`
		}
		query += `]}}`
	}
	if winner != "" {
		if len(query) > 0 {
			query += ","
		}
		query += `{"terms":{"s_winner":[`
		for k, v := range strings.Split(winner, ",") {
			if k > 0 {
				query += `,`
			}
			query += `"` + v + `"`
		}
		query += `]}}`
	}
	if buyerclass != "" {
		if len(query) > 0 {
			query += ","
		}
		query += `{"terms":{"buyerclass":[`
		for k, v := range strings.Split(buyerclass, ",") {
			if k > 0 {
				query += `,`
			}
			query += `"` + v + `"`
		}
		query += `]}}`
	}
	return query
}

func GetSearchQuery(keyword, industry, minprice, maxprice, hasBuyerTel, hasWinnerTel, fileExists, findfields, mustquery, notkey string) (qstr string) {
	multi_match := `{"multi_match": {"query": "%s","type": "phrase", "fields": [%s]}}`
	query := `{"query":{"bool":{"must":[%s],"must_not":[%s]}}}`
	query_bool_should := `{"bool":{"should":[%s],"minimum_should_match": 1}}`
	query_bools_must := `{"bool":{"must":[{"range":{"bidamount":{%s}}}]}},{"bool":{"must":[{"range":{"budget":{%s}}}],"must_not":[{"range":{"bidamount":{"gte":-1}}}]}}`
	query_bool_must := `{"bool":{"must":[{"terms":{"s_subscopeclass":[%s]}}]}}`
	query_missing := `{"constant_score":{"filter":{"missing":{"field":"%s"}}}}`
	gte := `"gte": %s`
	lte := `"lte": %s`
	musts, must_not := []string{}, []string{}
	if mustquery != "" {
		musts = append(musts, mustquery)
	}

	var isFileSearch bool = false //搜索范围是否有附件选择
	if keyword != "" {
		if strings.Contains(findfields, "filetext") { //搜索范围选择附件，是否有附件条件无效；
			isFileSearch = true
		}
		keyword_multi_match := fmt.Sprintf(multi_match, "%s", findfields)
		shoulds := []string{}
		for _, v := range strings.Split(keyword, "+") {
			shoulds = append(shoulds, fmt.Sprintf(keyword_multi_match, elastic.ReplaceYH(v)))
		}
		musts = append(musts, fmt.Sprintf(elastic.NgramMust, strings.Join(shoulds, ",")))
	}
	if industry != "" {
		industrys := strings.Split(industry, ",")
		musts = append(musts, fmt.Sprintf(query_bool_must, `"`+strings.Join(industrys, `","`)+`"`))
	}
	if minprice != "" || maxprice != "" {
		sq := ``
		if minprice != "" {
			min, _ := strconv.ParseFloat(minprice, 64)
			minprice = fmt.Sprintf("%.0f", min*10000)
			if minprice == "0" {
				minprice = ""
			}
		}
		if maxprice != "" {
			max, _ := strconv.ParseFloat(maxprice, 64)
			maxprice = fmt.Sprintf("%.0f", max*10000)
			if maxprice == "0" {
				maxprice = ""
			}
		}
		if minprice != "" {
			sq += fmt.Sprintf(gte, minprice)
		}
		if minprice != "" && maxprice != "" {
			sq += `,`
		}
		if maxprice != "" {
			sq += fmt.Sprintf(lte, maxprice)
		}
		if minprice != "" || maxprice != "" {
			query_price := fmt.Sprintf(query_bool_should, fmt.Sprintf(query_bools_must, sq, sq))
			musts = append(musts, query_price)
		}
	}
	if hasBuyerTel != "" {
		if hasBuyerTel == "y" {
			must_not = append(must_not, fmt.Sprintf(query_missing, "buyertel"))
		} else {
			musts = append(musts, fmt.Sprintf(query_missing, "buyertel"))
		}
	}
	if hasWinnerTel != "" {
		if hasWinnerTel == "y" {
			must_not = append(must_not, fmt.Sprintf(query_missing, "winnertel"))
		} else {
			musts = append(musts, fmt.Sprintf(query_missing, "winnertel"))
		}
	}
	if notkey = strings.TrimSpace(notkey); notkey != "" {
		notkey_multi_match := fmt.Sprintf(multi_match, "%s", findfields)
		notkey_must_not := []string{}
		for _, v := range strings.Split(notkey, " ") {
			v = strings.TrimSpace(v)
			if v == "" {
				continue
			}
			notkey_must_not = append(notkey_must_not, fmt.Sprintf(notkey_multi_match, elastic.ReplaceYH(v)))
		}
		must_not = append(must_not, fmt.Sprintf(query_bool_should, strings.Join(notkey_must_not, ",")))
	}
	if !isFileSearch && fileExists != "" {
		if fileExists == "1" { //有附件
			must_not = append(must_not, fmt.Sprintf(query_missing, "filetext"))
		} else if fileExists == "-1" { //无附件
			musts = append(musts, fmt.Sprintf(query_missing, "filetext"))
		}
	}
	qstr = fmt.Sprintf(query, strings.Join(musts, ","), strings.Join(must_not, ","))
	log.Println(qstr)
	return
}

/*
 * 结果列表转换，目前只换行行业字段
 * 所有的招标搜索都要调用此方法，列表中有展示行业的也可以用
 * industry 搜索条件中的行业，默认为空
 */
func BidListConvert(industry string, list *[]map[string]interface{}) {
	if list == nil {
		return
	}
	commonSubstring := func(v string) (value string) {
		bcs := strings.Split(v, "_")
		if len(bcs) == 1 {
			value = bcs[0]
		} else if len(bcs) == 2 {
			value = bcs[0]
			if strings.TrimSpace(value) == "" {
				value = bcs[0]
			}
		}
		return
	}
	for _, v := range *list {
		budget, _ := v["budget"].(float64)
		bidamount, _ := v["bidamount"].(float64)
		if budget == 0 || strings.TrimSpace(fmt.Sprint(v["budget"])) == "" {
			delete(v, "budget")
		}
		if bidamount == 0 || strings.TrimSpace(fmt.Sprint(v["bidamount"])) == "" {
			delete(v, "bidamount")
		}
		value := ""
		subscopeclass, _ := v["s_subscopeclass"].(string)
		subscopeclass = strings.Trim(subscopeclass, ",")
		bct := strings.Split(subscopeclass, ",")
		if bct == nil || len(bct) == 0 {
			continue
		}
		//搜索条件中没有行业的话，取查询结果中第一个行业
		if industry == "" {
			value = commonSubstring(bct[0])
		} else { //搜索条件中有行业的话，取行业中和搜索条件相对应的第一个
			industrys := strings.Split(industry, ",")
		L:
			for _, bc := range bct {
				for _, is := range industrys {
					if bc == is {
						value = commonSubstring(bc)
						break L
					}
				}
			}
		}
		if strings.TrimSpace(value) == "" {
			continue
		}
		v["industry"] = value
	}
}