jyps/project_chuan/project_other.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/olivere/elastic/v7"
	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"regexp"
	"sort"
	"strings"
	"unicode"
)

var (
	FilterReg_3 = regexp.MustCompile("(项目|公告|公示)$")
	FilterReg_2 = regexp.MustCompile("^[）\\)>》】\\]}｝〕,，;；:：'\"“”。.\\?？、/+=\\_—*&……\\^%$￥@!！`~·（\\(<《【\\[{｛〔]+$")
	FilterReg_1 = regexp.MustCompile("^([0-9]{1,3}|[零一二三四五六七八九十]{1,2}|联系人?|电话|地址|编号|采购|政府采购|成交|更正|招标|中标|变更|结果)$")
	FilterReg   = regexp.MustCompile("^[的人号时元万公告项目地址电话邮编日期联系招标中结果成交项目项目采购采购项目政府采购公告更正公告]+$")
)

// getSearch 模拟网站所有，只提供精准和分词；不再打分；只过滤省市，只匹配标题、项目名称；根据参数决定是否匹配详情；
func getSearch(client *elastic.Client, projectName, areacode string, isDetail int) ([]map[string]interface{}, error) {
	var results []map[string]interface{}
	seenIDs := make(map[string]*elastic.SearchHit)
	province, city := "", ""
	if areacode != "" {
		code := areacode[:6]
		where := map[string]interface{}{
			"code": code,
		}

		res, _ := MgoQY.FindOne("address_new_2020", where)
		province = util.ObjToString((*res)["province"])
		city = util.ObjToString((*res)["city"])
	}
	//fmt.Println(province, city)
	projectName = RemoveInvisibleChars(projectName)
	projectName = FilterGeneric(projectName)

	// 1. 精准查询
	preciseHits, err := searchPreciseOther(client, projectName, province, city, isDetail)
	if err != nil {
		return nil, err
	}
	for _, hit := range preciseHits {
		if _, exists := seenIDs[hit.Id]; !exists {
			seenIDs[hit.Id] = hit
		}
	}

	// 2. 分词查询（
	tokenHits, err := searchByTokenOther(client, projectName, province, city, isDetail)
	if err != nil {
		return nil, err
	}
	for _, hit := range tokenHits {
		if _, exists := seenIDs[hit.Id]; !exists {
			seenIDs[hit.Id] = hit
		}
	}

	for id, hit := range seenIDs {
		var doc map[string]interface{}
		if err = json.Unmarshal(hit.Source, &doc); err != nil {
			continue
		}
		// 从 Mongo 读取 detail 字段用于后续 buyer 过滤
		bidd, _ := MgoB.FindById("bidding", id, nil)
		detail := util.ObjToString((*bidd)["detail"])
		if detail != "" {
			doc["detail"] = detail
		}
		results = append(results, doc)
	}

	sort.SliceStable(results, func(i, j int) bool {
		return util.Int64All(results[i]["publishtime"]) > util.Int64All(results[j]["publishtime"])
	})

	return results, nil

}

func getSearchNew(client *elastic.Client, projectName, areacode string, isDetail int) ([]map[string]interface{}, error) {
	var results []map[string]interface{}
	seenIDs := make(map[string]*elastic.SearchHit)
	province, city := "", ""
	if areacode != "" {
		code := areacode[:6]
		where := map[string]interface{}{
			"code": code,
		}

		res, _ := MgoQY.FindOne("address_new_2020", where)
		province = util.ObjToString((*res)["province"])
		city = util.ObjToString((*res)["city"])
	}
	//fmt.Println(province, city)
	projectName = RemoveInvisibleChars(projectName)
	projectName = FilterGeneric(projectName)

	// 1. 精准查询
	preciseHits, err := searchPreciseOther(client, projectName, province, city, isDetail)
	if err != nil {
		return nil, err
	}
	for _, hit := range preciseHits {
		if _, exists := seenIDs[hit.Id]; !exists {
			seenIDs[hit.Id] = hit
		}
	}

	// 2. 分词查询（
	tokenHits, err := searchByTokenOtherNew(client, projectName, province, city, isDetail)
	if err != nil {
		return nil, err
	}

	for _, hit := range tokenHits {
		if _, exists := seenIDs[hit.Id]; !exists {
			seenIDs[hit.Id] = hit
		}
	}

	for id, hit := range seenIDs {
		var doc map[string]interface{}
		if err = json.Unmarshal(hit.Source, &doc); err != nil {
			continue
		}
		// 从 Mongo 读取 detail 字段用于后续 buyer 过滤
		bidd, _ := MgoB.FindById("bidding", id, nil)
		detail := util.ObjToString((*bidd)["detail"])
		//has := true
		//for _, v := range tokens {
		//	if !strings.Contains(detail, v) {
		//		has = false
		//		break
		//	}
		//}
		//if !has {
		//	continue
		//}
		if detail != "" {
			doc["detail"] = detail
		}
		results = append(results, doc)
	}

	sort.SliceStable(results, func(i, j int) bool {
		return util.Int64All(results[i]["publishtime"]) > util.Int64All(results[j]["publishtime"])
	})

	return results, nil

}

// searchPreciseOther 精准搜索;m默认项目名称+标题；详情可选参数
func searchPreciseOther(client *elastic.Client, projectName, area, city string, isDetail int) ([]*elastic.SearchHit, error) {
	fieldsToTry := []string{"projectname.pname", "title"}
	//if isDetail > 0 {
	//	fieldsToTry = append(fieldsToTry, "detail")
	//}

	filtersToTry := [][]elastic.Query{
		{elastic.NewTermsQuery("subtype", "中标", "成交", "合同", "单一")},
		{elastic.NewTermsQuery("toptype", "招标", "预告", "采购意向", "拟建")},
	}

	runQuery := func(withArea bool) ([]*elastic.SearchHit, error) {
		var allResults []*elastic.SearchHit
		seenIDs := make(map[string]bool)
		for _, field := range fieldsToTry {
			for _, filters := range filtersToTry {
				var queries []*elastic.BoolQuery
				queryBase := elastic.NewBoolQuery().
					Must(elastic.NewMultiMatchQuery(projectName, field).Type("phrase")).
					Filter(filters...)

				if withArea {
					if area != "" {
						queryBase = queryBase.Must(elastic.NewTermQuery("area", area))
					}
					if city != "" {
						queryBase = queryBase.Must(elastic.NewTermQuery("city", city))
					}
				}

				queries = append(queries, queryBase)

				for _, query := range queries {
					fetchFields := elastic.NewFetchSourceContext(true).Include(
						"id", "title", "projectname", "projectcode", "bidamount", "area", "city",
						"toptype", "subtype", "buyer", "budget", "buyerperson", "buyertel",
						"s_winner", "winnertel", "agency", "publishtime")

					searchResult, err := client.Search().
						Index("bidding").
						Query(query).
						FetchSourceContext(fetchFields).
						Do(context.Background())
					if err != nil {
						return nil, err
					}

					for _, hit := range searchResult.Hits.Hits {
						if !seenIDs[hit.Id] {
							seenIDs[hit.Id] = true
							allResults = append(allResults, hit)
						}
					}
				}
			}
		}
		return allResults, nil
	}

	// 第一次查询：包含省份和城市过滤（如果有）
	results, err := runQuery(true)
	if err != nil {
		return nil, err
	}
	if len(results) == 0 && area != "" {
		// 如果查不到，并且存在省份条件，再执行一次去掉 area 的查询
		return runQuery(false)
	}
	return results, nil
}

// searchByTokenOther 分词查询;
func searchByTokenOther2(client *elastic.Client, projectName, province, city string, isDetail int) ([]*elastic.SearchHit, error) {
	fieldsToTry := []string{"projectname.pname", "title"}
	if isDetail > 0 {
		fieldsToTry = append(fieldsToTry, "detail")
	}
	filtersToTry := [][]elastic.Query{
		{elastic.NewTermsQuery("subtype", "中标", "成交", "合同", "单一")},
		{elastic.NewTermsQuery("toptype", "招标", "预告", "采购意向", "拟建")},
	}

	// 分词处理
	analyzeResp, err := client.IndexAnalyze().
		Index("bidding").
		Analyzer("ik_smart").
		Text(projectName).
		Do(context.Background())
	if err != nil {
		return nil, err
	}

	var tokens []string
	for _, token := range analyzeResp.Tokens {
		tokens = append(tokens, token.Token)
	}
	if len(tokens) == 0 {
		return nil, fmt.Errorf("no tokens found from ik_smart")
	}
	queryText := strings.Join(tokens, " ")

	// 指定返回字段
	fetchFields := elastic.NewFetchSourceContext(true).Include(
		"id", "title", "projectname", "projectcode", "bidamount", "area", "city",
		"toptype", "subtype", "buyer", "budget", "buyerperson", "buyertel",
		"s_winner", "winnertel", "agency", "publishtime")

	var allHits []*elastic.SearchHit
	seen := make(map[string]bool)

	for _, field := range fieldsToTry {
		for _, filters := range filtersToTry {
			var queries []*elastic.BoolQuery

			if field == "detail" {
				query := elastic.NewBoolQuery()
				if province != "" {
					query = query.Must(elastic.NewTermQuery("area", province))
				}
				query = query.Must(elastic.NewMatchQuery(field, queryText)).
					Filter(filters...)
				queries = append(queries, query)
			} else {
				// 省+市
				if province != "" && city != "" {
					q := elastic.NewBoolQuery().
						Must(elastic.NewMatchQuery(field, queryText)).
						Must(elastic.NewTermQuery("area", province)).
						Must(elastic.NewTermQuery("city", city)).
						Filter(filters...)
					queries = append(queries, q)
				}
				// 仅省
				if province != "" {
					q := elastic.NewBoolQuery().
						Must(elastic.NewMatchQuery(field, queryText)).
						Must(elastic.NewTermQuery("area", province)).
						Filter(filters...)
					queries = append(queries, q)
				}

			}

			for _, query := range queries {
				searchResult, err := client.Search().
					Index("bidding").
					Query(query).
					FetchSourceContext(fetchFields).
					Do(context.Background())
				if err != nil {
					continue
				}

				for _, hit := range searchResult.Hits.Hits {
					if !seen[hit.Id] {
						seen[hit.Id] = true
						allHits = append(allHits, hit)
					}
				}
			}
		}

	}

	return allHits, nil
}

func searchByTokenOther(client *elastic.Client, projectName, province, city string, isDetail int) ([]*elastic.SearchHit, error) {
	fieldsToTry := []string{"projectname.pname", "title"}
	if isDetail > 0 {
		fieldsToTry = append(fieldsToTry, "detail")
	}
	filtersToTry := [][]elastic.Query{
		{elastic.NewTermsQuery("subtype", "中标", "成交", "合同", "单一")},
		{elastic.NewTermsQuery("toptype", "招标", "预告", "采购意向", "拟建")},
	}

	// 分词处理
	analyzeResp, err := client.IndexAnalyze().
		Index("bidding").
		Analyzer("ik_smart").
		Text(projectName).
		Do(context.Background())
	if err != nil {
		return nil, err
	}

	var tokens []string
	for _, token := range analyzeResp.Tokens {
		tokens = append(tokens, token.Token)
	}
	if len(tokens) == 0 {
		return nil, fmt.Errorf("no tokens found from ik_smart")
	}
	queryText := strings.Join(tokens, " ")

	// 指定返回字段
	fetchFields := elastic.NewFetchSourceContext(true).Include(
		"id", "title", "projectname", "projectcode", "bidamount", "area", "city",
		"toptype", "subtype", "buyer", "budget", "buyerperson", "buyertel",
		"s_winner", "winnertel", "agency", "publishtime")

	// 抽象出内部查询逻辑，参数控制是否使用省份过滤
	runQuery := func(withProvince bool) ([]*elastic.SearchHit, error) {
		var allHits []*elastic.SearchHit
		seen := make(map[string]bool)

		for _, field := range fieldsToTry {
			for _, filters := range filtersToTry {
				//query := elastic.NewBoolQuery().
				//	Must(elastic.NewMatchQuery(field, queryText)).
				//	Filter(filters...)

				query := elastic.NewBoolQuery().
					Must(
						elastic.NewMultiMatchQuery(queryText, field).
							MinimumShouldMatch("100%"),
					).
					Filter(filters...)

				// 动态加上 area/city 条件
				if withProvince && province != "" {
					query = query.Must(elastic.NewTermQuery("area", province))
				}
				if city != "" {
					query = query.Must(elastic.NewTermQuery("city", city))
				}

				searchResult, err := client.Search().
					Index("bidding").
					Query(query).
					FetchSourceContext(fetchFields).
					Do(context.Background())
				if err != nil {
					continue
				}

				for _, hit := range searchResult.Hits.Hits {
					if !seen[hit.Id] {
						seen[hit.Id] = true
						allHits = append(allHits, hit)
					}
				}
			}
		}
		return allHits, nil
	}

	// 第一次尝试带上 province
	results, err := runQuery(true)
	if err != nil {
		return nil, err
	}
	// 如果查不到，并且设置了省份，则再试一次去掉 province
	if len(results) == 0 && province != "" {
		return runQuery(false)
	}
	return results, nil
}

// searchByTokenOtherNew searchByTokenOtherNew
func searchByTokenOtherNew(client *elastic.Client, projectName, province, city string, isDetail int) ([]*elastic.SearchHit, error) {
	var tokens []string
	fieldsToTry := []string{"projectname.pname", "title"}
	if isDetail > 0 {
		fieldsToTry = append(fieldsToTry, "detail")
	}
	filtersToTry := [][]elastic.Query{
		{elastic.NewTermsQuery("subtype", "中标", "成交", "合同", "单一")},
		{elastic.NewTermsQuery("toptype", "招标", "预告", "采购意向", "拟建")},
	}

	// 分词处理
	analyzeResp, err := client.IndexAnalyze().
		Index("bidding").
		Analyzer("ik_smart").
		Text(projectName).
		Do(context.Background())
	if err != nil {
		return nil, err
	}

	for _, token := range analyzeResp.Tokens {
		tokens = append(tokens, token.Token)
	}
	if len(tokens) == 0 {
		return nil, fmt.Errorf("no tokens found from ik_smart")
	}

	// 指定返回字段
	fetchFields := elastic.NewFetchSourceContext(true).Include(
		"id", "title", "projectname", "projectcode", "bidamount", "area", "city",
		"toptype", "subtype", "buyer", "budget", "buyerperson", "buyertel",
		"s_winner", "winnertel", "agency", "publishtime")

	// 抽象出内部查询逻辑，参数控制是否使用省份过滤
	runQuery := func(withProvince bool) ([]*elastic.SearchHit, error) {
		var allHits []*elastic.SearchHit
		seen := make(map[string]bool)

		for _, field := range fieldsToTry {
			boolQ := elastic.NewBoolQuery()
			for _, token := range tokens {
				boolQ = boolQ.Must(
					elastic.NewTermQuery(field, token), // 精确匹配分词结果
				)
			}
			for _, filters := range filtersToTry {
				query := elastic.NewBoolQuery().
					Must(
						boolQ,
					).
					Filter(filters...)

				// 动态加上 area/city 条件
				if withProvince && province != "" {
					query = query.Must(elastic.NewTermQuery("area", province))
				}
				if withProvince && city != "" {
					query = query.Must(elastic.NewTermQuery("city", city))
				}

				searchResult, err := client.Search().
					Index("bidding").
					Query(query).
					FetchSourceContext(fetchFields).
					Do(context.Background())
				if err != nil {
					continue
				}

				for _, hit := range searchResult.Hits.Hits {
					if !seen[hit.Id] {
						seen[hit.Id] = true
						allHits = append(allHits, hit)
					}
				}
			}
		}
		return allHits, nil
	}

	// 第一次尝试带上 province
	results, err := runQuery(true)
	if err != nil {
		return nil, err
	}
	// 如果查不到，并且设置了省份，则再试一次去掉 province
	if len(results) == 0 && province != "" {
		return runQuery(false)
	}
	return results, nil
}

// RemoveInvisibleChars 移除控制字符和不可见字符
func RemoveInvisibleChars(s string) string {
	return strings.Map(func(r rune) rune {
		// 保留普通字符、中文、标点等可见字符
		if unicode.IsGraphic(r) && !unicode.IsControl(r) {
			return r
		}
		return -1
	}, s)
}

// FilterGeneric 通用词处理
func FilterGeneric(keyWords string) string {
	keyWords = FilterReg_3.ReplaceAllString(keyWords, "")
	keyWords = FilterReg_2.ReplaceAllString(keyWords, "")
	keyWords = FilterReg_1.ReplaceAllString(keyWords, "")
	keyWords = FilterReg.ReplaceAllString(keyWords, "")
	return keyWords
}