Merge remote-tracking branch 'origin/master'

common/src/qfw/util/bidsearch/search.go

@@ -1,8 +1,6 @@
 package bidsearch
 
 import (
-	"app.yhyue.com/moapp/jybase/go-xweb/httpsession"
-	"app.yhyue.com/moapp/jypkg/middleground"
 	"fmt"
 	"log"
 	"net/http"
@@ -10,11 +8,14 @@ import (
 	"strings"
 	"time"
 
+	"app.yhyue.com/moapp/jybase/go-xweb/httpsession"
+	"app.yhyue.com/moapp/jypkg/middleground"
+
 	. "app.yhyue.com/moapp/jybase/encrypt"
 	"app.yhyue.com/moapp/jypkg/public"
 
 	util "app.yhyue.com/moapp/jybase/common"
-	elastic "app.yhyue.com/moapp/jybase/esv1"
+	elastic "app.yhyue.com/moapp/jybase/es"
 	"app.yhyue.com/moapp/jypkg/common/src/qfw/util/jy"
 )
 
@@ -269,8 +270,8 @@ func GetSearchQuery(keyword, industry, minprice, maxprice, hasBuyerTel, hasWinne
 	query_bool_should := `{"bool":{"should":[%s],"minimum_should_match": 1}}`
 	query_bools_must := `{"bool":{"must":[{"range":{"bidamount":{%s}}}]}},{"bool":{"must":[{"range":{"budget":{%s}}}],"must_not":[{"range":{"bidamount":{"gte":-1}}}]}}`
 	query_bool_must := `{"bool":{"must":[{"terms":{"s_subscopeclass":[%s]}}]}}`
-	query_missing := `{"constant_score":{"filter":{"missing":{"field":"%s"}}}}`
-	query_bool_must_term := `{"bool": {"must": [{ "term": {"isValidFile": %d }}]}}`
+	query_exists := `{"constant_score":{"filter":{"exists":{"field":"%s"}}}}`
+	query_bool_must_term := `{"bool": {"must": [{ "term": {"isValidFile": %t }}]}}`
 	query_bool_must_term_domain := `{"bool": {"must": [{ "term": {"bid_field": "%s" }}]}}` // 领域化数据类型
 	gte := `"gte": %s`
 	lte := `"lte": %s`
@@ -344,16 +345,16 @@ func GetSearchQuery(keyword, industry, minprice, maxprice, hasBuyerTel, hasWinne
 	}
 	if hasBuyerTel != "" {
 		if hasBuyerTel == "y" {
-			must_not = append(must_not, fmt.Sprintf(query_missing, "buyertel"))
+			musts = append(musts, fmt.Sprintf(query_exists, "buyertel"))
 		} else {
-			musts = append(musts, fmt.Sprintf(query_missing, "buyertel"))
+			must_not = append(must_not, fmt.Sprintf(query_exists, "buyertel"))
 		}
 	}
 	if hasWinnerTel != "" {
 		if hasWinnerTel == "y" {
-			must_not = append(must_not, fmt.Sprintf(query_missing, "winnertel"))
+			musts = append(musts, fmt.Sprintf(query_exists, "winnertel"))
 		} else {
-			musts = append(musts, fmt.Sprintf(query_missing, "winnertel"))
+			must_not = append(must_not, fmt.Sprintf(query_exists, "winnertel"))
 		}
 	}
 	if notkey = strings.TrimSpace(notkey); notkey != "" {
@@ -376,10 +377,9 @@ func GetSearchQuery(keyword, industry, minprice, maxprice, hasBuyerTel, hasWinne
 	}
 	if !isFileSearch && fileExists != "" {
 		if fileExists == "1" { //有附件
-			must_not = append(must_not, fmt.Sprintf(query_missing, "isValidFile"))
-			musts = append(musts, fmt.Sprintf(query_bool_must_term, 1))
+			musts = append(musts, fmt.Sprintf(query_bool_must_term, true))
 		} else if fileExists == "-1" { //无附件
-			musts = append(musts, fmt.Sprintf(query_missing, "isValidFile"))
+			must_not = append(must_not, fmt.Sprintf(query_bool_must_term, true))
 		}
 	}
 	// 如果是领域化数据则需要加标签

common/src/qfw/util/dataexport/common.go

@@ -4,7 +4,6 @@ import (
 	qutil "app.yhyue.com/moapp/jybase/common"
 	"app.yhyue.com/moapp/jybase/mysql"
 	"fmt"
-	"log"
 	"strings"
 )
 
@@ -46,17 +45,13 @@ func GetDataExportMatchKey(scd *SieveCondition, list *[]map[string]interface{})
 		return
 	}
 	if scd.Comeinfrom == "pushHistory" { //历史推送数据使用推送匹配关键词 if len(scd.PushKeyWords) == len(*list) {
-		if len(scd.SelectIds) == len(scd.PushKeyWords) {
-			for _, v := range *list {
-				for k, v1 := range scd.SelectIds {
-					if v1 == qutil.InterfaceToStr(v["_id"]) {
-						v["keyword"] = scd.PushKeyWords[k]
-					}
+		for _, v := range *list {
+			for k, v1 := range scd.SelectIds {
+				if v1 == qutil.InterfaceToStr(v["_id"]) {
+					v["keyword"] = scd.PushKeyWords[k]
 				}
-				//v["keyword"] = scd.PushKeyWords[qutil.InterfaceToStr(v["_id"])]
 			}
-		} else {
-			log.Println("历史推送数据订阅词匹配失败++++", len(scd.SelectIds), len(scd.PushKeyWords), scd.Id)
+			//v["keyword"] = scd.PushKeyWords[qutil.InterfaceToStr(v["_id"])]
 		}
 	} else if len(scd.Keyword) > 0 { //关键词二次匹配
 		types := scd.SelectType
@@ -71,7 +66,9 @@ func GetDataExportMatchKey(scd *SieveCondition, list *[]map[string]interface{})
 			}
 			if len(vk.Appended) > 0 {
 				for _, v := range vk.Appended {
-					key = append(key, strings.Split(v, " ")...)
+					if v != "" {
+						key = append(key, strings.Split(v, " ")...)
+					}
 				}
 			}
 			vk.Keyword = strings.Join(key, ",")
@@ -102,15 +99,15 @@ func GetDataExportMatchKey(scd *SieveCondition, list *[]map[string]interface{})
 					projectname := qutil.ObjToString(data["projectname"])
 					keyWord = KeyWordToDatas(projectname, key, keyWord)
 				}
-				if strings.Contains(types, "mbuyer") {
+				if strings.Contains(types, "buyer.mbuyer") {
 					buyer := qutil.ObjToString(data["buyer"])
 					keyWord = KeyWordToDatas(buyer, key, keyWord)
 				}
-				if strings.Contains(types, "mwinner") {
+				if strings.Contains(types, "s_winner.mwinner") {
 					winner := qutil.ObjToString(data["s_winner"])
 					keyWord = KeyWordToDatas(winner, key, keyWord)
 				}
-				if strings.Contains(types, "magency") {
+				if strings.Contains(types, "agency.magency") {
 					winner := qutil.ObjToString(data["agency"])
 					keyWord = KeyWordToDatas(winner, key, keyWord)
 				}

common/src/qfw/util/dataexport/dataexport.go

@@ -17,7 +17,7 @@ import (
 	qutil "app.yhyue.com/moapp/jybase/common"
 	. "app.yhyue.com/moapp/jybase/date"
 	. "app.yhyue.com/moapp/jybase/encrypt"
-	elastic "app.yhyue.com/moapp/jybase/esv1"
+	elastic "app.yhyue.com/moapp/jybase/es"
 	mg "app.yhyue.com/moapp/jybase/mongodb"
 	"app.yhyue.com/moapp/jypkg/common/src/qfw/util/jy"
 )
@@ -118,8 +118,8 @@ func getDataExportSql(scd *SieveCondition) string {
 	query_price := `{"bool":{"must":[{"range":{"bidamount":{%s}}}]}},{"bool":{"must":[{"range":{"budget":{%s}}}],"must_not":[{"range":{"bidamount":{"gte":-1}}}]}}`
 	query_bool_must := `{"terms":{"%s":[%s]}}`
 	query_bool_must_and := `{"bool":{"must":[%s]%s}}`
-	query_missing := `{"constant_score":{"filter":{"missing":{"field":"%s"}}}}`
-	query_bool_must_term := `{"bool": {"must": [{ "term": {"isValidFile": %d }}]}}`
+	query_exists := `{"constant_score":{"filter":{"exists":{"field":"%s"}}}}`
+	query_bool_must_term := `{"bool": {"must": [{ "term": {"isValidFile": %t }}]}}`
 	query_bool_must_term_bidField := `{"bool": {"must": [{ "term": {"bid_field": "%s" }}]}}` // 领域化数据类型
 
 	gte := `"gte": %s`
@@ -429,16 +429,16 @@ func getDataExportSql(scd *SieveCondition) string {
 	}
 	if scd.HasBuyerTel != "" {
 		if scd.HasBuyerTel == "y" {
-			must_not = append(must_not, fmt.Sprintf(query_missing, "buyertel"))
+			musts = append(musts, fmt.Sprintf(query_exists, "buyertel"))
 		} else {
-			musts = append(musts, fmt.Sprintf(query_missing, "buyertel"))
+			must_not = append(must_not, fmt.Sprintf(query_exists, "buyertel"))
 		}
 	}
 	if scd.HasWinnerTel != "" {
 		if scd.HasWinnerTel == "y" {
-			must_not = append(must_not, fmt.Sprintf(query_missing, "winnertel"))
+			musts = append(musts, fmt.Sprintf(query_exists, "winnertel"))
 		} else {
-			musts = append(musts, fmt.Sprintf(query_missing, "winnertel"))
+			must_not = append(must_not, fmt.Sprintf(query_exists, "winnertel"))
 		}
 	}
 
@@ -447,10 +447,9 @@ func getDataExportSql(scd *SieveCondition) string {
 	var isFileSearch = strings.ReplaceAll(selectType, ",", "\",\"") == "filetext"
 	if !isFileSearch && scd.FileExists != "" {
 		if scd.FileExists == "1" { //有附件
-			must_not = append(must_not, fmt.Sprintf(query_missing, "isValidFile"))
-			musts = append(musts, fmt.Sprintf(query_bool_must_term, 1))
+			musts = append(musts, fmt.Sprintf(query_bool_must_term, true))
 		} else if scd.FileExists == "-1" { //无附件
-			musts = append(musts, fmt.Sprintf(query_missing, "isValidFile"))
+			must_not = append(must_not, fmt.Sprintf(query_bool_must_term, true))
 		}
 	}
 	// 如果是领域化数据则需要加标签

common/src/qfw/util/dataexport/entdataexport.go

@@ -15,7 +15,7 @@ import (
 	util "app.yhyue.com/moapp/jybase/common"
 	. "app.yhyue.com/moapp/jybase/date"
 	. "app.yhyue.com/moapp/jybase/encrypt"
-	elastic "app.yhyue.com/moapp/jybase/esv1"
+	elastic "app.yhyue.com/moapp/jybase/es"
 	"app.yhyue.com/moapp/jybase/mongodb"
 	mg "app.yhyue.com/moapp/jybase/mongodb"
 	"app.yhyue.com/moapp/jybase/mysql"

common/src/qfw/util/jy/entnichepush.go

@@ -13,7 +13,7 @@ import (
 	. "app.yhyue.com/moapp/jybase/common"
 	. "app.yhyue.com/moapp/jybase/date"
 	. "app.yhyue.com/moapp/jybase/encrypt"
-	elastic "app.yhyue.com/moapp/jybase/esv1"
+	elastic "app.yhyue.com/moapp/jybase/es"
 	mg "app.yhyue.com/moapp/jybase/mongodb"
 	"app.yhyue.com/moapp/jybase/mysql"
 	"app.yhyue.com/moapp/jybase/redis"

common/src/qfw/util/jy/jy.go

@@ -1,22 +1,17 @@
 package jy
 
 import (
-	"crypto/rand"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
-	"math/big"
-	"net/http"
-	"net/url"
 	"regexp"
 	"sort"
 	"strings"
 	"time"
-	"unicode"
 
 	util "app.yhyue.com/moapp/jybase/common"
 	. "app.yhyue.com/moapp/jybase/date"
+	"app.yhyue.com/moapp/jybase/es"
 	"app.yhyue.com/moapp/jybase/go-xweb/httpsession"
 	"app.yhyue.com/moapp/jybase/mail"
 	. "app.yhyue.com/moapp/jybase/mongodb"
@@ -61,12 +56,13 @@ var filterReg = regexp.MustCompile("^[的人号时元万公告项目地址电话
 var PhoneReg = regexp.MustCompile("^[1][3-9][0-9]{9}$")
 var EmailPattern = regexp.MustCompile("^([a-zA-Z0-9_\\-\\.]+)@((\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.)|(([a-zA-Z0-9\\-]+\\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\\]?)$")
 
+//P279删除通用过滤词，例如“公告”
 func FilteKey(k string) string {
 	k = strings.TrimSpace(k)
-	k = filterReg_3.ReplaceAllString(k, "")
+	// k = filterReg_3.ReplaceAllString(k, "")
 	k = filterReg_2.ReplaceAllString(k, "")
-	k = filterReg_1.ReplaceAllString(k, "")
-	k = filterReg.ReplaceAllString(k, "")
+	// k = filterReg_1.ReplaceAllString(k, "")
+	// k = filterReg.ReplaceAllString(k, "")
 	return k
 }
 
@@ -97,64 +93,8 @@ func InterceptSearchKW(word string, keywordsLimit int, isFilter bool) (b_word, a
 	return
 }
 
-func HttpEs(ques, analyzer, esAddress string) (res string) {
-	var addrs []string
-	surl := ""
-	for _, s := range strings.Split(esAddress, ",") {
-		addrs = append(addrs, s)
-	}
-	i, _ := rand.Int(rand.Reader, big.NewInt(int64(len(addrs)))) //随机
-	surl = addrs[int(i.Int64())] + "/bidding/_analyze"
-	URL, _ := url.Parse(surl)
-	Q := URL.Query()
-	Q.Add("text", ques)
-	Q.Add("analyzer", analyzer)
-	URL.RawQuery = Q.Encode()
-	resp, err := http.Get(URL.String())
-	if err != nil {
-		log.Println("es连接失败 err1:", err)
-		resp, err = getesResp(ques, analyzer, addrs)
-		if err != nil {
-			return
-		}
-	}
-	result, err := ioutil.ReadAll(resp.Body)
-	if err == nil {
-		defer resp.Body.Close()
-		var resmap map[string]interface{}
-		json.Unmarshal(result, &resmap)
-		if resmap != nil && resmap["tokens"] != nil {
-			tokens := util.ObjArrToMapArr(resmap["tokens"].([]interface{}))
-			for _, v := range tokens {
-				token := util.ObjToString(v["token"])
-				if len([]rune(token)) == 1 && !unicode.Is(unicode.Scripts["Han"], []rune(token)[0]) { //(P260保留单个汉字)
-					continue
-				}
-				if res != "" {
-					res += "+"
-				}
-				res += token
-			}
-		}
-	}
-	return
-}
-
-//
-func getesResp(ques, analyzer string, addrs []string) (resp *http.Response, err error) {
-	for _, v := range addrs {
-		surl := v + "/bidding/_analyze"
-		URL, _ := url.Parse(surl)
-		Q := URL.Query()
-		Q.Add("text", ques)
-		Q.Add("analyzer", analyzer)
-		URL.RawQuery = Q.Encode()
-		resp, err = http.Get(URL.String())
-		if err == nil {
-			break
-		}
-	}
-	return resp, err
+func HttpEs(ques, analyzer, esAddress string) string {
+	return es.Analyze(ques, "bidding", analyzer)
 }
 
 //发送邮箱验证码

common/src/qfw/util/jy/payUser.go

@@ -68,11 +68,11 @@ func (vs *VipState) GetQueryItems(selectType string, limitOldTime int64) (items
 			if t == "content" {
 				items = append(items, "detail")
 			} else if t == "buyer" {
-				items = append(items, "mbuyer")
+				items = append(items, "buyer.mbuyer")
 			} else if t == "winner" {
-				items = append(items, "mwinner")
+				items = append(items, "s_winner.mwinner")
 			} else if t == "agency" {
-				items = append(items, "magency")
+				items = append(items, "agency.magency")
 			} else if t == "title" {
 				items = append(items, "title")
 			} else if t == "ppa" {
@@ -87,7 +87,7 @@ func (vs *VipState) GetQueryItems(selectType string, limitOldTime int64) (items
 	for _, t := range strings.Split(selectType, ",") {
 		if t == "winner" {
 			if isOldUser {
-				items = append(items, "mwinner")
+				items = append(items, "s_winner.mwinner")
 			}
 		} else if t == "title" {
 			items = append(items, "title")

common/src/qfw/util/jy/subscribepush.go

@@ -12,7 +12,7 @@ import (
 	. "app.yhyue.com/moapp/jybase/common"
 	. "app.yhyue.com/moapp/jybase/date"
 	. "app.yhyue.com/moapp/jybase/encrypt"
-	elastic "app.yhyue.com/moapp/jybase/esv1"
+	elastic "app.yhyue.com/moapp/jybase/es"
 	mg "app.yhyue.com/moapp/jybase/mongodb"
 	"app.yhyue.com/moapp/jybase/mysql"
 	"app.yhyue.com/moapp/jybase/redis"

ent/entity/entinfo.go

@@ -10,7 +10,7 @@ import (
 	util "app.yhyue.com/moapp/jybase/common"
 	. "app.yhyue.com/moapp/jybase/date"
 	"app.yhyue.com/moapp/jybase/encrypt"
-	elastic "app.yhyue.com/moapp/jybase/esv1"
+	elastic "app.yhyue.com/moapp/jybase/es"
 	. "app.yhyue.com/moapp/jypkg/ent/util"
 )
 

go.mod

@@ -4,7 +4,7 @@ go 1.18
 
 require (
 	app.yhyue.com/moapp/jyMarketing v0.0.2-0.20230304035551-21bb1eedf547
-	app.yhyue.com/moapp/jybase v0.0.0-20230117032034-ad7c00ffe11a
+	app.yhyue.com/moapp/jybase v0.0.0-20230517064222-e0bdfc8ee0e8
 	app.yhyue.com/moapp/message v0.0.0-20221223100203-6402e389d9ae
 	bp.jydev.jianyu360.cn/BaseService/entManageApplication v0.0.0-20230214091519-89a98c01ab0e
 	bp.jydev.jianyu360.cn/BaseService/powerCheckCenter v0.0.0-20230222052351-9d6fad062447

go.sum

@@ -4,8 +4,9 @@ app.yhyue.com/moapp/jyMarketing v0.0.2-0.20230304035551-21bb1eedf547 h1:cCmWQW8D
 app.yhyue.com/moapp/jyMarketing v0.0.2-0.20230304035551-21bb1eedf547/go.mod h1:JvIs8uKjdT963+7JnZGIEcL4ctBiBjwkoz0kNyigE78=
 app.yhyue.com/moapp/jyPoints v1.1.1/go.mod h1:SvP8p5L3jGrejHiH2LXfgCg/NPlFiKBC5Yd0gsI12FU=
 app.yhyue.com/moapp/jybase v0.0.0-20220427020729-974c1a148186/go.mod h1:qNRA0sHuYqcLoYoP8irpaWnW9YsXixe6obBIkwaXpD0=
-app.yhyue.com/moapp/jybase v0.0.0-20230117032034-ad7c00ffe11a h1:wD4aWPSYdiX1cIP4lzzPD2s7fYhKa3muIf97l9tonJE=
 app.yhyue.com/moapp/jybase v0.0.0-20230117032034-ad7c00ffe11a/go.mod h1:zB47XTeJvpcbtBRYgkQuxOICWNexiZfbUO+7aUf6mNs=
+app.yhyue.com/moapp/jybase v0.0.0-20230517064222-e0bdfc8ee0e8 h1:0ukFbYC7yRkFvRsr0bFCgV2SHsc1TvE+kW5MlttQEzE=
+app.yhyue.com/moapp/jybase v0.0.0-20230517064222-e0bdfc8ee0e8/go.mod h1:zB47XTeJvpcbtBRYgkQuxOICWNexiZfbUO+7aUf6mNs=
 app.yhyue.com/moapp/message v0.0.0-20221223100203-6402e389d9ae h1:6rDDaz6yxvE8viTSzEBwKYOFWq14TMfuBivSazUZMz4=
 app.yhyue.com/moapp/message v0.0.0-20221223100203-6402e389d9ae/go.mod h1:b0zZHev3gmJao1Fo+2Z2KPVjsuLOJVvVxf+kCnu9WkA=
 bp.jydev.jianyu360.cn/BaseService/entManageApplication v0.0.0-20230214091519-89a98c01ab0e h1:h+VEI3o1qC0jeCzkFGTrLI4f27cfa/W/y+0sXokWMgE=

jylabutil/entsearch.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"strings"
 
-	elastic "app.yhyue.com/moapp/jybase/esv1"
+	elastic "app.yhyue.com/moapp/jybase/es"
 )
 
 /**即时获取企业名称列表
@@ -49,8 +49,7 @@ func GetProjectByEntName(entname string, area string, lower, upper float64, fiel
 	}
 	order := `CUSTOM_"sort": {
 	    "_script": {
-	      	"script": "doc['jgtime']?doc['jgtime'].value:doc['zbtime'].value",
-	      	"lang": "groovy",
+	      	"script": "if(doc['jgtime'].size()>0){return doc['jgtime'].value}else if(doc['zbtime'].size()>0){return doc['zbtime'].value}else if(doc['lasttime'].size()>0){return doc['lasttime'].value}else{return doc['createtime'].value}",
 	      	"type": "number",
 	      	"order": "desc"
 	    }

public/db.go

@@ -1,13 +1,14 @@
 package public
 
 import (
+	"log"
+
 	util "app.yhyue.com/moapp/jybase/common"
-	elastic "app.yhyue.com/moapp/jybase/esv1"
+	elastic "app.yhyue.com/moapp/jybase/es"
 	es "app.yhyue.com/moapp/jybase/esv7"
 	m "app.yhyue.com/moapp/jybase/mongodb"
 	"app.yhyue.com/moapp/jybase/mysql"
 	"app.yhyue.com/moapp/jybase/redis"
-	"log"
 )
 
 var (
@@ -21,8 +22,6 @@ var (
 	Mgo_Log     m.MongodbSim
 	Mgo_Bidding m.MongodbSim
 	//Mgo_Qyfw    m.MongodbSim
-
-	EsDoc *elastic.Elastic
 )
 
 type dbConf struct {
@@ -60,6 +59,7 @@ type mgoConf struct {
 }
 
 type esConf struct {
+	Version  string
 	Address  string
 	Size     int
 	UserName string
@@ -81,7 +81,7 @@ func init() {
 	util.ReadConfig("./db.json", &DbConf)
 	if DbConf != nil {
 		if DbConf.Elasticsearch.Main != nil {
-			elastic.InitElasticSize(DbConf.Elasticsearch.Main.Address, DbConf.Elasticsearch.Main.Size)
+			elastic.NewEs(DbConf.Elasticsearch.Main.Version, DbConf.Elasticsearch.Main.Address, DbConf.Elasticsearch.Main.Size, DbConf.Elasticsearch.Main.UserName, DbConf.Elasticsearch.Main.Password)
 			log.Println("初始化 elasticsearch")
 		}
 

public/public.go

@@ -10,7 +10,7 @@ import (
 
 	util "app.yhyue.com/moapp/jybase/common"
 	. "app.yhyue.com/moapp/jybase/encrypt"
-	elastic "app.yhyue.com/moapp/jybase/esv1"
+	elastic "app.yhyue.com/moapp/jybase/es"
 	. "app.yhyue.com/moapp/jybase/mongodb"
 	"app.yhyue.com/moapp/jybase/redis"
 	"app.yhyue.com/moapp/jypkg/common/src/qfw/util/jy"