Jelajahi Sumber

wip:es7兼容开发

wangkaiyue 2 tahun lalu
induk
melakukan
762db56b34
7 mengubah file dengan 68 tambahan dan 54 penghapusan
  1. 5 2
      db.json
  2. 6 9
      entity/search/advancedProject.go
  3. 2 2
      entity/search/marketAnalysisEntity.go
  4. 5 1
      go.mod
  5. 6 2
      go.sum
  6. 36 33
      public/aggsSearchUtil.go
  7. 8 5
      public/db.go

+ 5 - 2
db.json

@@ -17,8 +17,11 @@
 	},
 	"elasticsearch": {
 		"main": {
-			"address": "http://192.168.3.206:9800",
-			"size": 5
+			"address": "http://192.168.3.241:9205,http://192.168.3.149:9200",
+			"size": 5,
+			"version": "v7",
+			"userName": "",
+			"password": ""
 		}
 	},
     "redis": {

+ 6 - 9
entity/search/advancedProject.go

@@ -74,25 +74,22 @@ func AdvancedProject(userid, keyWord string, t int64) map[string]interface{} {
 			}
 		}
 	}
-	resData := make([]map[string]interface{}, len(data))
+
 	for i, v := range data {
-		if json.Unmarshal(*v.Source, &resData[i]) != nil {
-			continue
-		}
 		var ss []string
 		for k := range sMap {
 			//匹配高亮
-			if strings.Contains(qutil.InterfaceToStr(resData[i]["title"]), k) {
+			if strings.Contains(qutil.InterfaceToStr(v["title"]), k) {
 				ss = append(ss, k)
 			}
 		}
-		resData[i]["keyWord"] = ss
+		data[i]["keyWord"] = ss
 		//id加密
-		resData[i]["_id"] = public.EncodeId(qutil.InterfaceToStr(resData[i]["_id"]))
+		data[i]["_id"] = public.EncodeId(qutil.InterfaceToStr(v["_id"]))
 	}
-	log.Println("[SEARCH-INFO]AdvancedProject top2 :", resData)
+	log.Println("[SEARCH-INFO]AdvancedProject top2 :", data)
 	return map[string]interface{}{
-		"projectTop2":  resData,
+		"projectTop2":  data,
 		"subTypeCount": redisData,
 	}
 }

+ 2 - 2
entity/search/marketAnalysisEntity.go

@@ -9,9 +9,9 @@ const (
 	projectsSortSource = `,"sort": [{"publishtime":"desc"}],"_source": ["_id","title"]`
 
 	//采购单位分布
-	buyer_procurement_scale = `"project_amount":{"sum":{"field":"sortprice"}},"buyer_amount_distribution": {"terms": {"field": "buyer","size": 0},"aggs": {"amount": {"sum": {"field": "sortprice"}}}}`
+	buyer_procurement_scale = `"project_amount":{"sum":{"field":"sortprice"}},"buyer_amount_distribution": {"terms": {"field": "buyer"},"aggs": {"amount": {"sum": {"field": "sortprice"}}}}`
 	//中标单位分布
-	winner_procurement_scale = `"winner_amount_distribution": {"terms": {"field": "entidlist","size": 0},"aggs": {"amount": {"sum": {"field": "sortprice"}}}}`
+	winner_procurement_scale = `"winner_amount_distribution": {"terms": {"field": "entidlist"},"aggs": {"amount": {"sum": {"field": "sortprice"}}}}`
 
 	localMultiMatch     = `{"multi_match": {"query": %s,"type": "phrase", "fields": [%s]}}`
 	query_bool_must_and = `{"bool": {"must": [%s]%s}}`

+ 5 - 1
go.mod

@@ -4,7 +4,7 @@ go 1.17
 
 require (
 	app.yhyue.com/moapp/esv1 v0.0.0-20220414031211-3da4123e648d
-	app.yhyue.com/moapp/jybase v0.0.0-20221010080805-39dc6a853eff
+	app.yhyue.com/moapp/jybase v0.0.0-20230517064222-e0bdfc8ee0e8
 	github.com/robfig/cron/v3 v3.0.1
 )
 
@@ -14,11 +14,15 @@ require (
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/gomodule/redigo v1.8.9 // indirect
 	github.com/howeyc/fsnotify v0.9.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/olivere/elastic v6.2.37+incompatible // indirect
+	github.com/olivere/elastic/v7 v7.0.22 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.0.2 // indirect

+ 6 - 2
go.sum

@@ -1,7 +1,7 @@
 app.yhyue.com/moapp/esv1 v0.0.0-20220414031211-3da4123e648d h1:WPsYuuptAd3UEgN+jPzpnsDe/OvcshDUUtOTZPYGSJ8=
 app.yhyue.com/moapp/esv1 v0.0.0-20220414031211-3da4123e648d/go.mod h1:91/lSD/hS+ckMVP3WdidRzDhC60lLMdyce9QHy0cSMA=
-app.yhyue.com/moapp/jybase v0.0.0-20221010080805-39dc6a853eff h1:uYssVU5ODQHRdRpUt8jSVuJPFdbxQ+0G3CH6I7seRwU=
-app.yhyue.com/moapp/jybase v0.0.0-20221010080805-39dc6a853eff/go.mod h1:HelrO6tcD9TcKb/HOP2BLbzppyDz2kpQSFhPMQTUgbQ=
+app.yhyue.com/moapp/jybase v0.0.0-20230517064222-e0bdfc8ee0e8 h1:0ukFbYC7yRkFvRsr0bFCgV2SHsc1TvE+kW5MlttQEzE=
+app.yhyue.com/moapp/jybase v0.0.0-20230517064222-e0bdfc8ee0e8/go.mod h1:zB47XTeJvpcbtBRYgkQuxOICWNexiZfbUO+7aUf6mNs=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -114,6 +114,7 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -186,6 +187,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
@@ -316,6 +319,7 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/olivere/elastic v6.2.37+incompatible h1:UfSGJem5czY+x/LqxgeCBgjDn6St+z8OnsCuxwD3L0U=
 github.com/olivere/elastic v6.2.37+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
+github.com/olivere/elastic/v7 v7.0.22 h1:esBA6JJwvYgfms0EVlH7Z+9J4oQ/WUADF2y/nCNDw7s=
 github.com/olivere/elastic/v7 v7.0.22/go.mod h1:VDexNy9NjmtAkrjNoI7tImv7FR4tf5zUA3ickqu5Pc8=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

+ 36 - 33
public/aggsSearchUtil.go

@@ -1,45 +1,48 @@
 package public
 
 import (
-	elastic2 "app.yhyue.com/moapp/esv1/gopkg.in/olivere/elastic.v1"
-	util "app.yhyue.com/moapp/jybase/common"
 	"app.yhyue.com/moapp/jybase/encrypt"
-	elastic "app.yhyue.com/moapp/jybase/esv1"
-	"log"
-	"runtime"
+	elastic "app.yhyue.com/moapp/jybase/es"
+	esV7 "github.com/olivere/elastic/v7"
 )
 
 // GetAggs 聚合查询
-func GetAggs(index, itype, query string) (aggs elastic2.Aggregations, count int64, hist []*elastic2.SearchHit) {
-	defer util.Catch()
-	client := elastic.GetEsConn()
-	defer func() {
-		go elastic.DestoryEsConn(client)
-	}()
-	if client != nil {
-		defer func() {
-			if r := recover(); r != nil {
-				log.Println("[SEARCH-ERR]", r)
-				for skip := 1; ; skip++ {
-					_, file, line, ok := runtime.Caller(skip)
-					if !ok {
-						break
-					}
-					go log.Printf("[SEARCH-INFO]%v,%v\n", file, line)
-				}
-			}
-		}()
-		searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
-		if err != nil {
-			log.Println("[SEARCH-ERR]从ES查询出错", err.Error())
-		}
-		count = searchResult.Hits.TotalHits
-		hist = searchResult.Hits.Hits
-		aggs = searchResult.Aggregations
-	}
-	return
+func GetAggs(index, itype, query string) (aggs esV7.Aggregations, count int64, res []map[string]interface{}) {
+	v1, c, v2 := elastic.GetAggs(index, itype, query)
+	return v1, c, v2
 }
 
+// GetAggs 聚合查询
+//func GetAggs_old(index, itype, query string) (aggs elastic2.Aggregations, count int64, hist []*elastic2.SearchHit) {
+//	defer util.Catch()
+//	client := elastic.GetEsConn()
+//	defer func() {
+//		go elastic.DestoryEsConn(client)
+//	}()
+//	if client != nil {
+//		defer func() {
+//			if r := recover(); r != nil {
+//				log.Println("[SEARCH-ERR]", r)
+//				for skip := 1; ; skip++ {
+//					_, file, line, ok := runtime.Caller(skip)
+//					if !ok {
+//						break
+//					}
+//					go log.Printf("[SEARCH-INFO]%v,%v\n", file, line)
+//				}
+//			}
+//		}()
+//		searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
+//		if err != nil {
+//			log.Println("[SEARCH-ERR]从ES查询出错", err.Error())
+//		}
+//		count = searchResult.Hits.TotalHits
+//		hist = searchResult.Hits.Hits
+//		aggs = searchResult.Aggregations
+//	}
+//	return
+//}
+
 // EncodeId 加密
 func EncodeId(sid string) string {
 	if sid == "" {

+ 8 - 5
public/db.go

@@ -2,7 +2,7 @@ package public
 
 import (
 	util "app.yhyue.com/moapp/jybase/common"
-	elastic "app.yhyue.com/moapp/jybase/esv1"
+	elastic "app.yhyue.com/moapp/jybase/es"
 	m "app.yhyue.com/moapp/jybase/mongodb"
 	"app.yhyue.com/moapp/jybase/mysql"
 	"app.yhyue.com/moapp/jybase/redis"
@@ -48,8 +48,11 @@ type mgoConf struct {
 }
 
 type esConf struct {
-	Address string
-	Size    int
+	Version  string
+	Address  string
+	Size     int
+	UserName string
+	Password string
 }
 type redisConf struct {
 	Address string
@@ -67,8 +70,8 @@ func init() {
 	util.ReadConfig("./db.json", &DbConf)
 	if DbConf != nil {
 		if DbConf.Elasticsearch.Main != nil {
-			elastic.InitElasticSize(DbConf.Elasticsearch.Main.Address, DbConf.Elasticsearch.Main.Size)
-			log.Println("[SYSTEM]初始化 elasticsearch")
+			elastic.NewEs(DbConf.Elasticsearch.Main.Version, DbConf.Elasticsearch.Main.Address, DbConf.Elasticsearch.Main.Size, DbConf.Elasticsearch.Main.UserName, DbConf.Elasticsearch.Main.Password)
+			log.Println("初始化 elasticsearch")
 		}
 		//初始化redis
 		if DbConf.Redis.Main != nil {