|
@@ -16,7 +16,7 @@ import (
|
|
es "github.com/olivere/elastic/v7"
|
|
es "github.com/olivere/elastic/v7"
|
|
)
|
|
)
|
|
|
|
|
|
-//检索库服务地址
|
|
|
|
|
|
+// 检索库服务地址
|
|
var (
|
|
var (
|
|
addrs []string
|
|
addrs []string
|
|
LocCity = map[string]string{}
|
|
LocCity = map[string]string{}
|
|
@@ -36,7 +36,7 @@ var ntimeout int
|
|
var syncPool sync.Pool
|
|
var syncPool sync.Pool
|
|
var filterReg = regexp.MustCompile(`,\s*"should"\s*:\s*\[\s*\]\s*,\s*"minimum_should_match"\s*:\s*1`)
|
|
var filterReg = regexp.MustCompile(`,\s*"should"\s*:\s*\[\s*\]\s*,\s*"minimum_should_match"\s*:\s*1`)
|
|
|
|
|
|
-//初始化全文检索
|
|
|
|
|
|
+// 初始化全文检索
|
|
func InitElastic(addr string) {
|
|
func InitElastic(addr string) {
|
|
InitElasticSize(addr, SIZE)
|
|
InitElasticSize(addr, SIZE)
|
|
}
|
|
}
|
|
@@ -61,12 +61,12 @@ var httpclient = &http.Client{Transport: &http.Transport{
|
|
//var op = es.SetHttpClient(httpclient)
|
|
//var op = es.SetHttpClient(httpclient)
|
|
var poolsize = int32(20)
|
|
var poolsize = int32(20)
|
|
|
|
|
|
-//n倍的池
|
|
|
|
|
|
+// n倍的池
|
|
func InitElasticSize(addr string, size int) {
|
|
func InitElasticSize(addr string, size int) {
|
|
InitElasticSizeByAuth(addr, size, "", "")
|
|
InitElasticSizeByAuth(addr, size, "", "")
|
|
}
|
|
}
|
|
|
|
|
|
-//初始化es,带有用户名密码认证
|
|
|
|
|
|
+// 初始化es,带有用户名密码认证
|
|
func InitElasticSizeByAuth(addr string, size int, u, p string) {
|
|
func InitElasticSizeByAuth(addr string, size int, u, p string) {
|
|
poolsize = int32(3 * size)
|
|
poolsize = int32(3 * size)
|
|
pool = make(chan *es.Client, poolsize)
|
|
pool = make(chan *es.Client, poolsize)
|
|
@@ -88,7 +88,7 @@ func newClient() (*es.Client, error) {
|
|
return es.NewClient(opt...)
|
|
return es.NewClient(opt...)
|
|
}
|
|
}
|
|
|
|
|
|
-//关闭连接
|
|
|
|
|
|
+// 关闭连接
|
|
func DestoryEsConn(client *es.Client) {
|
|
func DestoryEsConn(client *es.Client) {
|
|
select {
|
|
select {
|
|
case pool <- client:
|
|
case pool <- client:
|
|
@@ -106,7 +106,7 @@ var (
|
|
lastTimeLock = &sync.Mutex{}
|
|
lastTimeLock = &sync.Mutex{}
|
|
)
|
|
)
|
|
|
|
|
|
-//获取连接
|
|
|
|
|
|
+// 获取连接
|
|
func GetEsConn() *es.Client {
|
|
func GetEsConn() *es.Client {
|
|
select {
|
|
select {
|
|
case c := <-pool:
|
|
case c := <-pool:
|
|
@@ -141,7 +141,7 @@ func GetEsConn() *es.Client {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
-//保存对象
|
|
|
|
|
|
+// 保存对象
|
|
func Save(index, itype string, obj interface{}) bool {
|
|
func Save(index, itype string, obj interface{}) bool {
|
|
client := GetEsConn()
|
|
client := GetEsConn()
|
|
defer DestoryEsConn(client)
|
|
defer DestoryEsConn(client)
|
|
@@ -168,23 +168,48 @@ func Save(index, itype string, obj interface{}) bool {
|
|
} else {
|
|
} else {
|
|
return true
|
|
return true
|
|
}
|
|
}
|
|
|
|
+}
|
|
|
|
|
|
|
|
+func SaveNew(index string, obj interface{}) bool {
|
|
|
|
+ client := GetEsConn()
|
|
|
|
+ defer DestoryEsConn(client)
|
|
|
|
+ defer func() {
|
|
|
|
+ if r := recover(); r != nil {
|
|
|
|
+ log.Println("[E]", r)
|
|
|
|
+ for skip := 1; ; skip++ {
|
|
|
|
+ _, file, line, ok := runtime.Caller(skip)
|
|
|
|
+ if !ok {
|
|
|
|
+ break
|
|
|
|
+ }
|
|
|
|
+ go log.Printf("%v,%v\n", file, line)
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }()
|
|
|
|
+ data := objToMap(obj)
|
|
|
|
+ _id := fmt.Sprint((*data)["id"])
|
|
|
|
+ _, err := client.Index().Index(index).Id(_id).BodyJson(data).Do(context.TODO())
|
|
|
|
+ if err != nil {
|
|
|
|
+ log.Println("保存到ES出错", err.Error(), obj)
|
|
|
|
+ return false
|
|
|
|
+ } else {
|
|
|
|
+ return true
|
|
|
|
+ }
|
|
}
|
|
}
|
|
|
|
|
|
-//通用查询
|
|
|
|
-//{"query": {"bool":{"must":[{"query_string":{"default_field":"name","query":"*"}}]}}}
|
|
|
|
-//{"query":{"bool":{"must":{"match":{"content":{"query":"fulltextsearch","operator":"and"}}},"should":[{"match":{"content":{"query":"Elasticsearch","boost":3}}},{"match":{"content":{"query":"Lucene","boost":2}}}]}}}
|
|
|
|
-//prefix
|
|
|
|
-//{"query":{"match":{"title":{"query":"brownfox","operator":"and"}}}} //默认为or
|
|
|
|
-//{"query":{"multi_match":{"query":"PolandStreetW1V","type":"most_fields","fields":["*_street","city^2","country","postcode"]}}}
|
|
|
|
-//{"query":{"wildcard":{"postcode":"W?F*HW"}}}
|
|
|
|
-//{"query":{"regexp":{"postcode":"W[0-9].+"}}}
|
|
|
|
-//{"query":{"filtered":{"filter":{"range":{"price":{"gte":10000}}}}},"aggs":{"single_avg_price":{"avg":{"field":"price"}}}}
|
|
|
|
-//{"query":{"match":{"make":"ford"}},"aggs":{"colors":{"terms":{"field":"color"}}}}//查fork有几种颜色
|
|
|
|
-//过滤器不会计算相关度的得分,所以它们在计算上更快一些
|
|
|
|
-//{"query":{"filtered":{"query":{"match_all":{}},"filter":{"range":{"balance":{"gte":20000,"lte":30000}}}}}}
|
|
|
|
-//{"query":{"match_all":{}},"from":10,"size":10,"_source":["account_number","balance"],"sort":{"balance":{"order":"desc"}}}
|
|
|
|
-//{"query":{"match_phrase":{"address":"milllane"}}}和match不同会去匹配整个短语,相当于must[]
|
|
|
|
|
|
+// 通用查询
|
|
|
|
+// {"query": {"bool":{"must":[{"query_string":{"default_field":"name","query":"*"}}]}}}
|
|
|
|
+// {"query":{"bool":{"must":{"match":{"content":{"query":"fulltextsearch","operator":"and"}}},"should":[{"match":{"content":{"query":"Elasticsearch","boost":3}}},{"match":{"content":{"query":"Lucene","boost":2}}}]}}}
|
|
|
|
+// prefix
|
|
|
|
+// {"query":{"match":{"title":{"query":"brownfox","operator":"and"}}}} //默认为or
|
|
|
|
+// {"query":{"multi_match":{"query":"PolandStreetW1V","type":"most_fields","fields":["*_street","city^2","country","postcode"]}}}
|
|
|
|
+// {"query":{"wildcard":{"postcode":"W?F*HW"}}}
|
|
|
|
+// {"query":{"regexp":{"postcode":"W[0-9].+"}}}
|
|
|
|
+// {"query":{"filtered":{"filter":{"range":{"price":{"gte":10000}}}}},"aggs":{"single_avg_price":{"avg":{"field":"price"}}}}
|
|
|
|
+// {"query":{"match":{"make":"ford"}},"aggs":{"colors":{"terms":{"field":"color"}}}}//查fork有几种颜色
|
|
|
|
+// 过滤器不会计算相关度的得分,所以它们在计算上更快一些
|
|
|
|
+// {"query":{"filtered":{"query":{"match_all":{}},"filter":{"range":{"balance":{"gte":20000,"lte":30000}}}}}}
|
|
|
|
+// {"query":{"match_all":{}},"from":10,"size":10,"_source":["account_number","balance"],"sort":{"balance":{"order":"desc"}}}
|
|
|
|
+// {"query":{"match_phrase":{"address":"milllane"}}}和match不同会去匹配整个短语,相当于must[]
|
|
func GetBySearchType(index, searchType, query string) (int64, *[]map[string]interface{}) {
|
|
func GetBySearchType(index, searchType, query string) (int64, *[]map[string]interface{}) {
|
|
t, _, l := get(index, "", searchType, query, true, true)
|
|
t, _, l := get(index, "", searchType, query, true, true)
|
|
return t, l
|
|
return t, l
|
|
@@ -260,14 +285,14 @@ func GetNoLimit(index, itype, query string) *[]map[string]interface{} {
|
|
return l
|
|
return l
|
|
}
|
|
}
|
|
|
|
|
|
-//分页查询
|
|
|
|
-//{"name":"张三","$and":[{"age":{"$gt":10}},{"age":{"$lte":20}}]}
|
|
|
|
-//fields直接是 `"_id","title"`
|
|
|
|
|
|
+// 分页查询
|
|
|
|
+// {"name":"张三","$and":[{"age":{"$gt":10}},{"age":{"$lte":20}}]}
|
|
|
|
+// fields直接是 `"_id","title"`
|
|
func GetPage(index, itype, query, order, field string, start, limit int) *[]map[string]interface{} {
|
|
func GetPage(index, itype, query, order, field string, start, limit int) *[]map[string]interface{} {
|
|
return Get(index, itype, MakeQuery(query, order, field, start, limit))
|
|
return Get(index, itype, MakeQuery(query, order, field, start, limit))
|
|
}
|
|
}
|
|
|
|
|
|
-//openapi
|
|
|
|
|
|
+// openapi
|
|
func GetOAPage(index, itype, query, order, field string, start, limit int) (*[]map[string]interface{}, int) {
|
|
func GetOAPage(index, itype, query, order, field string, start, limit int) (*[]map[string]interface{}, int) {
|
|
return GetOA(index, itype, MakeQuery(query, order, field, start, limit))
|
|
return GetOA(index, itype, MakeQuery(query, order, field, start, limit))
|
|
}
|
|
}
|
|
@@ -294,7 +319,7 @@ func MakeQuery(query, order, fileds string, start, limit int) string {
|
|
return ""
|
|
return ""
|
|
}
|
|
}
|
|
|
|
|
|
-//{"name":"aaa"}
|
|
|
|
|
|
+// {"name":"aaa"}
|
|
func AnalyQuery(query interface{}, parent string, result string) string {
|
|
func AnalyQuery(query interface{}, parent string, result string) string {
|
|
m := make(map[string]interface{})
|
|
m := make(map[string]interface{})
|
|
if q1, ok := query.(string); ok {
|
|
if q1, ok := query.(string); ok {
|
|
@@ -416,7 +441,7 @@ func GetByIdField(index, itype, id, fields string) *map[string]interface{} {
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
-//根据id来查询文档
|
|
|
|
|
|
+// 根据id来查询文档
|
|
func GetById(index, itype string, ids ...string) *[]map[string]interface{} {
|
|
func GetById(index, itype string, ids ...string) *[]map[string]interface{} {
|
|
client := GetEsConn()
|
|
client := GetEsConn()
|
|
defer DestoryEsConn(client)
|
|
defer DestoryEsConn(client)
|
|
@@ -460,7 +485,7 @@ func GetById(index, itype string, ids ...string) *[]map[string]interface{} {
|
|
return &res
|
|
return &res
|
|
}
|
|
}
|
|
|
|
|
|
-//根据语句更新对象
|
|
|
|
|
|
+// 根据语句更新对象
|
|
func Update(index, itype, id string, updateStr string) bool {
|
|
func Update(index, itype, id string, updateStr string) bool {
|
|
client := GetEsConn()
|
|
client := GetEsConn()
|
|
defer DestoryEsConn(client)
|
|
defer DestoryEsConn(client)
|
|
@@ -545,7 +570,7 @@ func NewBulkUpdate(index string, params ...[]string) bool {
|
|
return false
|
|
return false
|
|
}
|
|
}
|
|
|
|
|
|
-//根据id删除索引对象
|
|
|
|
|
|
+// 根据id删除索引对象
|
|
func DelById(index, itype, id string) bool {
|
|
func DelById(index, itype, id string) bool {
|
|
client := GetEsConn()
|
|
client := GetEsConn()
|
|
defer DestoryEsConn(client)
|
|
defer DestoryEsConn(client)
|
|
@@ -574,7 +599,7 @@ func DelById(index, itype, id string) bool {
|
|
return b
|
|
return b
|
|
}
|
|
}
|
|
|
|
|
|
-//先删除后增
|
|
|
|
|
|
+// 先删除后增
|
|
func UpdateNewDoc(index, itype string, obj ...interface{}) bool {
|
|
func UpdateNewDoc(index, itype string, obj ...interface{}) bool {
|
|
client := GetEsConn()
|
|
client := GetEsConn()
|
|
defer DestoryEsConn(client)
|
|
defer DestoryEsConn(client)
|
|
@@ -616,6 +641,45 @@ func UpdateNewDoc(index, itype string, obj ...interface{}) bool {
|
|
return b
|
|
return b
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+func UpdateNew(index string, obj ...interface{}) bool {
|
|
|
|
+ client := GetEsConn()
|
|
|
|
+ defer DestoryEsConn(client)
|
|
|
|
+ b := false
|
|
|
|
+ if client != nil {
|
|
|
|
+ defer func() {
|
|
|
|
+ if r := recover(); r != nil {
|
|
|
|
+ log.Println("[E]", r)
|
|
|
|
+ for skip := 1; ; skip++ {
|
|
|
|
+ _, file, line, ok := runtime.Caller(skip)
|
|
|
|
+ if !ok {
|
|
|
|
+ break
|
|
|
|
+ }
|
|
|
|
+ go log.Printf("%v,%v\n", file, line)
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }()
|
|
|
|
+ var err error
|
|
|
|
+ for _, v := range obj {
|
|
|
|
+ tempObj := objToMap(v)
|
|
|
|
+ if tempObj == nil || len(*tempObj) == 0 {
|
|
|
|
+ continue
|
|
|
|
+ }
|
|
|
|
+ _id := fmt.Sprint((*tempObj)["id"])
|
|
|
|
+ if _id != "" {
|
|
|
|
+ client.Delete().Index(index).Id(_id).Do(context.TODO())
|
|
|
|
+ }
|
|
|
|
+ _, err = client.Index().Index(index).Id(_id).BodyJson(tempObj).Do(context.TODO())
|
|
|
|
+ if err != nil {
|
|
|
|
+ log.Println("保存到ES出错", err.Error())
|
|
|
|
+ } else {
|
|
|
|
+ b = true
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ }
|
|
|
|
+ return b
|
|
|
|
+}
|
|
|
|
+
|
|
func BulkSave(index, itype string, obj *[]map[string]interface{}, isDelBefore bool) {
|
|
func BulkSave(index, itype string, obj *[]map[string]interface{}, isDelBefore bool) {
|
|
client := GetEsConn()
|
|
client := GetEsConn()
|
|
defer DestoryEsConn(client)
|
|
defer DestoryEsConn(client)
|
|
@@ -801,7 +865,7 @@ const (
|
|
HL_IK = `"highlight": {"pre_tags": ["` + IK_pre_tags + `"],"post_tags": ["` + IK_post_tags + `"],"fields": {%s}}`
|
|
HL_IK = `"highlight": {"pre_tags": ["` + IK_pre_tags + `"],"post_tags": ["` + IK_post_tags + `"],"fields": {%s}}`
|
|
)
|
|
)
|
|
|
|
|
|
-//替换了"号
|
|
|
|
|
|
+// 替换了"号
|
|
func GetNgramQuery(query interface{}, mustquery, findfields string) (qstr string) {
|
|
func GetNgramQuery(query interface{}, mustquery, findfields string) (qstr string) {
|
|
var words []string
|
|
var words []string
|
|
if q, ok := query.(string); ok {
|
|
if q, ok := query.(string); ok {
|
|
@@ -894,7 +958,7 @@ func GetByNgram(index, itype string, query interface{}, mustquery, findfields, o
|
|
return GetByNgramAll(index, itype, query, mustquery, findfields, order, fields, start, limit, false, false)
|
|
return GetByNgramAll(index, itype, query, mustquery, findfields, order, fields, start, limit, false, false)
|
|
}
|
|
}
|
|
|
|
|
|
-//增加高亮、过滤查询、高亮截取字数
|
|
|
|
|
|
+// 增加高亮、过滤查询、高亮截取字数
|
|
func GetByNgramOther(index, itype string, query interface{}, mustquery, findfields, order, fields string, start, limit int, highlight bool, filtermode bool, count int) *[]map[string]interface{} {
|
|
func GetByNgramOther(index, itype string, query interface{}, mustquery, findfields, order, fields string, start, limit int, highlight bool, filtermode bool, count int) *[]map[string]interface{} {
|
|
defer catch()
|
|
defer catch()
|
|
qstr := ""
|
|
qstr := ""
|
|
@@ -928,8 +992,8 @@ func GetByNgramOther(index, itype string, query interface{}, mustquery, findfiel
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
-//增加高亮、过滤查询
|
|
|
|
-//替换了"号
|
|
|
|
|
|
+// 增加高亮、过滤查询
|
|
|
|
+// 替换了"号
|
|
func GetByNgramAll(index, itype string, query interface{}, mustquery, findfields, order, fields string, start, limit int, highlight bool, filtermode bool) *[]map[string]interface{} {
|
|
func GetByNgramAll(index, itype string, query interface{}, mustquery, findfields, order, fields string, start, limit int, highlight bool, filtermode bool) *[]map[string]interface{} {
|
|
defer catch()
|
|
defer catch()
|
|
qstr := ""
|
|
qstr := ""
|
|
@@ -965,7 +1029,7 @@ func GetByNgramAll(index, itype string, query interface{}, mustquery, findfields
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
-//增加高亮、过滤查询
|
|
|
|
|
|
+// 增加高亮、过滤查询
|
|
func GetByNgramAll_New(index, itype string, querystring, querymust interface{}, mustquery, findfields, order, fields string, start, limit int, highlight bool, filtermode bool) *[]map[string]interface{} {
|
|
func GetByNgramAll_New(index, itype string, querystring, querymust interface{}, mustquery, findfields, order, fields string, start, limit int, highlight bool, filtermode bool) *[]map[string]interface{} {
|
|
defer catch()
|
|
defer catch()
|
|
qstr := ""
|
|
qstr := ""
|
|
@@ -1005,7 +1069,7 @@ type KeyConfig struct {
|
|
Areas []string `json:"area"`
|
|
Areas []string `json:"area"`
|
|
}
|
|
}
|
|
|
|
|
|
-//替换了"号
|
|
|
|
|
|
+// 替换了"号
|
|
func GetResForJY(index, itype string, keys []KeyConfig, allquery, findfields, SortQuery, fields string, start, limit int) *[]map[string]interface{} {
|
|
func GetResForJY(index, itype string, keys []KeyConfig, allquery, findfields, SortQuery, fields string, start, limit int) *[]map[string]interface{} {
|
|
if len(keys) > 0 {
|
|
if len(keys) > 0 {
|
|
qstr := ""
|
|
qstr := ""
|
|
@@ -1063,7 +1127,6 @@ func ReplaceYH(src string) (rpl string) {
|
|
return strings.Replace(src, `"`, `\"`, -1)
|
|
return strings.Replace(src, `"`, `\"`, -1)
|
|
}
|
|
}
|
|
|
|
|
|
-//
|
|
|
|
func GetAllByNgram(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
|
|
func GetAllByNgram(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
|
|
if qstr != "" {
|
|
if qstr != "" {
|
|
if highlight {
|
|
if highlight {
|
|
@@ -1089,7 +1152,7 @@ func GetAllByNgram(index, itype, qstr, findfields, order, fields string, start,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
-//数据标记2019-07-10
|
|
|
|
|
|
+// 数据标记2019-07-10
|
|
func GetAllByNgram_MP(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
|
|
func GetAllByNgram_MP(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
|
|
if qstr != "" {
|
|
if qstr != "" {
|
|
if highlight {
|
|
if highlight {
|
|
@@ -1115,7 +1178,7 @@ func GetAllByNgram_MP(index, itype, qstr, findfields, order, fields string, star
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
-//ik 分词
|
|
|
|
|
|
+// ik 分词
|
|
func GetAllByIk(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
|
|
func GetAllByIk(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
|
|
if qstr != "" {
|
|
if qstr != "" {
|
|
if highlight {
|
|
if highlight {
|