elasticutil.go 40 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427
  1. package elastic
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "log"
  6. "net/url"
  7. "reflect"
  8. "runtime"
  9. "strconv"
  10. "strings"
  11. "sync"
  12. "time"
  13. es "app.yhyue.com/moapp/esv1/gopkg.in/olivere/elastic.v1"
  14. )
  15. //检索库服务地址
  16. var addrs []string
  17. var LocCity = map[string]string{}
  18. var SIZE = 30
  19. const (
  20. QStr = `{"query":{"bool":{"must":[$and],"must_not":[],
  21. "should":[$or],"minimum_should_match" : 1}}}`
  22. )
  23. var pool chan *es.Client
  24. var ntimeout int
  25. var syncPool sync.Pool
  26. //初始化全文检索
  27. func InitElastic(addr string) {
  28. InitElasticSize(addr, SIZE)
  29. }
  30. //自定义HttpClient
  31. /**
  32. var httpclient = &http.Client{Transport: &http.Transport{
  33. Dial: func(netw, addr string) (net.Conn, error) {
  34. deadline := time.Now().Add(5000 * time.Millisecond)
  35. c, err := net.DialTimeout(netw, addr, 10000*time.Millisecond)
  36. if err != nil {
  37. return nil, err
  38. }
  39. tcp_conn := c.(*net.TCPConn)
  40. tcp_conn.SetKeepAlive(false)
  41. tcp_conn.SetDeadline(deadline)
  42. return tcp_conn, nil
  43. },
  44. DisableKeepAlives: true, //不保持,这样才能释放
  45. }}
  46. **/
  47. //var op = es.SetHttpClient(httpclient)
  48. var poolsize = int32(20)
  49. //n倍的池
  50. func InitElasticSize(addr string, size int) {
  51. poolsize = int32(3 * size)
  52. pool = make(chan *es.Client, poolsize)
  53. for _, s := range strings.Split(addr, ",") {
  54. addrs = append(addrs, s)
  55. }
  56. for i := 0; i < size; i++ {
  57. client, _ := es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false))
  58. pool <- client
  59. }
  60. }
  61. //关闭连接
  62. func DestoryEsConn(client *es.Client) {
  63. select {
  64. case pool <- client:
  65. break
  66. case <-time.After(time.Second * 1):
  67. if client != nil {
  68. client.Stop()
  69. }
  70. client = nil
  71. }
  72. }
  73. var (
  74. lastTime = int64(0)
  75. lastTimeLock = &sync.Mutex{}
  76. )
  77. //获取连接
  78. func GetEsConn() *es.Client {
  79. select {
  80. case c := <-pool:
  81. if c == nil || !c.IsRunning() {
  82. log.Println("new esclient.", len(pool))
  83. client, err := es.NewClient(es.SetURL(addrs...),
  84. es.SetMaxRetries(2), es.SetSniff(false))
  85. if err == nil && client.IsRunning() {
  86. return client
  87. }
  88. }
  89. return c
  90. case <-time.After(time.Second * 4):
  91. //超时
  92. ntimeout++
  93. lastTimeLock.Lock()
  94. defer lastTimeLock.Unlock()
  95. //12秒后允许创建链接
  96. c := time.Now().Unix() - lastTime
  97. if c > 12 {
  98. lastTime = time.Now().Unix()
  99. log.Println("add client..", len(pool))
  100. c, _ := es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false))
  101. go func() {
  102. for i := 0; i < 2; i++ {
  103. client, _ := es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false))
  104. pool <- client
  105. }
  106. }()
  107. return c
  108. }
  109. return nil
  110. }
  111. }
  112. //保存对象
  113. func Save(index, itype string, obj interface{}) bool {
  114. client := GetEsConn()
  115. defer DestoryEsConn(client)
  116. defer func() {
  117. if r := recover(); r != nil {
  118. log.Println("[E]", r)
  119. for skip := 1; ; skip++ {
  120. _, file, line, ok := runtime.Caller(skip)
  121. if !ok {
  122. break
  123. }
  124. go log.Printf("%v,%v\n", file, line)
  125. }
  126. }
  127. }()
  128. _, err := client.Index().Index(index).Type(itype).BodyJson(objToMap(obj)).Do()
  129. if err != nil {
  130. log.Println("保存到ES出错", err.Error(), obj)
  131. return false
  132. } else {
  133. return true
  134. }
  135. }
  136. //通用查询
  137. //{"query": {"bool":{"must":[{"query_string":{"default_field":"name","query":"*"}}]}}}
  138. //{"query":{"bool":{"must":{"match":{"content":{"query":"fulltextsearch","operator":"and"}}},"should":[{"match":{"content":{"query":"Elasticsearch","boost":3}}},{"match":{"content":{"query":"Lucene","boost":2}}}]}}}
  139. //prefix
  140. //{"query":{"match":{"title":{"query":"brownfox","operator":"and"}}}} //默认为or
  141. //{"query":{"multi_match":{"query":"PolandStreetW1V","type":"most_fields","fields":["*_street","city^2","country","postcode"]}}}
  142. //{"query":{"wildcard":{"postcode":"W?F*HW"}}}
  143. //{"query":{"regexp":{"postcode":"W[0-9].+"}}}
  144. //{"query":{"filtered":{"filter":{"range":{"price":{"gte":10000}}}}},"aggs":{"single_avg_price":{"avg":{"field":"price"}}}}
  145. //{"query":{"match":{"make":"ford"}},"aggs":{"colors":{"terms":{"field":"color"}}}}//查fork有几种颜色
  146. //过滤器不会计算相关度的得分,所以它们在计算上更快一些
  147. //{"query":{"filtered":{"query":{"match_all":{}},"filter":{"range":{"balance":{"gte":20000,"lte":30000}}}}}}
  148. //{"query":{"match_all":{}},"from":10,"size":10,"_source":["account_number","balance"],"sort":{"balance":{"order":"desc"}}}
  149. //{"query":{"match_phrase":{"address":"milllane"}}}和match不同会去匹配整个短语,相当于must[]
  150. func Get(index, itype, query string) *[]map[string]interface{} {
  151. //log.Println("query -- ", query)
  152. client := GetEsConn()
  153. defer func() {
  154. go DestoryEsConn(client)
  155. }()
  156. var res []map[string]interface{}
  157. if client != nil {
  158. defer func() {
  159. if r := recover(); r != nil {
  160. log.Println("[E]", r)
  161. for skip := 1; ; skip++ {
  162. _, file, line, ok := runtime.Caller(skip)
  163. if !ok {
  164. break
  165. }
  166. go log.Printf("%v,%v\n", file, line)
  167. }
  168. }
  169. }()
  170. searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
  171. if err != nil {
  172. log.Println("从ES查询出错", err.Error())
  173. return nil
  174. }
  175. if searchResult.Hits != nil {
  176. resNum := len(searchResult.Hits.Hits)
  177. if resNum < 5000 {
  178. res = make([]map[string]interface{}, resNum)
  179. for i, hit := range searchResult.Hits.Hits {
  180. //d := json.NewDecoder(bytes.NewBuffer(*hit.Source))
  181. //d.UseNumber()
  182. //d.Decode(&res[i])
  183. parseErr := json.Unmarshal(*hit.Source, &res[i])
  184. if parseErr == nil && hit.Highlight != nil && res[i] != nil {
  185. res[i]["highlight"] = map[string][]string(hit.Highlight)
  186. }
  187. }
  188. } else {
  189. log.Println("查询结果太多,查询到:", resNum, "条")
  190. }
  191. }
  192. }
  193. return &res
  194. }
  195. func GetOA(index, itype, query string) (*[]map[string]interface{}, int) {
  196. //log.Println("query -- ", query)
  197. client := GetEsConn()
  198. defer func() {
  199. go DestoryEsConn(client)
  200. }()
  201. var res []map[string]interface{}
  202. var resNum int
  203. if client != nil {
  204. defer func() {
  205. if r := recover(); r != nil {
  206. log.Println("[E]", r)
  207. for skip := 1; ; skip++ {
  208. _, file, line, ok := runtime.Caller(skip)
  209. if !ok {
  210. break
  211. }
  212. go log.Printf("%v,%v\n", file, line)
  213. }
  214. }
  215. }()
  216. searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
  217. if err != nil {
  218. log.Println("从ES查询出错", err.Error())
  219. return nil, 0
  220. }
  221. if searchResult.Hits != nil {
  222. resNum = len(searchResult.Hits.Hits)
  223. if resNum < 5000 {
  224. res = make([]map[string]interface{}, resNum)
  225. for i, hit := range searchResult.Hits.Hits {
  226. //d := json.NewDecoder(bytes.NewBuffer(*hit.Source))
  227. //d.UseNumber()
  228. //d.Decode(&res[i])
  229. parseErr := json.Unmarshal(*hit.Source, &res[i])
  230. if parseErr == nil && hit.Highlight != nil && res[i] != nil {
  231. res[i]["highlight"] = map[string][]string(hit.Highlight)
  232. }
  233. }
  234. } else {
  235. log.Println("查询结果太多,查询到:", resNum, "条")
  236. }
  237. }
  238. }
  239. return &res, resNum
  240. }
  241. func GetNoLimit(index, itype, query string) *[]map[string]interface{} {
  242. //log.Println("query -- ", query)
  243. client := GetEsConn()
  244. defer DestoryEsConn(client)
  245. var res []map[string]interface{}
  246. if client != nil {
  247. defer func() {
  248. if r := recover(); r != nil {
  249. log.Println("[E]", r)
  250. for skip := 1; ; skip++ {
  251. _, file, line, ok := runtime.Caller(skip)
  252. if !ok {
  253. break
  254. }
  255. go log.Printf("%v,%v\n", file, line)
  256. }
  257. }
  258. }()
  259. searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
  260. if err != nil {
  261. log.Println("从ES查询出错", err.Error())
  262. return nil
  263. }
  264. if searchResult.Hits != nil {
  265. resNum := len(searchResult.Hits.Hits)
  266. res = make([]map[string]interface{}, resNum)
  267. for i, hit := range searchResult.Hits.Hits {
  268. json.Unmarshal(*hit.Source, &res[i])
  269. }
  270. }
  271. }
  272. return &res
  273. }
  274. //分页查询
  275. //{"name":"张三","$and":[{"age":{"$gt":10}},{"age":{"$lte":20}}]}
  276. //fields直接是 `"_id","title"`
  277. func GetPage(index, itype, query, order, field string, start, limit int) *[]map[string]interface{} {
  278. return Get(index, itype, MakeQuery(query, order, field, start, limit))
  279. }
  280. //openapi
  281. func GetOAPage(index, itype, query, order, field string, start, limit int) (*[]map[string]interface{}, int) {
  282. return GetOA(index, itype, MakeQuery(query, order, field, start, limit))
  283. }
  284. var SR = strings.Replace
  285. func MakeQuery(query, order, fileds string, start, limit int) string {
  286. res := AnalyQuery(query, "", QStr)
  287. if len(res) > 10 {
  288. if strings.Contains(res, "###剑鱼###") {
  289. res = strings.ReplaceAll(res, "###剑鱼###", "\\\"")
  290. }
  291. res = SR(SR(SR(SR(res, ",$and", "", -1), "$and", "", -1), ",$or", "", -1), "$or", "", -1)
  292. if len(fileds) > 0 {
  293. //"_source":["account_number","balance"]
  294. res = res[:len(res)-1] + `,"_source":[` + fileds + "]}"
  295. }
  296. //{"name":-1,"age":1}
  297. if len(order) > 0 {
  298. res = res[:len(res)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
  299. }
  300. if start > -1 {
  301. res = res[:len(res)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
  302. }
  303. return res
  304. }
  305. return ""
  306. }
  307. //{"name":"aaa"}
  308. func AnalyQuery(query interface{}, parent string, result string) string {
  309. m := make(map[string]interface{})
  310. if q1, ok := query.(string); ok {
  311. json.Unmarshal([]byte(q1), &m)
  312. } else if q2, ok2 := query.(map[string]interface{}); ok2 {
  313. m = q2
  314. }
  315. if len(parent) == 0 {
  316. for k, v := range m {
  317. if k == "$and" || k == "$or" {
  318. temps := ""
  319. if map1, ok := v.([]interface{}); ok {
  320. for i := 0; i < len(map1); i++ {
  321. temps += "," + AnalyQuery(map1[i], k, "")
  322. }
  323. }
  324. if len(temps) > 0 {
  325. temps = temps[1:]
  326. }
  327. result = SR(result, k, temps+","+k, 1)
  328. } else {
  329. switch reflect.TypeOf(v).String() {
  330. case "string":
  331. if strings.Index(k, "TERM_") == 0 {
  332. result = SR(result, "$and", `{"term":{"`+SR(k, "TERM_", "", 1)+`":"`+fmt.Sprintf("%v", v)+`"}},$and`, 1)
  333. } else {
  334. result = SR(result, "$and", `{"query_string":{"default_field":"`+k+`","query":"`+fmt.Sprintf("%v", v)+`"}},$and`, 1)
  335. }
  336. case "int", "int8", "int32", "int64", "float32", "float64":
  337. if strings.Index(k, "TERM_") == 0 {
  338. result = SR(result, "$and", `{"term":{"`+SR(k, "TERM_", "", 1)+`":`+fmt.Sprintf("%v", v)+`}},$and`, 1)
  339. } else {
  340. result = SR(result, "$and", `{"query_string":{"default_field":"`+k+`","query":`+fmt.Sprintf("%v", v)+`}},$and`, 1)
  341. }
  342. default:
  343. result = SR(result, "$and", AnalyQuery(v, k, "")+",$and", 1)
  344. }
  345. }
  346. }
  347. return result
  348. } else {
  349. for k, v := range m {
  350. if k == "$in" {
  351. s := ""
  352. if map1, ok := v.([]interface{}); ok {
  353. for i := 0; i < len(map1); i++ {
  354. s += "," + `"` + fmt.Sprintf("%v", map1[i]) + `"`
  355. }
  356. }
  357. if len(s) > 0 {
  358. s = s[1:]
  359. }
  360. return `{"terms":{"` + parent + `":[` + s + `]}}`
  361. } else if strings.Contains(k, "$lt") || strings.Contains(k, "$gt") {
  362. return `{"range":{"` + parent + `":{"` + SR(k, "$", "", 1) + `":` + fmt.Sprintf("%v", v) + `}}}`
  363. } else {
  364. switch reflect.TypeOf(v).String() {
  365. case "string":
  366. if strings.Index(k, "TERM_") == 0 {
  367. return `{"term":{"` + SR(k, "TERM_", "", 1) + `":"` + fmt.Sprintf("%v", v) + `"}}`
  368. } else {
  369. return `{"query_string":{"default_field":"` + k + `","query":"` + fmt.Sprintf("%v", v) + `"}}`
  370. }
  371. case "int", "int8", "int32", "int64", "float32", "float64":
  372. if strings.Index(k, "TERM_") == 0 {
  373. return `{"term":{"` + SR(k, "TERM_", "", 1) + `":` + fmt.Sprintf("%v", v) + `}}`
  374. } else {
  375. return `{"query_string":{"default_field":"` + k + `","query":` + fmt.Sprintf("%v", v) + `}}`
  376. }
  377. default:
  378. return AnalyQuery(v, k, result)
  379. }
  380. }
  381. }
  382. }
  383. return result
  384. }
  385. func GetByIdField(index, itype, id, fields string) *map[string]interface{} {
  386. client := GetEsConn()
  387. defer DestoryEsConn(client)
  388. if client != nil {
  389. defer func() {
  390. if r := recover(); r != nil {
  391. log.Println("[E]", r)
  392. for skip := 1; ; skip++ {
  393. _, file, line, ok := runtime.Caller(skip)
  394. if !ok {
  395. break
  396. }
  397. go log.Printf("%v,%v\n", file, line)
  398. }
  399. }
  400. }()
  401. query := `{"query":{"term":{"_id":"` + id + `"}}`
  402. if len(fields) > 0 {
  403. query = query + `,"_source":[` + fields + `]`
  404. }
  405. query = query + "}"
  406. searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
  407. if err != nil {
  408. log.Println("从ES查询出错", err.Error())
  409. return nil
  410. }
  411. var res map[string]interface{}
  412. if searchResult.Hits != nil {
  413. resNum := len(searchResult.Hits.Hits)
  414. if resNum == 1 {
  415. res = make(map[string]interface{})
  416. for _, hit := range searchResult.Hits.Hits {
  417. json.Unmarshal(*hit.Source, &res)
  418. }
  419. return &res
  420. }
  421. }
  422. }
  423. return nil
  424. }
  425. //根据id来查询文档
  426. func GetById(index, itype string, ids ...string) *[]map[string]interface{} {
  427. client := GetEsConn()
  428. defer DestoryEsConn(client)
  429. var res []map[string]interface{}
  430. if client != nil {
  431. defer func() {
  432. if r := recover(); r != nil {
  433. log.Println("[E]", r)
  434. for skip := 1; ; skip++ {
  435. _, file, line, ok := runtime.Caller(skip)
  436. if !ok {
  437. break
  438. }
  439. go log.Printf("%v,%v\n", file, line)
  440. }
  441. }
  442. }()
  443. query := es.NewIdsQuery().Ids(ids...)
  444. searchResult, err := client.Search().Index(index).Type(itype).Query(&query).Do()
  445. if err != nil {
  446. log.Println("从ES查询出错", err.Error())
  447. return nil
  448. }
  449. if searchResult.Hits != nil {
  450. resNum := len(searchResult.Hits.Hits)
  451. if resNum < 5000 {
  452. res = make([]map[string]interface{}, resNum)
  453. for i, hit := range searchResult.Hits.Hits {
  454. json.Unmarshal(*hit.Source, &res[i])
  455. }
  456. } else {
  457. log.Println("查询结果太多,查询到:", resNum, "条")
  458. }
  459. }
  460. }
  461. return &res
  462. }
  463. //删除某个索引,根据查询
  464. func Del(index, itype string, query interface{}) bool {
  465. client := GetEsConn()
  466. defer DestoryEsConn(client)
  467. b := false
  468. if client != nil {
  469. defer func() {
  470. if r := recover(); r != nil {
  471. log.Println("[E]", r)
  472. for skip := 1; ; skip++ {
  473. _, file, line, ok := runtime.Caller(skip)
  474. if !ok {
  475. break
  476. }
  477. go log.Printf("%v,%v\n", file, line)
  478. }
  479. }
  480. }()
  481. var err error
  482. if qs, ok := query.(string); ok {
  483. temp := es.BoolQuery{
  484. QueryStrings: qs,
  485. }
  486. _, err = client.DeleteByQuery().Index(index).Type(itype).Query(temp).Do()
  487. } else if qi, ok2 := query.(es.Query); ok2 {
  488. _, err = client.DeleteByQuery().Index(index).Type(itype).Query(qi).Do()
  489. }
  490. if err != nil {
  491. log.Println("删除索引出错:", err.Error())
  492. } else {
  493. b = true
  494. }
  495. }
  496. return b
  497. }
  498. //根据语句更新对象
  499. func Update(index, itype, id string, updateStr string) bool {
  500. client := GetEsConn()
  501. defer DestoryEsConn(client)
  502. b := false
  503. if client != nil {
  504. defer func() {
  505. if r := recover(); r != nil {
  506. log.Println("[E]", r)
  507. for skip := 1; ; skip++ {
  508. _, file, line, ok := runtime.Caller(skip)
  509. if !ok {
  510. break
  511. }
  512. go log.Printf("%v,%v\n", file, line)
  513. }
  514. }
  515. }()
  516. var err error
  517. _, err = client.Update().Index(index).Type(itype).Id(id).Script(updateStr).ScriptLang("groovy").Do()
  518. if err != nil {
  519. log.Println("更新检索出错:", err.Error())
  520. } else {
  521. b = true
  522. }
  523. }
  524. return b
  525. }
  526. func BulkUpdate(index, itype string, ids []string, updateStr string) {
  527. client := GetEsConn()
  528. defer DestoryEsConn(client)
  529. if client != nil {
  530. defer func() {
  531. if r := recover(); r != nil {
  532. log.Println("[E]", r)
  533. for skip := 1; ; skip++ {
  534. _, file, line, ok := runtime.Caller(skip)
  535. if !ok {
  536. break
  537. }
  538. go log.Printf("%v,%v\n", file, line)
  539. }
  540. }
  541. }()
  542. for _, id := range ids {
  543. _, err := client.Update().Index(index).Type(itype).Id(id).Script(updateStr).ScriptLang("groovy").Do()
  544. if err != nil {
  545. log.Println("更新检索出错:", err.Error())
  546. }
  547. }
  548. }
  549. }
  550. //根据id删除索引对象
  551. func DelById(index, itype, id string) bool {
  552. client := GetEsConn()
  553. defer DestoryEsConn(client)
  554. b := false
  555. if client != nil {
  556. defer func() {
  557. if r := recover(); r != nil {
  558. log.Println("[E]", r)
  559. for skip := 1; ; skip++ {
  560. _, file, line, ok := runtime.Caller(skip)
  561. if !ok {
  562. break
  563. }
  564. go log.Printf("%v,%v\n", file, line)
  565. }
  566. }
  567. }()
  568. var err error
  569. _, err = client.Delete().Index(index).Type(itype).Id(id).Do()
  570. if err != nil {
  571. log.Println("更新检索出错:", err.Error())
  572. } else {
  573. b = true
  574. }
  575. }
  576. return b
  577. }
  578. //先删除后增
  579. func UpdateNewDoc(index, itype string, obj ...interface{}) bool {
  580. client := GetEsConn()
  581. defer DestoryEsConn(client)
  582. b := false
  583. if client != nil {
  584. defer func() {
  585. if r := recover(); r != nil {
  586. log.Println("[E]", r)
  587. for skip := 1; ; skip++ {
  588. _, file, line, ok := runtime.Caller(skip)
  589. if !ok {
  590. break
  591. }
  592. go log.Printf("%v,%v\n", file, line)
  593. }
  594. }
  595. }()
  596. var err error
  597. for _, v := range obj {
  598. tempObj := objToMap(v)
  599. id := fmt.Sprintf("%v", (*tempObj)["_id"])
  600. client.Delete().Index(index).Type(itype).Id(id).Do()
  601. _, err = client.Index().Index(index).Type(itype).BodyJson(tempObj).Do()
  602. if err != nil {
  603. log.Println("保存到ES出错", err.Error())
  604. } else {
  605. b = true
  606. }
  607. }
  608. }
  609. return b
  610. }
  611. //把地市代码转为地市
  612. func getLoc(code string, res *map[string]string) (loc string) {
  613. switch len(code) {
  614. case 6:
  615. loc = (*res)[code[:2]] + " " + (*res)[code[:4]] + " " + (*res)[code]
  616. break
  617. case 4:
  618. loc = (*res)[code[:2]] + " " + (*res)[code]
  619. break
  620. case 2:
  621. loc = (*res)[code]
  622. break
  623. }
  624. return
  625. }
  626. func ConverData(ent *map[string]interface{}) map[string]interface{} {
  627. tmp := *ent
  628. id64, _ := tmp["ID"].(int64)
  629. ids := fmt.Sprintf("%d", id64)
  630. tmp2 := make(map[string]interface{})
  631. tmp2["ID"] = ids
  632. tmp2["_id"] = tmp["_id"]
  633. tmp2["Area"] = tmp["Area"]
  634. tmp2["LeRep"] = tmp["LeRep"]
  635. tmp2["RegNo"] = tmp["RegNo"]
  636. tmp2["EntType"] = tmp["EntType"]
  637. tmp2["EntName"] = tmp["EntName"]
  638. tmp2["EntTypeName"] = tmp["EntTypeName"]
  639. tmp2["Dom"] = tmp["Dom"]
  640. tmp2["EstDate"] = tmp["EstDate"]
  641. tmp2["OpStateName"] = tmp["OpStateName"]
  642. tmp2["OpScope"] = tmp["OpScope"]
  643. tmp2["OpState"] = tmp["OpState"]
  644. tmp2["s_submitid"] = tmp["s_submitid"]
  645. tmp2["l_submittime"] = tmp["l_submittime"]
  646. tmp2["s_submitname"] = tmp["s_submitname"]
  647. tmp2["RegCapCurName"] = tmp["RegCapCurName"]
  648. //增加营业状态排序
  649. if tmp2["OpState"] == "06" {
  650. tmp2["OpSint"] = true
  651. } else {
  652. tmp2["OpSint"] = false
  653. }
  654. tmp2["OpLocDistrict"] = tmp["OpLocDistrict"]
  655. //增加代码转名称
  656. tmpLoc, _ := tmp["OpLocDistrict"].(string)
  657. tmp2["OpLocDistrictName"] = getLoc(tmpLoc, &LocCity)
  658. tmp2["RecCap"] = tmp["RecCap"]
  659. tmp2["RegCap"] = tmp["RegCap"]
  660. tmp2["IndustryPhy"] = tmp["IndustryPhy"]
  661. tmp2["IndustryPhyName"] = tmp["IndustryPhyName"]
  662. tmp2["RegOrg"] = tmp["RegOrg"]
  663. tmp2["RegOrgName"] = tmp["RegOrgName"]
  664. tmp2["Tel"] = tmp["Tel"]
  665. tmp2["CompForm"] = tmp["CompForm"]
  666. tmp2["CompFormName"] = tmp["CompFormName"]
  667. //增加异常名录标记 Ycml可能是bool也可能是string
  668. Ycmlb, _ := tmp["Ycml"].(bool)
  669. Ycmls, _ := tmp["Ycml"].(string)
  670. if Ycmlb || Ycmls == "1" {
  671. tmp2["Ycml"] = true
  672. } else {
  673. tmp2["Ycml"] = false
  674. }
  675. //增加年报联系信息
  676. if tmp["Nb_email"] != nil {
  677. tmp2["Nb_email"] = tmp["Nb_email"]
  678. }
  679. if tmp["Nb_tel"] != nil {
  680. tmp2["Nb_tel"] = tmp["Nb_tel"]
  681. }
  682. if tmp["Nb_addr"] != nil {
  683. tmp2["Nb_addr"] = tmp["Nb_addr"]
  684. }
  685. s_synopsis := tmp["s_synopsis"]
  686. if s_synopsis == nil {
  687. s_synopsis = ""
  688. }
  689. tmp2["s_synopsis"] = s_synopsis //企业简介
  690. //股东
  691. stock := getStock(tmp["investor"])
  692. tmp2["stock"] = stock
  693. tmp2["LegCerNO"] = tmp["LegCerNO"]
  694. if tmp["s_microwebsite"] != nil {
  695. tmp2["s_microwebsite"] = tmp["s_microwebsite"]
  696. }
  697. tmp2["SourceType"] = tmp["SourceType"] //数据来源
  698. s_servicenames := tmp["s_servicenames"]
  699. if s_servicenames == nil {
  700. s_servicenames = ""
  701. }
  702. tmp2["s_servicenames"] = s_servicenames //服务名称
  703. s_action := tmp["s_action"]
  704. if s_action == nil {
  705. s_action = "N"
  706. }
  707. tmp2["s_action"] = s_action
  708. tmp2["s_persion"] = tmp["s_persion"]
  709. tmp2["s_mobile"] = tmp["s_mobile"]
  710. tmp2["s_enturl"] = tmp["s_enturl"]
  711. tmp2["s_weixin"] = tmp["s_weixin"]
  712. tmp2["s_avatar"] = tmp["s_avatar"]
  713. return tmp2
  714. }
  715. func getStock(obj interface{}) string {
  716. stock := ""
  717. if ns, ok := obj.([]interface{}); ok {
  718. stock = " "
  719. for _, ns1 := range ns {
  720. if nn, ok1 := ns1.(map[string]interface{}); ok1 {
  721. tmp := fmt.Sprintf("%s", nn["Inv"])
  722. if strings.Index(stock, tmp) < 0 {
  723. stock += tmp + " "
  724. }
  725. }
  726. }
  727. }
  728. return stock
  729. }
  730. func BulkSave(index, itype string, obj *[]map[string]interface{}, isDelBefore bool) {
  731. client := GetEsConn()
  732. defer DestoryEsConn(client)
  733. if client != nil {
  734. defer func() {
  735. if r := recover(); r != nil {
  736. log.Println("[E]", r)
  737. for skip := 1; ; skip++ {
  738. _, file, line, ok := runtime.Caller(skip)
  739. if !ok {
  740. break
  741. }
  742. go log.Printf("%v,%v\n", file, line)
  743. }
  744. }
  745. }()
  746. req := client.Bulk()
  747. for _, v := range *obj {
  748. if isDelBefore {
  749. req = req.Add(es.NewBulkDeleteRequest().Index(index).Type(itype).Id(fmt.Sprintf("%v", v["_id"])))
  750. }
  751. req = req.Add(es.NewBulkIndexRequest().Index(index).Type(itype).Doc(v))
  752. }
  753. _, err := req.Do()
  754. if err != nil {
  755. log.Println("批量保存到ES出错", err.Error())
  756. }
  757. }
  758. }
  759. func Count(index, itype string, query interface{}) int64 {
  760. client := GetEsConn()
  761. defer DestoryEsConn(client)
  762. if client != nil {
  763. defer func() {
  764. if r := recover(); r != nil {
  765. log.Println("[E]", r)
  766. for skip := 1; ; skip++ {
  767. _, file, line, ok := runtime.Caller(skip)
  768. if !ok {
  769. break
  770. }
  771. go log.Printf("%v,%v\n", file, line)
  772. }
  773. }
  774. }()
  775. var qq es.Query
  776. if qs, ok := query.(string); ok {
  777. temp := es.BoolQuery{
  778. QueryStrings: qs,
  779. }
  780. qq = temp
  781. } else if qi, ok2 := query.(es.Query); ok2 {
  782. qq = qi
  783. }
  784. n, err := client.Count(index).Type(itype).Query(qq).Do()
  785. if err != nil {
  786. log.Println("统计出错", err.Error())
  787. }
  788. return n
  789. }
  790. return 0
  791. }
  792. //ngram精确查询
  793. /*
  794. {
  795. "query": {
  796. "bool": {
  797. "should": [
  798. {
  799. "bool":{
  800. "must":[
  801. { "multi_match": {
  802. "query": "智能",
  803. "type": "phrase",
  804. "fields": [
  805. "title"
  806. ],
  807. "analyzer": "my_ngram"
  808. }
  809. },{
  810. "multi_match": {
  811. "query": "机器",
  812. "type": "phrase",
  813. "fields": [
  814. "title"
  815. ],
  816. "analyzer": "my_ngram"
  817. }
  818. },{
  819. "multi_match": {
  820. "query": "2016",
  821. "type": "phrase",
  822. "fields": [
  823. "title"
  824. ],
  825. "analyzer": "my_ngram"
  826. }
  827. }
  828. ]
  829. }
  830. },
  831. {
  832. "bool":{
  833. "must":[
  834. { "multi_match": {
  835. "query": "河南",
  836. "type": "phrase",
  837. "fields": [
  838. "title"
  839. ],
  840. "analyzer": "my_ngram"
  841. }
  842. },{
  843. "multi_match": {
  844. "query": "工商",
  845. "type": "phrase",
  846. "fields": [
  847. "title"
  848. ],
  849. "analyzer": "my_ngram"
  850. }
  851. },{
  852. "multi_match": {
  853. "query": "2016",
  854. "type": "phrase",
  855. "fields": [
  856. "title"
  857. ],
  858. "analyzer": "my_ngram"
  859. }
  860. }
  861. ]
  862. }
  863. }
  864. ],"minimum_should_match": 1
  865. }
  866. },
  867. "_source": [
  868. "_id",
  869. "title"
  870. ],
  871. "from": 0,
  872. "size": 10,
  873. "sort": [{
  874. "publishtime": "desc"
  875. }]
  876. }
  877. */
  878. //"2016+智能+办公,"河南+工商"
  879. //["2016+智能+办公","河南+工商"]
  880. //QStr = `{"query":{"bool":{should":[$or],"minimum_should_match" : 1}}}`
  881. //{"bool":{"must":[]}}
  882. //{"multi_match": {"query": "$word","type": "phrase", "fields": [$field],"analyzer": "my_ngram"}}
  883. //"highlight": {"pre_tags": [""],"post_tags": [""],"fields": {"detail": {"fragment_size": 1,"number_of_fragments": 1},"title": {"fragment_size": 1,"number_of_fragments": 1}}}
  884. const (
  885. //此处最后少一个},正好NgramStr取[1:]多一个}
  886. FilterQuery = `{"query": {"filtered": {"filter": {"bool": {"must": [%s]}},%s}}`
  887. NgramStr = `{"query":{"bool":{"must":[%s],"should":[%s],"minimum_should_match": 1}}}`
  888. NgramMust = `{"bool":{"must":[%s]}}`
  889. NgramMustAndNot = `{"bool":{"must":[%s],"must_not":[%s]}}`
  890. minq = `{"multi_match": {"query": "%s","type": "phrase", "fields": [%s]}}`
  891. HL = `"highlight": {"pre_tags": [""],"post_tags": [""],"fields": {%s}}`
  892. highlightStr = `%s: {"fragment_size": %d,"number_of_fragments": 1}`
  893. FilterQuery_New = `{"query":{"bool":{"must": [%s%s%s],"should":[]}}}`
  894. MatchQueryString = `{"match": {%s: { "query":"%s", "operator": "and"}}}`
  895. HL_New = `"highlight": {"pre_tags": ["<HL>"],"post_tags": ["<HL>"],"fields": {%s}}`
  896. //数据查询高亮标记2019-07-10
  897. HL_MP = `"highlight": {"pre_tags": ["<HL>"],"post_tags": ["</HL>"],"fields": {%s}}`
  898. ik_highlightStr = `%s: {"fragment_size": %d,"number_of_fragments": 1,"require_field_match": true}`
  899. IK_pre_tags = `<font class=\"es-highlight\">`
  900. IK_post_tags = `</font>`
  901. HL_IK = `"highlight": {"pre_tags": ["` + IK_pre_tags + `"],"post_tags": ["` + IK_post_tags + `"],"fields": {%s}}`
  902. )
  903. //替换了"号
  904. func GetNgramQuery(query interface{}, mustquery, findfields string) (qstr string) {
  905. var words []string
  906. if q, ok := query.(string); ok {
  907. if q != "" {
  908. words = strings.Split(q, ",")
  909. }
  910. } else if q, ok := query.([]string); ok {
  911. words = q
  912. } else if q, ok := query.([]interface{}); ok {
  913. words = objArrToStringArr(q)
  914. }
  915. if words != nil {
  916. new_minq := fmt.Sprintf(minq, "%s", findfields)
  917. musts := []string{}
  918. for _, qs_words := range words {
  919. qws := strings.Split(qs_words, "+")
  920. mq := []string{}
  921. for _, qs_word := range qws {
  922. mq = append(mq, fmt.Sprintf(new_minq, ReplaceYH(qs_word)))
  923. }
  924. musts = append(musts, fmt.Sprintf(NgramMust, strings.Join(mq, ",")))
  925. }
  926. qstr = fmt.Sprintf(NgramStr, mustquery, strings.Join(musts, ","))
  927. //log.Println("ngram-query", qstr)
  928. } else {
  929. qstr = fmt.Sprintf(NgramStr, mustquery, "")
  930. }
  931. return
  932. }
  933. func GetNgramQuery_New(querystring, querymust interface{}, must, findfields string) (qstring string) {
  934. querymust_string := ""
  935. var wordsMust []string
  936. if q, ok := querymust.(string); ok {
  937. if q != "" {
  938. wordsMust = strings.Split(q, ",")
  939. }
  940. } else if q, ok := querymust.([]string); ok {
  941. wordsMust = q
  942. } else if q, ok := querymust.([]interface{}); ok {
  943. wordsMust = objArrToStringArr(q)
  944. }
  945. if wordsMust != nil {
  946. new_minq := fmt.Sprintf(minq, "%s", findfields)
  947. musts := []string{}
  948. for _, qs_wordsMust := range wordsMust {
  949. qws := strings.Split(qs_wordsMust, "+")
  950. mq := []string{}
  951. for _, qs_word := range qws {
  952. mq = append(mq, fmt.Sprintf(new_minq, qs_word))
  953. }
  954. musts = append(musts, fmt.Sprintf(NgramMust, strings.Join(mq, ",")))
  955. }
  956. querymust_string = strings.Join(musts, ",")
  957. }
  958. //log.Println("must", must, querymust_string)
  959. //querystring---------------------------------------------
  960. query_string := ""
  961. var querysShold []string
  962. if q, ok := querystring.(string); ok {
  963. if q != "" {
  964. querysShold = strings.Split(q, ",")
  965. }
  966. } else if q, ok := querystring.([]string); ok {
  967. querysShold = q
  968. } else if q, ok := querystring.([]interface{}); ok {
  969. querysShold = objArrToStringArr(q)
  970. }
  971. if querysShold != nil {
  972. for k, name := range strings.Split(findfields, ",") {
  973. for _, qs_querysShold := range querysShold {
  974. if k > 0 {
  975. query_string = query_string + "," + fmt.Sprintf(MatchQueryString, fmt.Sprint(name), qs_querysShold)
  976. } else {
  977. query_string = query_string + fmt.Sprintf(MatchQueryString, fmt.Sprint(name), qs_querysShold)
  978. }
  979. }
  980. }
  981. }
  982. //log.Println("querystring", query_string)
  983. if querymust_string == "" {
  984. qstring = fmt.Sprintf(FilterQuery_New, must, query_string, querymust_string)
  985. } else {
  986. qstring = fmt.Sprintf(FilterQuery_New, must, query_string, ","+querymust_string)
  987. }
  988. return
  989. }
  990. func GetByNgram(index, itype string, query interface{}, mustquery, findfields, order, fields string, start, limit int) *[]map[string]interface{} {
  991. return GetByNgramAll(index, itype, query, mustquery, findfields, order, fields, start, limit, false, false)
  992. }
  993. //增加高亮、过滤查询、高亮截取字数
  994. func GetByNgramOther(index, itype string, query interface{}, mustquery, findfields, order, fields string, start, limit int, highlight bool, filtermode bool, count int) *[]map[string]interface{} {
  995. defer catch()
  996. qstr := ""
  997. if mustquery != "" && filtermode {
  998. qstr = GetNgramQuery(query, "", findfields)
  999. qstr = fmt.Sprintf(FilterQuery, mustquery, qstr[1:])
  1000. } else {
  1001. qstr = GetNgramQuery(query, mustquery, findfields)
  1002. }
  1003. if qstr != "" {
  1004. if highlight {
  1005. ws := []string{}
  1006. for _, w := range strings.Split(findfields, ",") {
  1007. ws = append(ws, fmt.Sprintf(highlightStr, w, count))
  1008. }
  1009. qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL, strings.Join(ws, ",")) + `}`
  1010. }
  1011. if len(fields) > 0 {
  1012. qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
  1013. }
  1014. if len(order) > 0 {
  1015. qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
  1016. }
  1017. if start > -1 {
  1018. qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
  1019. }
  1020. //log.Println("ngram-find", qstr)
  1021. return Get(index, itype, qstr)
  1022. } else {
  1023. return nil
  1024. }
  1025. }
  1026. //增加高亮、过滤查询
  1027. //替换了"号
  1028. func GetByNgramAll(index, itype string, query interface{}, mustquery, findfields, order, fields string, start, limit int, highlight bool, filtermode bool) *[]map[string]interface{} {
  1029. defer catch()
  1030. qstr := ""
  1031. if mustquery != "" && filtermode {
  1032. qstr = GetNgramQuery(query, "", findfields)
  1033. qstr = fmt.Sprintf(FilterQuery, mustquery, qstr[1:])
  1034. } else {
  1035. qstr = GetNgramQuery(query, mustquery, findfields)
  1036. }
  1037. if qstr != "" {
  1038. if highlight {
  1039. ws := []string{}
  1040. for _, w := range strings.Split(findfields, ",") {
  1041. ws = append(ws, fmt.Sprintf(highlightStr, w, 1))
  1042. }
  1043. qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL, strings.Join(ws, ",")) + `}`
  1044. }
  1045. if len(fields) > 0 {
  1046. qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
  1047. }
  1048. if strings.HasPrefix(order, "CUSTOM_") {
  1049. qstr = qstr[:len(qstr)-1] + `,` + strings.TrimLeft(order, "CUSTOM_") + `}`
  1050. } else if len(order) > 0 {
  1051. qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
  1052. }
  1053. if start > -1 {
  1054. qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
  1055. }
  1056. //log.Println("ngram-find", qstr)
  1057. return Get(index, itype, qstr)
  1058. } else {
  1059. return nil
  1060. }
  1061. }
  1062. //增加高亮、过滤查询
  1063. func GetByNgramAll_New(index, itype string, querystring, querymust interface{}, mustquery, findfields, order, fields string, start, limit int, highlight bool, filtermode bool) *[]map[string]interface{} {
  1064. defer catch()
  1065. qstr := ""
  1066. if filtermode {
  1067. qstr = GetNgramQuery_New(querystring, querymust, mustquery, findfields)
  1068. } else {
  1069. qstr = GetNgramQuery_New(querystring, "", mustquery, findfields)
  1070. }
  1071. if qstr != "" {
  1072. if highlight {
  1073. ws := []string{}
  1074. for _, w := range strings.Split(findfields, ",") {
  1075. ws = append(ws, w+`:{"force_source": true}`)
  1076. }
  1077. qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL_New, strings.Join(ws, ",")) + `}`
  1078. }
  1079. if len(fields) > 0 {
  1080. qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
  1081. }
  1082. if len(order) > 0 {
  1083. qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", ",", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
  1084. }
  1085. if start > -1 {
  1086. qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
  1087. }
  1088. //log.Println("ngram-find", order, qstr)
  1089. return Get(index, itype, qstr)
  1090. } else {
  1091. return nil
  1092. }
  1093. }
  1094. type KeyConfig struct {
  1095. Keys []string `json:"key"`
  1096. NotKeys []string `json:"notkey"`
  1097. InfoTypes []string `json:"infotype"`
  1098. Areas []string `json:"area"`
  1099. }
  1100. //替换了"号
  1101. func GetResForJY(index, itype string, keys []KeyConfig, allquery, findfields, SortQuery, fields string, start, limit int) *[]map[string]interface{} {
  1102. if len(keys) > 0 {
  1103. qstr := ""
  1104. new_minq := fmt.Sprintf(minq, "%s", findfields)
  1105. not_new_minq := fmt.Sprintf(minq, "%s", findfields) //排除词只查询标题
  1106. musts := []string{}
  1107. for _, qs_words := range keys {
  1108. mq := []string{}
  1109. notmq := []string{}
  1110. for _, qs_word := range qs_words.Keys {
  1111. mq = append(mq, fmt.Sprintf(new_minq, ReplaceYH(qs_word)))
  1112. /*
  1113. qs := AnalyzerWord("bidding", qs_word)
  1114. for _, qw := range qs {
  1115. mq = append(mq, fmt.Sprintf(new_minq, ReplaceYH(qw)))
  1116. }
  1117. */
  1118. }
  1119. for _, qs_word := range qs_words.NotKeys {
  1120. notmq = append(notmq, fmt.Sprintf(not_new_minq, ReplaceYH(qs_word)))
  1121. }
  1122. if len(qs_words.Areas) > 0 {
  1123. mq = append(mq, fmt.Sprintf(`{"terms":{"area":["%s"]}}`, strings.Join(qs_words.Areas, `","`)))
  1124. }
  1125. if len(qs_words.InfoTypes) > 0 {
  1126. mq = append(mq, fmt.Sprintf(`{"terms":{"toptype":["%s"]}}`, strings.Join(qs_words.InfoTypes, `","`)))
  1127. }
  1128. musts = append(musts, fmt.Sprintf(NgramMustAndNot, strings.Join(mq, ","), strings.Join(notmq, ",")))
  1129. }
  1130. qstr = fmt.Sprintf(NgramStr, "", strings.Join(musts, ","))
  1131. qstr = fmt.Sprintf(FilterQuery, allquery, qstr[1:])
  1132. ws := []string{}
  1133. for _, w := range strings.Split(findfields, ",") {
  1134. ws = append(ws, fmt.Sprintf(highlightStr, w, 1))
  1135. }
  1136. qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL, strings.Join(ws, ",")) + `}`
  1137. if len(fields) > 0 {
  1138. qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
  1139. }
  1140. if len(SortQuery) > 0 {
  1141. qstr = qstr[:len(qstr)-1] + `,"sort":` + SortQuery + `}`
  1142. }
  1143. if start > -1 {
  1144. qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
  1145. }
  1146. //log.Println("jy-ngram-find", qstr)
  1147. return Get(index, itype, qstr)
  1148. } else {
  1149. return nil
  1150. }
  1151. }
  1152. func ReplaceYH(src string) (rpl string) {
  1153. return strings.Replace(src, `"`, `\"`, -1)
  1154. }
  1155. //
  1156. func GetAllByNgram(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
  1157. if qstr != "" {
  1158. if highlight {
  1159. ws := []string{}
  1160. for _, w := range strings.Split(findfields, ",") {
  1161. ws = append(ws, fmt.Sprintf(highlightStr, w, count))
  1162. }
  1163. qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL, strings.Join(ws, ",")) + `}`
  1164. }
  1165. if len(fields) > 0 {
  1166. qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
  1167. }
  1168. if len(order) > 0 {
  1169. qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
  1170. }
  1171. if start > -1 {
  1172. qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
  1173. }
  1174. //log.Println("GetAllByNgram:", qstr)
  1175. return Get(index, itype, qstr)
  1176. } else {
  1177. return nil
  1178. }
  1179. }
  1180. //数据标记2019-07-10
  1181. func GetAllByNgram_MP(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
  1182. if qstr != "" {
  1183. if highlight {
  1184. ws := []string{}
  1185. for _, w := range strings.Split(findfields, ",") {
  1186. ws = append(ws, fmt.Sprintf(highlightStr, w, count))
  1187. }
  1188. qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL_MP, strings.Join(ws, ",")) + `}`
  1189. }
  1190. if len(fields) > 0 {
  1191. qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
  1192. }
  1193. if len(order) > 0 {
  1194. qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
  1195. }
  1196. if start > -1 {
  1197. qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
  1198. }
  1199. // log.Println("GetAllByNgram:", qstr)
  1200. return Get(index, itype, qstr)
  1201. } else {
  1202. return nil
  1203. }
  1204. }
  1205. //ik 分词
  1206. func GetAllByIk(index, itype, qstr, findfields, order, fields string, start, limit, count int, highlight bool) *[]map[string]interface{} {
  1207. if qstr != "" {
  1208. if highlight {
  1209. ws := []string{}
  1210. for _, w := range strings.Split(findfields, ",") {
  1211. ws = append(ws, fmt.Sprintf(ik_highlightStr, w, count))
  1212. }
  1213. qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL_IK, strings.Join(ws, ",")) + `}`
  1214. }
  1215. if len(fields) > 0 {
  1216. qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
  1217. }
  1218. if len(order) > 0 {
  1219. qstr = qstr[:len(qstr)-1] + `,"sort":[` + SR(SR(SR(SR(order, ",", "},{", -1), " ", "", -1), ":-1", `:"desc"`, -1), ":1", `:"asc"`, -1) + `]}`
  1220. }
  1221. if start > -1 {
  1222. qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
  1223. }
  1224. //log.Println("GetAllByNgram:", qstr)
  1225. return Get(index, itype, qstr)
  1226. } else {
  1227. return nil
  1228. }
  1229. }
  1230. //分词
  1231. func AnalyzerWord(index, word string) (result []string) {
  1232. client := GetEsConn()
  1233. defer DestoryEsConn(client)
  1234. result = []string{}
  1235. p := url.Values{}
  1236. p["text"] = []string{word}
  1237. p["analyzer"] = []string{"ik"}
  1238. by, err := client.PerformRequest("GET", "/"+index+"/_analyze", p, nil)
  1239. if err != nil {
  1240. log.Println("AnalyzerWord Error:", err)
  1241. return
  1242. }
  1243. b, err := by.Body.MarshalJSON()
  1244. if err != nil {
  1245. log.Println("AnalyzerWord MarshalJSON Error:", err)
  1246. return
  1247. }
  1248. var res map[string][]map[string]interface{}
  1249. err = json.Unmarshal(b, &res)
  1250. if err != nil {
  1251. log.Println("AnalyzerWord Unmarshal Error:", err)
  1252. return
  1253. }
  1254. if res == nil {
  1255. return
  1256. }
  1257. for _, v := range res["tokens"] {
  1258. token, _ := v["token"].(string)
  1259. if token != "" {
  1260. result = append(result, token)
  1261. }
  1262. }
  1263. return
  1264. }
  1265. func GetResForJYView(index, itype string, keys []KeyConfig, allquery, findfields, SortQuery, fields string, start, limit int) *[]map[string]interface{} {
  1266. if len(keys) > 0 {
  1267. qstr := ""
  1268. new_minq := fmt.Sprintf(minq, "%s", findfields)
  1269. not_new_minq := fmt.Sprintf(minq, "%s", findfields) //排除词只查询标题
  1270. musts := []string{}
  1271. for _, qs_words := range keys {
  1272. mq := []string{}
  1273. notmq := []string{}
  1274. for _, qs_word := range qs_words.Keys {
  1275. mq = append(mq, fmt.Sprintf(new_minq, ReplaceYH(qs_word)))
  1276. }
  1277. for _, qs_word := range qs_words.NotKeys {
  1278. notmq = append(notmq, fmt.Sprintf(not_new_minq, ReplaceYH(qs_word)))
  1279. }
  1280. if len(qs_words.Areas) > 0 {
  1281. mq = append(mq, fmt.Sprintf(`{"terms":{"area":["%s"]}}`, strings.Join(qs_words.Areas, `","`)))
  1282. }
  1283. if len(qs_words.InfoTypes) > 0 {
  1284. mq = append(mq, fmt.Sprintf(`{"terms":{"toptype":["%s"]}}`, strings.Join(qs_words.InfoTypes, `","`)))
  1285. }
  1286. musts = append(musts, fmt.Sprintf(NgramMustAndNot, strings.Join(mq, ","), strings.Join(notmq, ",")))
  1287. }
  1288. qstr = fmt.Sprintf(NgramStr, "", strings.Join(musts, ","))
  1289. qstr = fmt.Sprintf(FilterQuery, allquery, qstr[1:])
  1290. ws := []string{}
  1291. for _, w := range strings.Split(findfields, ",") {
  1292. ws = append(ws, fmt.Sprintf(highlightStr, w, 1))
  1293. }
  1294. qstr = qstr[:len(qstr)-1] + `,` + fmt.Sprintf(HL, strings.Join(ws, ",")) + `}`
  1295. if len(fields) > 0 {
  1296. qstr = qstr[:len(qstr)-1] + `,"_source":[` + fields + "]}"
  1297. }
  1298. if len(SortQuery) > 0 {
  1299. qstr = qstr[:len(qstr)-1] + `,"sort":` + SortQuery + `}`
  1300. }
  1301. if start > -1 {
  1302. qstr = qstr[:len(qstr)-1] + `,"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(limit) + "}"
  1303. }
  1304. return Get(index, itype, qstr)
  1305. } else {
  1306. return nil
  1307. }
  1308. }
  1309. //返回count 和 res
  1310. func GetWithCount(index, itype, query string) (int64, *[]map[string]interface{}) {
  1311. client := GetEsConn()
  1312. defer func() {
  1313. go DestoryEsConn(client)
  1314. }()
  1315. var res []map[string]interface{}
  1316. var count int64 = 0
  1317. if client != nil {
  1318. defer func() {
  1319. if r := recover(); r != nil {
  1320. log.Println("[E]", r)
  1321. for skip := 1; ; skip++ {
  1322. _, file, line, ok := runtime.Caller(skip)
  1323. if !ok {
  1324. break
  1325. }
  1326. go log.Printf("%v,%v\n", file, line)
  1327. }
  1328. }
  1329. }()
  1330. searchResult, err := client.Search().Index(index).Type(itype).Source(query).Do()
  1331. if err != nil {
  1332. log.Println("从ES查询出错", err.Error())
  1333. return count, nil
  1334. }
  1335. if searchResult.Hits != nil {
  1336. resNum := len(searchResult.Hits.Hits)
  1337. count = searchResult.Hits.TotalHits
  1338. if resNum < 5000 {
  1339. res = make([]map[string]interface{}, resNum)
  1340. for i, hit := range searchResult.Hits.Hits {
  1341. //d := json.NewDecoder(bytes.NewBuffer(*hit.Source))
  1342. //d.UseNumber()
  1343. //d.Decode(&res[i])
  1344. parseErr := json.Unmarshal(*hit.Source, &res[i])
  1345. if parseErr == nil && hit.Highlight != nil && res[i] != nil {
  1346. res[i]["highlight"] = map[string][]string(hit.Highlight)
  1347. }
  1348. }
  1349. } else {
  1350. log.Println("查询结果太多,查询到:", resNum, "条")
  1351. }
  1352. }
  1353. }
  1354. return count, &res
  1355. }