utiltag.go 44 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306
  1. package util
  2. import (
  3. "cmplatform/models"
  4. sql "cmplatform/util/sqlmodel"
  5. "context"
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "log"
  10. "regexp"
  11. "strings"
  12. "time"
  13. "app.yhyue.com/moapp/jybase/common"
  14. "app.yhyue.com/moapp/jybase/encrypt"
  15. elastic "app.yhyue.com/moapp/jybase/es"
  16. "app.yhyue.com/moapp/jybase/redis"
  17. "go.mongodb.org/mongo-driver/bson"
  18. )
  19. const (
  20. TitleMatchType = "1" //标题匹配
  21. DetailMatchType = "2" //全文匹配
  22. PurchasingMatchType = "3" //标的物匹配
  23. AttachmentsMatchType = "4" //附件匹配
  24. ProjectnameMatchType = "5" //项目名称匹配
  25. BuyerMatchType = "6" //采购单位匹配
  26. SWinnerMatchType = "7" //中标单位匹配
  27. BuyerMatchType1 = "8" //采购单位(正则)匹配
  28. SWinnerMatchType1 = "9" //中标单位(正则)匹配
  29. ProjectnameMatchTypes = "10" //采购意向名称匹配
  30. ProjectscopeMatchTypes = "11" //采购意向内容匹配
  31. BuyerMatchTypes = "12" //采购意向采购单位匹配
  32. TitleMatchTypeStr = "title" //标题匹配
  33. DetailMatchTypeStr = "detail" //全文匹配
  34. PurchasingMatchTypeStr = "purchasing" //标的物匹配
  35. AttachmentsMatchTypeStr = "filetext" //附件匹配
  36. ProjectnameMatchTypeStr = "projectname.pname" //项目名称匹配
  37. BuyerMatchTypeStr = "buyer" //项目名称匹配
  38. BuyerRegexpMatchTypeStr = "buyer.mbuyer" //采购单位正则匹配
  39. SWinnerMatchTypeStr = "s_winner" //项目名称匹配
  40. SWinnerRegexpMatchTypeStr = "s_winner.mwinner" //中标单位正则匹配
  41. TitleMatchTypeField = "title" //标题匹配
  42. DetailMatchTypeField = "detail" //全文匹配
  43. PurchasingMatchTypeField = "purchasing" //标的物匹配
  44. AttachmentsMatchTypeField = "filetext" //附件匹配
  45. ProjectnameMatchTypeField = "projectname" //项目名称匹配
  46. BuyerMatchTypeField = "buyer" //项目名称匹配
  47. SWinnerMatchTypeField = "s_winner" //项目名称匹配
  48. ProjectnameMatchTypesField = "procurementlist.projectname" //采购意向名称匹配
  49. ProjectscopeMatchTypesField = "procurementlist.projectscope" //采购意向内容匹配
  50. BuyerMatchTypesField = "procurementlist.buyer" //采购意向采购单位匹配
  51. MultiMatchType = "phrase"
  52. Standard = 1 //1:标准字段包,
  53. Advanced = 2 //2:高级字段包
  54. Standardstr = "standard" //1:标准字段包
  55. Advancedstr = "advanced" //2:高级字段包
  56. Url = "https://www.jianyu360.cn/article/content/%s.html"
  57. )
  58. func UtilEsSaveData(sdataid, index string, datas *[]map[string]interface{}) error {
  59. defer common.Catch()
  60. if !Mgo.Del("tagsdata", bson.M{"s_dataid": sdataid, "esIndex": index}) {
  61. return errors.New("mongo del err")
  62. }
  63. for _, v := range *datas {
  64. v["esIndex"] = index
  65. Mgo.Save("tagsdata", v)
  66. }
  67. return nil
  68. }
  69. // 客户规则
  70. func UtilEsFind1(tags map[string]interface{}, dataType, userId string) (error, string, int64) {
  71. defer common.Catch()
  72. sdataid := common.ObjToString(tags["s_dataid"])
  73. esquery := common.ObjToString(tags["s_esquery"])
  74. if IsNewSql != 0 {
  75. esquery = common.ObjToString(tags["s_esquery_search"])
  76. }
  77. if len(esquery) < 1 || len(sdataid) < 1 {
  78. return errors.New("s_esquery or s_dataid no found"), "", 0
  79. }
  80. i_maxnum := common.Int64All(tags["i_maxnum"])
  81. if i_maxnum <= 0 {
  82. i_maxnum = 100
  83. }
  84. maths := make([]map[string]string, 0)
  85. if orules, ok := tags["o_rules"].([]interface{}); ok {
  86. for _, v := range orules {
  87. orule, _ := v.(map[string]interface{})
  88. maths = append(maths, map[string]string{
  89. "s_matchkey": common.ObjToString(orule["s_matchkey"]),
  90. "s_keymatch": common.ObjToString(orule["s_keymatch"]),
  91. })
  92. }
  93. } else if orules, ok := tags["o_rules"].([]map[string]interface{}); ok {
  94. for _, v := range orules {
  95. maths = append(maths, map[string]string{
  96. "s_matchkey": common.ObjToString(v["s_matchkey"]),
  97. "s_keymatch": common.ObjToString(v["s_keymatch"]),
  98. })
  99. }
  100. } else {
  101. return errors.New("o_rules no found"), "", 0
  102. }
  103. index := EsIndex
  104. if strings.Contains(esquery, "site") { //选择网站名称,使用全量数据索引
  105. index = EsAllIndex
  106. }
  107. dataTypes := ""
  108. if dataType == "1" {
  109. index = InterimIndex
  110. dataTypes = "1"
  111. startTime := time.Now().Unix() - 86400*90
  112. endTime := time.Now().Unix()
  113. esquery, _ = FilterTimeSql(esquery, startTime, endTime)
  114. } else {
  115. index = TotalIndex
  116. dataTypes = "2"
  117. }
  118. // esquery = esquery[:len(esquery)-1] + `,"size":` + fmt.Sprintf("%d", i_maxnum) + `}`
  119. err, counts := searchDataArr(index, esquery, sdataid, userId, i_maxnum, tags, maths)
  120. if counts == 0 && err == nil && index == InterimIndex {
  121. err, counts = searchDataArr(TotalIndex, esquery, sdataid, userId, i_maxnum, tags, maths)
  122. if counts > 0 {
  123. dataTypes = "2"
  124. }
  125. }
  126. return err, dataTypes, counts
  127. }
  128. func searchDataArr(index, esquery, sdataid, userId string, i_maxnum int64, tags map[string]interface{}, maths []map[string]string) (error, int64) {
  129. var (
  130. err error
  131. counts = int64(0)
  132. // times = 0
  133. times2 = 0
  134. )
  135. // for {
  136. // listLen := redis.GetInt("session", "es_status")
  137. // if listLen == 0 {
  138. // log.Println("es空闲!")
  139. // break
  140. // } else if times > 10 {
  141. // err = errors.New("系统繁忙,请稍后再试")
  142. // break
  143. // } else {
  144. // log.Println("es繁忙,", listLen)
  145. // }
  146. // times += 2
  147. // time.Sleep(2 * time.Second)
  148. // }
  149. for {
  150. listLens := int(redis.LLEN("datag", "jyqyfw_es_query"))
  151. if listLens < 2 {
  152. // if isExists, _ := redis.Exists("datag", "jyqyfw_es_query_times_"+userId); isExists {
  153. // qt := redis.GetInt("datag", "jyqyfw_es_query_times_"+userId)
  154. // if qt > 2 {
  155. // log.Println("单个账号一分钟内超过3次 ", qt)
  156. // err = errors.New("系统繁忙,请稍后再试")
  157. // break
  158. // } else {
  159. // redis.Incr("datag", "jyqyfw_es_query_times_"+userId)
  160. // }
  161. // } else {
  162. // redis.Put("datag", "jyqyfw_es_query_times_"+userId, 1, 60)
  163. // }
  164. log.Println("es空闲,当前进程数 ", listLens)
  165. redis.RPUSH("datag", "jyqyfw_es_query", 1)
  166. err, counts = searchData(index, esquery, sdataid, i_maxnum, tags, maths)
  167. redis.LPOP("datag", "jyqyfw_es_query")
  168. break
  169. } else if times2 > 10 {
  170. err = errors.New("系统繁忙,请稍后再试")
  171. break
  172. } else {
  173. log.Println("企业级服务es进程数过多,", listLens)
  174. }
  175. times2 += 2
  176. time.Sleep(2 * time.Second)
  177. }
  178. return err, counts
  179. }
  180. type MySource struct {
  181. Querys string
  182. }
  183. func (m *MySource) Source() (interface{}, error) {
  184. mp := make(map[string]interface{})
  185. json.Unmarshal([]byte(m.Querys), &mp)
  186. return mp["query"], nil
  187. }
  188. func searchData(index, esquery, sdataid string, i_maxnum int64, tags map[string]interface{}, maths []map[string]string) (error, int64) {
  189. esCon := elastic.VarEs.(*elastic.EsV7)
  190. client := esCon.GetEsConn()
  191. defer esCon.DestoryEsConn(client)
  192. ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute)
  193. cc := &MySource{
  194. Querys: esquery,
  195. }
  196. searchResult, err := client.Search(index).Query(cc).Size(int(i_maxnum)).Do(ctx)
  197. if err == nil && searchResult.Hits != nil {
  198. datas := make([]map[string]interface{}, 0)
  199. log.Println("es查询到的数量:", searchResult.Hits.TotalHits.Value)
  200. log.Println("es查询数据数量:", len(searchResult.Hits.Hits))
  201. for _, v := range searchResult.Hits.Hits {
  202. item := make(map[string]interface{})
  203. if json.Unmarshal(v.Source, &item) == nil {
  204. delete(item, "_id")
  205. item["info_id"] = common.ObjToString(item["id"])
  206. item["s_dataid"] = sdataid
  207. item["s_jyhref"] = fmt.Sprintf(Url, encrypt.CommonEncodeArticle("content", common.ObjToString(item["id"])))
  208. item["i_createtime"] = time.Now().Unix()
  209. var d *DFA = &DFA{}
  210. var analyKeys []string //找到的关键词
  211. var matchType []string //匹配方式
  212. for _, math := range maths {
  213. fileds := strsToArr(math["s_keymatch"], "field")
  214. d.AddWord(strings.Split(math["s_matchkey"], ",")...)
  215. mkMap := make(map[string]interface{})
  216. tmpMap := make(map[string]interface{})
  217. for _, mk := range strings.Split(math["s_matchkey"], ",") {
  218. if strings.Contains(mk, "&&") {
  219. arr := strings.Split(mk, "&&")
  220. for _, s := range arr {
  221. if s != "" {
  222. tmpMap[s] = mk
  223. if b, _ := regexp.MatchString("[A-Z]", s); b {
  224. mkMap[strings.ToLower(s)] = s
  225. d.AddWord(strings.ToLower(s))
  226. } else {
  227. d.AddWord(s)
  228. }
  229. }
  230. }
  231. } else {
  232. if b, _ := regexp.MatchString("[A-Z]", mk); b {
  233. mkMap[strings.ToLower(mk)] = mk
  234. d.AddWord(strings.ToLower(mk))
  235. } else {
  236. d.AddWord(mk)
  237. }
  238. }
  239. }
  240. for _, filed := range fileds {
  241. filed1 := strings.ToLower(common.ObjToString(item[filed]))
  242. ddds := d.Analy(filed1)
  243. analyKeys = append(analyKeys, ddds...)
  244. }
  245. if len(analyKeys) > 0 {
  246. matchType = append(matchType, strings.Join(fileds, ","))
  247. for k, v := range analyKeys {
  248. if tmpMap[v] != "" && tmpMap[v] != nil {
  249. analyKeys[k] = common.ObjToString(tmpMap[v])
  250. }
  251. }
  252. for _, v1 := range analyKeys {
  253. if mkMap[v1] != "" && mkMap[v1] != nil {
  254. analyKeys = deleteSlice(analyKeys, v1)
  255. analyKeys = append(analyKeys, common.ObjToString(mkMap[v1]))
  256. }
  257. }
  258. }
  259. d.Clear()
  260. }
  261. //去重
  262. ssavekey := make(map[string]bool)
  263. for _, v := range analyKeys {
  264. ssavekey[v] = true
  265. }
  266. ssavekeys := []string{}
  267. for k := range ssavekey {
  268. ssavekeys = append(ssavekeys, k)
  269. }
  270. item["s_matchkey"] = GetMactchKey(maths, item)
  271. item["s_matchtype"] = strings.Join(matchType, ",")
  272. findwinner := strings.TrimSpace(common.ObjToString(item["winner"]))
  273. if findwinner != "" {
  274. finddata, _ := MgoEnps.FindOne(EnpsColl, bson.M{"company_name": findwinner})
  275. if finddata != nil {
  276. if legal_person := common.ObjToString((*finddata)["legal_person"]); legal_person != "" {
  277. item["legal_person"] = legal_person
  278. }
  279. if email := common.ObjToString((*finddata)["company_email"]); email != "" {
  280. item["company_email"] = email
  281. }
  282. if phone := common.ObjToString((*finddata)["company_phone"]); phone != "" {
  283. item["company_phone"] = phone
  284. }
  285. item["qyk"] = finddata
  286. }
  287. }
  288. datas = append(datas, item)
  289. }
  290. }
  291. counts := Es.Count(index, Itype, esquery)
  292. log.Println("esCount查询的数据量 ", counts)
  293. Mgo.Update("cuserdepartrule", bson.M{"_id": tags["_id"]}, bson.M{
  294. "$set": bson.M{
  295. "i_estotal": counts,
  296. }}, false, false)
  297. return UtilEsSaveData(sdataid, index, &datas), counts
  298. } else {
  299. log.Println("err", err)
  300. return err, 0
  301. }
  302. }
  303. func FilterTimeSql(esquery string, startTime, endTime int64) (string, error) {
  304. query := map[string]*sql.QueryObjecct{}
  305. if json.Unmarshal([]byte(esquery), &query) == nil {
  306. qb := query["query"]
  307. for i := 0; i <= 2; i++ {
  308. filter := qb.Bool
  309. if filter != nil { //有filter
  310. index := -1 //记录range的位置
  311. for i, m := range filter.Must {
  312. mMap := m.(map[string]interface{})
  313. if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range
  314. if esRange["publishtime"] != nil {
  315. index = i
  316. break
  317. }
  318. }
  319. }
  320. // 生成新的es语句
  321. tmpRange_ := bson.M{
  322. "range": bson.M{
  323. "publishtime": bson.M{
  324. "lte": endTime,
  325. "gt": startTime,
  326. },
  327. },
  328. }
  329. if index > -1 {
  330. filter.Must[index] = tmpRange_
  331. } else {
  332. filter.Must = append(filter.Must, tmpRange_)
  333. }
  334. tmpQuery := query
  335. strQuery, err := json.Marshal(tmpQuery)
  336. if err == nil {
  337. return string(strQuery), nil
  338. } else {
  339. log.Println("失败", err)
  340. return esquery, err
  341. }
  342. } else {
  343. // qb.Filter = &sql.Filter{}
  344. qb.Bool = &sql.BoolObject{}
  345. continue
  346. }
  347. }
  348. } else {
  349. log.Println(json.Unmarshal([]byte(esquery), &query), "err")
  350. return esquery, errors.New("err")
  351. }
  352. return esquery, nil
  353. }
  354. // 查询语句过长时根据开始结束时间分段查询数据
  355. func UtilEsFind2(tags map[string]interface{}, n int) (error, int64) {
  356. defer common.Catch()
  357. sdataid := common.ObjToString(tags["s_dataid"])
  358. esquery := common.ObjToString(tags["s_esquery"])
  359. if len(esquery) < 1 || len(sdataid) < 1 {
  360. return errors.New("s_esquery or s_dataid no found"), 0
  361. }
  362. i_maxnum := common.Int64All(tags["i_maxnum"])
  363. if i_maxnum <= 0 {
  364. i_maxnum = 100
  365. }
  366. maths := make([]map[string]string, 0)
  367. if orules, ok := tags["o_rules"].([]interface{}); ok {
  368. for _, v := range orules {
  369. orule, _ := v.(map[string]interface{})
  370. maths = append(maths, map[string]string{
  371. "s_matchkey": common.ObjToString(orule["s_matchkey"]),
  372. "s_keymatch": common.ObjToString(orule["s_keymatch"]),
  373. })
  374. }
  375. } else if orules, ok := tags["o_rules"].([]map[string]interface{}); ok {
  376. for _, v := range orules {
  377. maths = append(maths, map[string]string{
  378. "s_matchkey": common.ObjToString(v["s_matchkey"]),
  379. "s_keymatch": common.ObjToString(v["s_keymatch"]),
  380. })
  381. }
  382. } else {
  383. return errors.New("o_rules no found"), 0
  384. }
  385. esCon := elastic.VarEs.(*elastic.EsV7)
  386. client := esCon.GetEsConn()
  387. defer esCon.DestoryEsConn(client)
  388. ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
  389. //for _, k := range {
  390. ////todo 修改开始结束时间 重新生成es语句
  391. //}
  392. // 获取分隔区间
  393. timeList := GetTimeInterval(time.Unix(common.Int64All(tags["i_starttime"]), 0), time.Unix(common.Int64All(tags["i_endtime"]), 0), n)
  394. log.Println(timeList, "分隔后的时间===================")
  395. esQueryList := []string{}
  396. //esquery = esquery[:len(esquery)-1] + `,"size":` + fmt.Sprintf("%d", i_maxnum) + `}`
  397. // log.Println(esquery)
  398. query := map[string]*sql.QueryObjecct{}
  399. if json.Unmarshal([]byte(esquery), &query) == nil {
  400. qb := query["query"]
  401. filter := qb.Bool
  402. if filter != nil { //有filter
  403. index := -1 //记录range的位置
  404. for i, m := range filter.Must {
  405. mMap := m.(map[string]interface{})
  406. if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range
  407. if esRange["publishtime"] != nil {
  408. index = i
  409. break
  410. }
  411. }
  412. }
  413. log.Println("timeList================", timeList)
  414. // 生成新的es语句
  415. for _, v := range timeList {
  416. tmpRange_ := bson.M{
  417. "range": bson.M{
  418. "publishtime": bson.M{
  419. "lte": v["endTime"],
  420. "gt": v["startTime"],
  421. },
  422. },
  423. }
  424. if index > -1 {
  425. filter.Must[index] = tmpRange_
  426. } else {
  427. filter.Must = append(filter.Must, tmpRange_)
  428. }
  429. tmpQuery := query
  430. strQuery, err := json.Marshal(tmpQuery)
  431. if err == nil {
  432. esQuery := string(strQuery)
  433. esQueryList = append(esQueryList, esQuery)
  434. } else {
  435. log.Println("失败", err)
  436. return err, 0
  437. }
  438. }
  439. }
  440. } else {
  441. log.Println(json.Unmarshal([]byte(esquery), &query), "err")
  442. }
  443. // for _, v := range esQueryList {
  444. // log.Println("es语句", v)
  445. // }
  446. log.Println("重新生成的es语句", esQueryList)
  447. var totalCount int64
  448. datas := make([]map[string]interface{}, 0)
  449. for _, esSql := range esQueryList {
  450. // 查询
  451. esSql = esSql[:len(esSql)-1] + `,"size":` + fmt.Sprintf("%d", i_maxnum) + `}`
  452. log.Println("本次查询==", esSql)
  453. index := EsIndex
  454. if strings.Contains(esSql, "site") { //选择网站名称,使用全量数据索引
  455. index = EsAllIndex
  456. }
  457. log.Println("es索引-------------------- ", index)
  458. cc := &MySource{
  459. Querys: esSql,
  460. }
  461. searchResult, err := client.Search(index).Query(cc).Do(ctx)
  462. if err == nil && searchResult.Hits != nil {
  463. log.Println("es查询到的数量:", searchResult.Hits.TotalHits)
  464. log.Println("开始处理")
  465. for _, v := range searchResult.Hits.Hits {
  466. item := make(map[string]interface{})
  467. if json.Unmarshal(v.Source, &item) == nil {
  468. delete(item, "_id")
  469. item["info_id"] = common.ObjToString(item["id"])
  470. item["s_dataid"] = sdataid
  471. item["s_jyhref"] = fmt.Sprintf(Url, encrypt.CommonEncodeArticle("content", common.ObjToString(item["id"])))
  472. item["i_createtime"] = time.Now().Unix()
  473. var d *DFA = &DFA{}
  474. var analyKeys []string //找到的关键词
  475. var matchType []string //匹配方式
  476. for _, math := range maths {
  477. fileds := strsToArr(math["s_keymatch"], "field")
  478. d.AddWord(strings.Split(math["s_matchkey"], ",")...)
  479. mkMap := make(map[string]interface{})
  480. tmpMap := make(map[string]interface{})
  481. for _, mk := range strings.Split(math["s_matchkey"], ",") {
  482. if strings.Contains(mk, "&&") {
  483. arr := strings.Split(mk, "&&")
  484. for _, s := range arr {
  485. if s != "" {
  486. tmpMap[s] = mk
  487. if b, _ := regexp.MatchString("[A-Z]", s); b {
  488. mkMap[strings.ToLower(s)] = s
  489. d.AddWord(strings.ToLower(s))
  490. } else {
  491. d.AddWord(s)
  492. }
  493. }
  494. }
  495. } else {
  496. if b, _ := regexp.MatchString("[A-Z]", mk); b {
  497. mkMap[strings.ToLower(mk)] = mk
  498. d.AddWord(strings.ToLower(mk))
  499. } else {
  500. d.AddWord(mk)
  501. }
  502. }
  503. }
  504. for _, filed := range fileds {
  505. filed1 := strings.ToLower(common.ObjToString(item[filed]))
  506. ddds := d.Analy(filed1)
  507. analyKeys = append(analyKeys, ddds...)
  508. }
  509. if len(analyKeys) > 0 {
  510. matchType = append(matchType, strings.Join(fileds, ","))
  511. for k, v := range analyKeys {
  512. if tmpMap[v] != "" && tmpMap[v] != nil {
  513. analyKeys[k] = common.ObjToString(tmpMap[v])
  514. }
  515. }
  516. for _, v1 := range analyKeys {
  517. if mkMap[v1] != "" && mkMap[v1] != nil {
  518. analyKeys = deleteSlice(analyKeys, v1)
  519. analyKeys = append(analyKeys, common.ObjToString(mkMap[v1]))
  520. }
  521. }
  522. }
  523. d.Clear()
  524. }
  525. //去重
  526. ssavekey := make(map[string]bool)
  527. for _, v := range analyKeys {
  528. ssavekey[v] = true
  529. }
  530. ssavekeys := []string{}
  531. for k := range ssavekey {
  532. ssavekeys = append(ssavekeys, k)
  533. }
  534. item["s_matchkey"] = GetMactchKey(maths, item)
  535. item["s_matchtype"] = strings.Join(matchType, ",")
  536. findwinner := strings.TrimSpace(common.ObjToString(item["winner"]))
  537. if findwinner != "" {
  538. finddata, _ := MgoEnps.FindOne(EnpsColl, bson.M{"company_name": findwinner})
  539. if finddata != nil {
  540. if legal_person := common.ObjToString((*finddata)["legal_person"]); legal_person != "" {
  541. item["legal_person"] = legal_person
  542. }
  543. if email := common.ObjToString((*finddata)["company_email"]); email != "" {
  544. item["company_email"] = email
  545. }
  546. if phone := common.ObjToString((*finddata)["company_phone"]); phone != "" {
  547. item["company_phone"] = phone
  548. }
  549. //从最新年报中获取 中标单位联系电话、中标单位邮箱
  550. // if annual_reports, ok := finddata["annual_reports"].(primitive.A); ok && len(annual_reports) > 0 {
  551. // report_year := Sort_year_report(annual_reports) //最新年报
  552. // if len(report_year) > 0 {
  553. // if email := common.ObjToString(report_year["company_email"]); email != "" {
  554. // item["company_email"] = email
  555. // }
  556. // if phone := common.ObjToString(report_year["company_phone"]); phone != "" {
  557. // item["company_phone"] = phone
  558. // }
  559. // }
  560. // }
  561. item["qyk"] = finddata
  562. }
  563. }
  564. datas = append(datas, item)
  565. }
  566. }
  567. log.Println("处理完成", len(datas))
  568. count := searchResult.Hits.TotalHits.Value
  569. totalCount += count
  570. } else {
  571. log.Println(err)
  572. return err, 0
  573. }
  574. }
  575. Mgo.Update("cuserdepartrule", bson.M{"_id": tags["_id"]}, bson.M{
  576. "$set": bson.M{
  577. "i_estotal": totalCount,
  578. }}, false, false)
  579. if int64(len(datas)) > i_maxnum {
  580. datas = datas[:i_maxnum]
  581. }
  582. return UtilEsSaveData(sdataid, "", &datas), totalCount
  583. }
  584. func GetTimeInterval(startTime, entTime time.Time, n int) []map[string]interface{} {
  585. timeList := []map[string]interface{}{}
  586. endTimeTmp := startTime
  587. for endTimeTmp.Unix() < entTime.Unix() {
  588. var endTime_ time.Time
  589. startTime = endTimeTmp
  590. endTimeTmp = startTime.AddDate(0, n, 0)
  591. if endTimeTmp.Unix() > entTime.Unix() {
  592. endTime_ = entTime
  593. timeList = append(timeList, map[string]interface{}{"startTime": startTime.Unix(), "endTime": endTime_.Unix()})
  594. break
  595. } else {
  596. endTime_ = endTimeTmp
  597. timeList = append(timeList, map[string]interface{}{"startTime": startTime.Unix(), "endTime": endTime_.Unix()})
  598. }
  599. }
  600. //log.Println(timeList)
  601. return timeList
  602. }
  603. func Utiltags(tag map[string]interface{}) string {
  604. defer common.Catch()
  605. tmpbyte, _ := json.Marshal(tag)
  606. tab := models.Tag{}
  607. err := json.Unmarshal(tmpbyte, &tab)
  608. if err != nil {
  609. return "json err:" + err.Error()
  610. }
  611. //log.Println("前端数据", tab)
  612. QueryObjecct := sql.QueryObjecct{}
  613. ffBoolObject := sql.BoolObject{}
  614. adsBoolObect := sql.NewEsObject{}
  615. if tab.Sarea != "" {
  616. adsBoolObect.Bool.Should = append(adsBoolObect.Bool.Should, sql.AreaCityDistrictMust{AreaCityDistrict: &sql.AreaCityDistrict{Area: strings.Split(tab.Sarea, ",")}})
  617. }
  618. if tab.Scity != "" {
  619. adsBoolObect.Bool.Should = append(adsBoolObect.Bool.Should, sql.AreaCityDistrictMust{AreaCityDistrict: &sql.AreaCityDistrict{City: strings.Split(tab.Scity, ",")}})
  620. }
  621. if tab.Sdistrict != "" {
  622. //城市——区县
  623. cityds := strings.Split(tab.Sdistrict, ",")
  624. var ds []string
  625. for _, v := range cityds {
  626. ds = append(ds, strings.Split(v, "-")[1])
  627. }
  628. adsBoolObect.Bool.Should = append(adsBoolObect.Bool.Should, sql.AreaCityDistrictMust{AreaCityDistrict: &sql.AreaCityDistrict{District: ds}})
  629. }
  630. if len(adsBoolObect.Bool.Should) > 0 {
  631. ffBoolObject.Must = append(ffBoolObject.Must, adsBoolObect)
  632. }
  633. if tab.Stoptype != "" || tab.Ssubtype != "" {
  634. if len(tab.Stoptype) > 0 {
  635. toptypeSubtype := sql.ToptypeSubtype{}
  636. toptypeSubtype.Toptype = strings.Split(tab.Stoptype, ",")
  637. ffBoolObject.Must = append(ffBoolObject.Must, sql.ToptypeSubtypeMust{&toptypeSubtype})
  638. }
  639. if len(tab.Ssubtype) > 0 {
  640. toptypeSubtype := sql.ToptypeSubtype{}
  641. toptypeSubtype.Subtype = strings.Split(tab.Ssubtype, ",")
  642. ffBoolObject.Must = append(ffBoolObject.Must, sql.ToptypeSubtypeMust{&toptypeSubtype})
  643. }
  644. }
  645. if tab.Site != "" {
  646. ffBoolObject.Must = append(ffBoolObject.Must, sql.SiteMust{Site: &sql.Site{Site: strings.Split(tab.Site, ",")}})
  647. }
  648. if tab.Istarttime > 0 && tab.Iendtime > 0 {
  649. ffBoolObject.Must = append(ffBoolObject.Must, sql.PublishtimeMust{PublishtimeObject: &sql.PublishtimeObject{Publishtime: &sql.Publishtime{
  650. Gte: tab.Istarttime,
  651. Lt: tab.Iendtime,
  652. }}})
  653. } else if tab.Istarttime > 0 {
  654. ffBoolObject.Must = append(ffBoolObject.Must, sql.PublishtimeMust{PublishtimeObject: &sql.PublishtimeObject{Publishtime1: &sql.Publishtime1{
  655. Gte: tab.Istarttime,
  656. }}})
  657. } else if tab.Iendtime > 0 {
  658. ffBoolObject.Must = append(ffBoolObject.Must, sql.PublishtimeMust{PublishtimeObject: &sql.PublishtimeObject{Publishtime2: &sql.Publishtime2{
  659. Lt: tab.Iendtime,
  660. }}})
  661. }
  662. if tab.Sglobalbuyerclass != "" {
  663. if len(tab.Sglobalbuyerclass) > 0 {
  664. buyerclass := sql.BuyerclassObject{
  665. Terms: struct {
  666. Buyerclass []string `json:"buyerclass,omitempty"`
  667. }{
  668. Buyerclass: strings.Split(tab.Sglobalbuyerclass, ","),
  669. },
  670. }
  671. ffBoolObject.Must = append(ffBoolObject.Must, buyerclass)
  672. }
  673. }
  674. if tab.Sglobaltopscopeclass != "" || tab.Sglobalsubscopeclass != "" {
  675. if len(tab.Sglobaltopscopeclass) > 0 {
  676. topScopeclass := sql.Scopeclass{}
  677. topScopeclass.Globaltopscopeclass = strings.Split(tab.Sglobaltopscopeclass, ",")
  678. ffBoolObject.Must = append(ffBoolObject.Must, sql.ScopeclassMust{&topScopeclass})
  679. } else if len(tab.Sglobalsubscopeclass) > 0 {
  680. subScopeclass := sql.Scopeclass{}
  681. subScopeclass.Globalsubscopeclass = strings.Split(tab.Sglobalsubscopeclass, ",")
  682. ffBoolObject.Must = append(ffBoolObject.Must, sql.ScopeclassMust{&subScopeclass})
  683. }
  684. }
  685. fqBoolObject := sql.BoolObject{}
  686. if len(tab.Sexistfields) > 0 {
  687. tmpsfields := strings.Split(tab.Sexistfields, ",")
  688. for _, v := range tmpsfields {
  689. // fqBoolObject.MustNot = append(fqBoolObject.MustNot, sql.ExistfieldsObjectMust{
  690. // ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  691. // Missing struct {
  692. // Field string `json:"field,omitempty"`
  693. // } `json:"missing,omitempty"`
  694. // }{
  695. // Missing: struct {
  696. // Field string `json:"field,omitempty"`
  697. // }{Field: v},
  698. // },
  699. // },
  700. // })
  701. ffBoolObject.Must = append(ffBoolObject.Must, sql.ExistfieldsObjectMust{
  702. ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  703. Exists struct {
  704. Field string `json:"field,omitempty"`
  705. } `json:"exists,omitempty"`
  706. }{
  707. Exists: struct {
  708. Field string `json:"field,omitempty"`
  709. }{Field: v},
  710. },
  711. },
  712. })
  713. }
  714. }
  715. newEsObject := sql.NewEsObject{}
  716. if tab.Sglobaladdkey != "" && tab.Sglobaladdkeymatch != "" {
  717. if tmps := strsToArr(tab.Sglobaladdkeymatch, "str"); tmps != nil {
  718. newEsObject = method1(newEsObject, tab.Sglobaladdkey, tab.Sglobaladdkeymatch, tmps)
  719. }
  720. }
  721. if len(newEsObject.Bool.Should) > 0 || len(newEsObject.Bool.MustNot) > 0 || len(newEsObject.Bool.Must) > 0 {
  722. fqBoolObject.Must = append(fqBoolObject.Must, newEsObject)
  723. }
  724. nots := sql.NewEsObject{}
  725. //
  726. if tab.ExpurasingtimeStart > 0 && tab.ExpurasingtimeEnd > 0 {
  727. ffBoolObject.MustNot = append(ffBoolObject.MustNot, sql.ExpurasingtimeMust{ExpurasingtimeObject: &sql.ExpurasingtimeObject{Expurasingtime: &sql.Expurasingtime{
  728. Gte: tab.ExpurasingtimeStart,
  729. Lt: tab.ExpurasingtimeEnd,
  730. }}})
  731. } else if tab.ExpurasingtimeStart > 0 {
  732. ffBoolObject.MustNot = append(ffBoolObject.MustNot, sql.ExpurasingtimeMust{ExpurasingtimeObject: &sql.ExpurasingtimeObject{Expurasingtime1: &sql.Expurasingtime1{
  733. Gte: tab.ExpurasingtimeStart,
  734. }}})
  735. } else if tab.ExpurasingtimeEnd > 0 {
  736. ffBoolObject.MustNot = append(ffBoolObject.MustNot, sql.ExpurasingtimeMust{ExpurasingtimeObject: &sql.ExpurasingtimeObject{Expurasingtime2: &sql.Expurasingtime2{
  737. Lt: tab.ExpurasingtimeEnd,
  738. }}})
  739. }
  740. //
  741. if tab.Sglobalnotkey != "" && tab.Sglobalnotkeymatch != "" {
  742. if tmps := strsToArr(tab.Sglobalnotkeymatch, "str"); tmps != nil {
  743. nots = method1(nots, tab.Sglobalnotkey, tab.Sglobalnotkeymatch, tmps)
  744. }
  745. }
  746. if len(nots.Bool.Should) > 0 {
  747. fqBoolObject.MustNot = append(fqBoolObject.MustNot, nots)
  748. }
  749. torules := sql.BoolObject{}
  750. for _, v := range tab.Orules {
  751. tmpses := sql.NewEsObject{}
  752. if skeymatchs := strsToArr(v.Skeymatch, "str"); skeymatchs != nil {
  753. tmpnewEsObject := sql.NewEsObject{}
  754. tmpnewEsObject = method1(tmpnewEsObject, v.Smatchkey, v.Skeymatch, skeymatchs)
  755. if len(tmpnewEsObject.Bool.Should) > 0 || len(tmpnewEsObject.Bool.Must) > 0 || len(tmpnewEsObject.Bool.MustNot) > 0 {
  756. tmpses.Bool.Must = append(tmpses.Bool.Must, tmpnewEsObject)
  757. }
  758. }
  759. if saddkeymatchs := strsToArr(v.Saddkeymatch, "str"); saddkeymatchs != nil {
  760. addkeyarr := sql.NewEsObject{}
  761. addkeyarr = method1(addkeyarr, v.Saddkey, v.Saddkeymatch, saddkeymatchs)
  762. if len(addkeyarr.Bool.Should) > 0 {
  763. tmpses.Bool.Must = append(tmpses.Bool.Must, addkeyarr)
  764. }
  765. }
  766. if snotkeymatchs := strsToArr(v.Snotkeymatch, "str"); snotkeymatchs != nil {
  767. tmpses = method2(tmpses, v.Snotkey, v.Snotkeymatch, snotkeymatchs)
  768. }
  769. if len(v.Stopscopeclass) > 0 {
  770. sArr := strings.Split(v.Stopscopeclass, ",")
  771. tmpses.Bool.Must = append(tmpses.Bool.Must, sql.TopscopeclassObject{
  772. Terms: struct {
  773. Topscopeclass []string `json:"s_topscopeclass,omitempty"`
  774. }{
  775. Topscopeclass: sArr,
  776. },
  777. })
  778. } else if len(v.Ssubscopeclass) > 0 {
  779. tmpses.Bool.Must = append(tmpses.Bool.Must, sql.SubscopeclassObject{
  780. Terms: struct {
  781. Subscopeclass []string `json:"s_subscopeclass,omitempty"`
  782. }{
  783. Subscopeclass: strings.Split(v.Ssubscopeclass, ","),
  784. },
  785. })
  786. }
  787. if len(v.Sbuyerclass) > 0 {
  788. tmpses.Bool.Must = append(tmpses.Bool.Must, sql.BuyerclassObject{
  789. Terms: struct {
  790. Buyerclass []string `json:"buyerclass,omitempty"`
  791. }{
  792. Buyerclass: strings.Split(v.Sbuyerclass, ","),
  793. },
  794. })
  795. }
  796. torules.Should = append(torules.Should, tmpses)
  797. }
  798. bidamountSql := sql.NewEsObject{}
  799. if tab.Sbidamount != "" {
  800. if strings.Contains(tab.Sbidamount, "大于") && strings.Contains(tab.Sbidamount, "小于") {
  801. arr := strings.Split(tab.Sbidamount, ",")
  802. limit := &sql.BudgetOrBidamount{
  803. Gte: common.Float64All(strings.Replace(arr[0], "大于", "", -1)),
  804. Lt: common.Float64All(strings.Replace(arr[1], "小于", "", -1)),
  805. }
  806. bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}})
  807. } else if strings.Contains(tab.Sbidamount, "大于") {
  808. limit := &sql.BudgetOrBidamount{
  809. Gte: common.Float64All(strings.Replace(tab.Sbidamount, "大于", "", -1)),
  810. }
  811. bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}})
  812. } else if strings.Contains(tab.Sbidamount, "小于") {
  813. limit := &sql.BudgetOrBidamount{
  814. Lt: common.Float64All(strings.Replace(tab.Sbidamount, "小于", "", -1)),
  815. }
  816. bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}})
  817. }
  818. if tab.SbidamountFieldExist == 1 {
  819. // bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.ExistfieldsObjectMust{
  820. // ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  821. // Missing struct {
  822. // Field string `json:"field,omitempty"`
  823. // } `json:"missing,omitempty"`
  824. // }{
  825. // Missing: struct {
  826. // Field string `json:"field,omitempty"`
  827. // }{Field: "bidamount"},
  828. // },
  829. // },
  830. // })
  831. SbidamountFieldExistBool := sql.NewEsObject{}
  832. SbidamountFieldExistBool.Bool.MustNot = append(SbidamountFieldExistBool.Bool.MustNot, sql.ExistfieldsObjectMust{
  833. ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  834. Exists struct {
  835. Field string `json:"field,omitempty"`
  836. } `json:"exists,omitempty"`
  837. }{
  838. Exists: struct {
  839. Field string `json:"field,omitempty"`
  840. }{Field: "bidamount"},
  841. },
  842. },
  843. })
  844. bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, SbidamountFieldExistBool)
  845. }
  846. }
  847. budgetSql := sql.NewEsObject{}
  848. if tab.Sbudget != "" {
  849. if strings.Contains(tab.Sbudget, "大于") && strings.Contains(tab.Sbudget, "小于") {
  850. arr := strings.Split(tab.Sbudget, ",")
  851. limit := &sql.BudgetOrBidamount{
  852. Gte: common.Float64All(strings.Replace(arr[0], "大于", "", -1)),
  853. Lt: common.Float64All(strings.Replace(arr[1], "小于", "", -1)),
  854. }
  855. budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}})
  856. } else if strings.Contains(tab.Sbudget, "大于") {
  857. limit := &sql.BudgetOrBidamount{
  858. Gte: common.Float64All(strings.Replace(tab.Sbudget, "大于", "", -1)),
  859. }
  860. budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}})
  861. } else if strings.Contains(tab.Sbudget, "小于") {
  862. limit := &sql.BudgetOrBidamount{
  863. Lt: common.Float64All(strings.Replace(tab.Sbudget, "小于", "", -1)),
  864. }
  865. budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}})
  866. }
  867. if tab.SbudgetFieldExist == 1 {
  868. // budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.ExistfieldsObjectMust{
  869. // ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  870. // Missing struct {
  871. // Field string `json:"field,omitempty"`
  872. // } `json:"missing,omitempty"`
  873. // }{
  874. // Missing: struct {
  875. // Field string `json:"field,omitempty"`
  876. // }{Field: "budget"},
  877. // },
  878. // },
  879. // })
  880. SbudgetFieldExistBool := sql.NewEsObject{}
  881. SbudgetFieldExistBool.Bool.MustNot = append(SbudgetFieldExistBool.Bool.MustNot, sql.ExistfieldsObjectMust{
  882. ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  883. Exists struct {
  884. Field string `json:"field,omitempty"`
  885. } `json:"exists,omitempty"`
  886. }{
  887. Exists: struct {
  888. Field string `json:"field,omitempty"`
  889. }{Field: "budget"},
  890. },
  891. },
  892. })
  893. budgetSql.Bool.Should = append(budgetSql.Bool.Should, SbudgetFieldExistBool)
  894. }
  895. }
  896. if len(ffBoolObject.Must) > 0 || len(ffBoolObject.MustNot) > 0 || len(ffBoolObject.Should) > 0 {
  897. // QueryObjecct.Filter = &sql.Filter{}
  898. QueryObjecct.Bool = &ffBoolObject
  899. }
  900. if len(fqBoolObject.Must) > 0 || len(fqBoolObject.MustNot) > 0 || len(fqBoolObject.Should) > 0 {
  901. if QueryObjecct.Bool == nil {
  902. QueryObjecct.Bool = &fqBoolObject
  903. } else {
  904. if fqBoolObject.Must != nil {
  905. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, fqBoolObject.Must...)
  906. }
  907. if fqBoolObject.Should != nil {
  908. QueryObjecct.Bool.Should = append(QueryObjecct.Bool.Should, fqBoolObject.Should...)
  909. }
  910. if fqBoolObject.MustNot != nil {
  911. QueryObjecct.Bool.MustNot = append(QueryObjecct.Bool.MustNot, fqBoolObject.MustNot...)
  912. }
  913. }
  914. if len(torules.Should) > 0 {
  915. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, map[string]interface{}{
  916. "bool": torules,
  917. })
  918. }
  919. } else if len(torules.Should) > 0 {
  920. if QueryObjecct.Bool == nil {
  921. // QueryObjecct.Query = &sql.Query{}
  922. QueryObjecct.Bool = &sql.BoolObject{}
  923. }
  924. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, map[string]interface{}{
  925. "bool": torules,
  926. })
  927. }
  928. if (len(budgetSql.Bool.Should) > 0 || len(bidamountSql.Bool.Should) > 0) && (len(fqBoolObject.Must) > 0 || len(fqBoolObject.MustNot) > 0 || len(fqBoolObject.Should) > 0) {
  929. if QueryObjecct.Bool == nil {
  930. QueryObjecct.Bool = &sql.BoolObject{}
  931. }
  932. if len(budgetSql.Bool.Should) > 0 {
  933. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, budgetSql)
  934. }
  935. if len(bidamountSql.Bool.Should) > 0 {
  936. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, bidamountSql)
  937. }
  938. } else {
  939. if QueryObjecct.Bool == nil {
  940. QueryObjecct.Bool = &sql.BoolObject{}
  941. }
  942. if len(budgetSql.Bool.Should) > 0 {
  943. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, budgetSql)
  944. }
  945. if len(bidamountSql.Bool.Should) > 0 {
  946. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, bidamountSql)
  947. }
  948. }
  949. rdata := make(map[string]interface{})
  950. rdata["query"] = QueryObjecct
  951. esbytes, _ := json.Marshal(rdata)
  952. esStr := strings.Replace(string(esbytes), "regexp1", "regexp", -1)
  953. esStr = strings.Replace(esStr, "publishtime1", "publishtime", -1)
  954. esStr = strings.Replace(esStr, "publishtime2", "publishtime", -1)
  955. esStr = strings.Replace(esStr, "expurasingtime1", "expurasingtime", -1)
  956. esStr = strings.Replace(esStr, "expurasingtime2", "expurasingtime", -1)
  957. return strings.Replace(esStr, "regexp2", "regexp", -1)
  958. }
  959. func strsToArr(strs string, typestr string) []string {
  960. if len(strs) > 0 {
  961. tmps := strings.Split(strs, ",")
  962. switch typestr {
  963. case "str":
  964. for index, vint := range tmps {
  965. switch vint {
  966. case TitleMatchType:
  967. tmps[index] = TitleMatchTypeStr
  968. case DetailMatchType:
  969. tmps[index] = DetailMatchTypeStr
  970. case PurchasingMatchType:
  971. tmps[index] = PurchasingMatchTypeStr
  972. case AttachmentsMatchType:
  973. tmps[index] = AttachmentsMatchTypeStr
  974. case ProjectnameMatchType:
  975. tmps[index] = ProjectnameMatchTypeStr
  976. case BuyerMatchType:
  977. tmps[index] = BuyerMatchTypeStr
  978. case SWinnerMatchType:
  979. tmps[index] = SWinnerMatchTypeStr
  980. case BuyerMatchType1:
  981. tmps[index] = BuyerRegexpMatchTypeStr
  982. case SWinnerMatchType1:
  983. tmps[index] = SWinnerRegexpMatchTypeStr
  984. case ProjectnameMatchTypes:
  985. tmps[index] = ProjectnameMatchTypesField
  986. case ProjectscopeMatchTypes:
  987. tmps[index] = ProjectscopeMatchTypesField
  988. case BuyerMatchTypes:
  989. tmps[index] = BuyerMatchTypesField
  990. }
  991. }
  992. return tmps
  993. case "field":
  994. for index, vint := range tmps {
  995. switch vint {
  996. case TitleMatchType:
  997. tmps[index] = TitleMatchTypeField
  998. case DetailMatchType:
  999. tmps[index] = DetailMatchTypeField
  1000. case PurchasingMatchType:
  1001. tmps[index] = PurchasingMatchTypeField
  1002. case AttachmentsMatchType:
  1003. tmps[index] = AttachmentsMatchTypeField
  1004. case ProjectnameMatchType:
  1005. tmps[index] = ProjectnameMatchTypeField
  1006. case BuyerMatchType:
  1007. tmps[index] = BuyerMatchTypeField
  1008. case SWinnerMatchType:
  1009. tmps[index] = SWinnerMatchTypeField
  1010. case BuyerMatchType1:
  1011. tmps[index] = BuyerMatchTypeField
  1012. case SWinnerMatchType1:
  1013. tmps[index] = SWinnerMatchTypeField
  1014. case ProjectnameMatchTypes:
  1015. tmps[index] = ProjectnameMatchTypesField
  1016. case ProjectscopeMatchTypes:
  1017. tmps[index] = ProjectscopeMatchTypesField
  1018. case BuyerMatchTypes:
  1019. tmps[index] = BuyerMatchTypesField
  1020. }
  1021. }
  1022. return tmps
  1023. }
  1024. }
  1025. return nil
  1026. }
  1027. func init() {
  1028. EsIndex = common.ObjToString(Sysconfig["elasticsearch_index"])
  1029. EsType = common.ObjToString(Sysconfig["elasticsearch_type"])
  1030. }
  1031. func method1(newEsObject sql.NewEsObject, keyword, keymatch string, tmps []string) sql.NewEsObject {
  1032. keywordArr := []string{}
  1033. for _, vv := range strings.Split(keyword, ",") {
  1034. if vv == "" {
  1035. continue
  1036. }
  1037. //log.Println(vv, tmps, keymatch)
  1038. if len(tmps) == 1 && (strings.Contains(keymatch, "8") || strings.Contains(keymatch, "9")) {
  1039. //中标单位、采购单位、中标单位(正则)、采购单位(正则) 单选
  1040. strs := ""
  1041. if strings.Contains(vv, "&&") {
  1042. //strs = ".*" + strings.Replace(vv, "&&", ".*", -1) + ".*"
  1043. strs = strings.Replace(vv, "&&", ".*", -1)
  1044. } else {
  1045. strs = vv
  1046. }
  1047. if tmps[0] == "buyer.mbuyer" {
  1048. newEsObject.Bool.Should = append(newEsObject.Bool.Should, sql.ShouldObj{
  1049. MultiMatch: &sql.MultiMatch{
  1050. Query: strs,
  1051. Type: MultiMatchType,
  1052. Fields: tmps,
  1053. },
  1054. })
  1055. } else if tmps[0] == "s_winner.mwinner" {
  1056. newEsObject.Bool.Should = append(newEsObject.Bool.Should, sql.ShouldObj{
  1057. MultiMatch: &sql.MultiMatch{
  1058. Query: strs,
  1059. Type: MultiMatchType,
  1060. Fields: tmps,
  1061. },
  1062. })
  1063. }
  1064. } else if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) {
  1065. addkeylines := strings.Split(vv, "&&")
  1066. keywordArr = append(keywordArr, addkeylines[0])
  1067. } else {
  1068. addkeylines := strings.Split(vv, "&&")
  1069. //log.Println(addkeylines, "addkeylines")
  1070. if len(addkeylines) > 1 {
  1071. addkeyline := sql.NewEsObject{}
  1072. for _, vvv := range addkeylines {
  1073. if vvv == "" {
  1074. continue
  1075. }
  1076. addkeyline.Bool.Must = append(addkeyline.Bool.Must, sql.ShouldObj{
  1077. MultiMatch: &sql.MultiMatch{
  1078. Query: vvv,
  1079. Type: MultiMatchType,
  1080. Fields: tmps,
  1081. },
  1082. })
  1083. }
  1084. if len(addkeyline.Bool.Must) > 0 {
  1085. newEsObject.Bool.Should = append(newEsObject.Bool.Should, addkeyline)
  1086. }
  1087. } else {
  1088. newEsObject.Bool.Should = append(newEsObject.Bool.Should, sql.ShouldObj{
  1089. MultiMatch: &sql.MultiMatch{
  1090. Query: vv,
  1091. Type: MultiMatchType,
  1092. Fields: tmps,
  1093. },
  1094. })
  1095. }
  1096. }
  1097. }
  1098. if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) {
  1099. addkeyline := sql.NewEsObject{}
  1100. if tmps[0] == "buyer" {
  1101. addkeyline.Bool.Must = append(addkeyline.Bool.Must, sql.BuyerMatch{
  1102. Buyer: &sql.Buyer{Buyer: keywordArr},
  1103. })
  1104. } else if tmps[0] == "s_winner" {
  1105. addkeyline.Bool.Must = append(addkeyline.Bool.Must, sql.WinnerMatch{
  1106. Winner: &sql.Winner{Winner: keywordArr},
  1107. })
  1108. }
  1109. if len(addkeyline.Bool.Must) > 0 {
  1110. newEsObject.Bool.Should = append(newEsObject.Bool.Should, addkeyline)
  1111. }
  1112. }
  1113. return newEsObject
  1114. }
  1115. func method2(newEsObject sql.NewEsObject, keyword, keymatch string, tmps []string) sql.NewEsObject {
  1116. notKeyWordArr := []string{}
  1117. for _, vv := range strings.Split(keyword, ",") {
  1118. if vv == "" {
  1119. continue
  1120. }
  1121. if len(tmps) == 1 && (strings.Contains(keymatch, "8") || strings.Contains(keymatch, "9")) {
  1122. //中标单位、采购单位、中标单位(正则)、采购单位(正则) 单选
  1123. //strs := ""
  1124. //if strings.Contains(vv, "&&") {
  1125. // strs = ".*" + strings.Replace(vv, "&&", ".*", -1) + ".*"
  1126. //} else {
  1127. // strs = ".*" + vv + ".*"
  1128. //}
  1129. if tmps[0] == "buyer.mbuyer" {
  1130. newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, sql.ShouldObj{
  1131. MultiMatch: &sql.MultiMatch{
  1132. Query: vv,
  1133. Type: MultiMatchType,
  1134. Fields: tmps,
  1135. },
  1136. })
  1137. } else if tmps[0] == "s_winner.mwinner" {
  1138. newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, sql.ShouldObj{
  1139. MultiMatch: &sql.MultiMatch{
  1140. Query: vv,
  1141. Type: MultiMatchType,
  1142. Fields: tmps,
  1143. },
  1144. })
  1145. }
  1146. } else if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) {
  1147. notkeylines := strings.Split(vv, "&&")
  1148. notKeyWordArr = append(notKeyWordArr, notkeylines[0])
  1149. } else {
  1150. notkeylines := strings.Split(vv, "&&")
  1151. if len(notkeylines) > 1 {
  1152. notkeyline := sql.NewEsObject{}
  1153. for _, vvv := range notkeylines {
  1154. if vvv == "" {
  1155. continue
  1156. }
  1157. notkeyline.Bool.Must = append(notkeyline.Bool.Must, sql.ShouldObj{
  1158. MultiMatch: &sql.MultiMatch{
  1159. Query: vvv,
  1160. Type: MultiMatchType,
  1161. Fields: tmps,
  1162. },
  1163. })
  1164. }
  1165. if len(notkeyline.Bool.Must) > 0 {
  1166. newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, notkeyline)
  1167. }
  1168. } else {
  1169. newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, sql.ShouldObj{
  1170. MultiMatch: &sql.MultiMatch{
  1171. Query: vv,
  1172. Type: MultiMatchType,
  1173. Fields: tmps,
  1174. },
  1175. })
  1176. }
  1177. }
  1178. }
  1179. if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) {
  1180. var notkeyline interface{}
  1181. if tmps[0] == "buyer" {
  1182. notkeyline = sql.BuyerMatch{
  1183. Buyer: &sql.Buyer{Buyer: notKeyWordArr},
  1184. }
  1185. } else if tmps[0] == "s_winner" {
  1186. notkeyline = sql.WinnerMatch{
  1187. Winner: &sql.Winner{Winner: notKeyWordArr},
  1188. }
  1189. }
  1190. if notkeyline != nil {
  1191. newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, notkeyline)
  1192. }
  1193. }
  1194. return newEsObject
  1195. }
  1196. /**
  1197. * 一级公告行业处理 d、t、p
  1198. */
  1199. func method3(arr []string) []string {
  1200. var sArr []string
  1201. for _, v := range arr {
  1202. sArr = append(sArr, v+"d")
  1203. sArr = append(sArr, v+"t")
  1204. sArr = append(sArr, v+"p")
  1205. }
  1206. return sArr
  1207. }
  1208. func GetMactchKey(match []map[string]string, data map[string]interface{}) string {
  1209. keyWord := []string{}
  1210. for _, keys := range match {
  1211. types := keys["s_keymatch"]
  1212. key := keys["s_matchkey"]
  1213. if strings.Contains(types, "1") {
  1214. title := common.ObjToString(data["title"])
  1215. keyWord = KeyWordToData(types, title, key, keyWord)
  1216. }
  1217. if strings.Contains(types, "2") {
  1218. detail := common.ObjToString(data["detail"])
  1219. keyWord = KeyWordToData(types, detail, key, keyWord)
  1220. }
  1221. if strings.Contains(types, "3") {
  1222. purchasing := common.ObjToString(data["purchasing"])
  1223. keyWord = KeyWordToData(types, purchasing, key, keyWord)
  1224. }
  1225. if strings.Contains(types, "4") {
  1226. filetext := common.ObjToString(data["filetext"])
  1227. keyWord = KeyWordToData(types, filetext, key, keyWord)
  1228. }
  1229. if strings.Contains(types, "5") {
  1230. projectname := common.ObjToString(data["projectname"])
  1231. keyWord = KeyWordToData(types, projectname, key, keyWord)
  1232. }
  1233. if strings.Contains(types, "6") || strings.Contains(types, "8") {
  1234. buyer := common.ObjToString(data["buyer"])
  1235. keyWord = KeyWordToData(types, buyer, key, keyWord)
  1236. }
  1237. if strings.Contains(types, "7") || strings.Contains(types, "9") {
  1238. winner := common.ObjToString(data["s_winner"])
  1239. keyWord = KeyWordToData(types, winner, key, keyWord)
  1240. }
  1241. }
  1242. keyMap := map[string]bool{}
  1243. keyArr := []string{}
  1244. for _, key := range keyWord {
  1245. keyMap[key] = true
  1246. }
  1247. for k, _ := range keyMap {
  1248. keyArr = append(keyArr, k)
  1249. }
  1250. return strings.Join(keyArr, ",")
  1251. }
  1252. func KeyWordToData(types, item, key string, keyWord []string) []string {
  1253. for _, mk := range strings.Split(key, ",") {
  1254. if strings.Contains(mk, "&&") {
  1255. arr := strings.Split(mk, "&&")
  1256. isok := true
  1257. for _, s := range arr {
  1258. if s != "" {
  1259. if !strings.Contains(strings.ToUpper(item), strings.ToUpper(s)) {
  1260. isok = false
  1261. }
  1262. }
  1263. }
  1264. if isok {
  1265. keyWord = append(keyWord, mk)
  1266. }
  1267. } else {
  1268. spaceReg := regexp.MustCompile("[\\s\\n \u3000\u2003\u00a0]+")
  1269. itemUpper := strings.ToUpper(item)
  1270. mkUpper := strings.ToUpper(mk)
  1271. matchResult := strings.Contains(spaceReg.ReplaceAllString(itemUpper, ""), spaceReg.ReplaceAllString(mkUpper, ""))
  1272. if matchResult {
  1273. keyWord = append(keyWord, mk)
  1274. }
  1275. }
  1276. }
  1277. return keyWord
  1278. }