utiltag.go 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312
  1. package util
  2. import (
  3. "cmplatform/models"
  4. sql "cmplatform/util/sqlmodel"
  5. "context"
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "regexp"
  10. "strings"
  11. "time"
  12. "app.yhyue.com/moapp/jybase/common"
  13. "app.yhyue.com/moapp/jybase/encrypt"
  14. elastic "app.yhyue.com/moapp/jybase/es"
  15. "app.yhyue.com/moapp/jybase/log"
  16. "go.mongodb.org/mongo-driver/bson"
  17. "go.uber.org/zap"
  18. )
  19. const (
  20. TitleMatchType = "1" //标题匹配
  21. DetailMatchType = "2" //全文匹配
  22. PurchasingMatchType = "3" //标的物匹配
  23. AttachmentsMatchType = "4" //附件匹配
  24. ProjectnameMatchType = "5" //项目名称匹配
  25. BuyerMatchType = "6" //采购单位匹配
  26. SWinnerMatchType = "7" //中标单位匹配
  27. BuyerMatchType1 = "8" //采购单位(正则)匹配
  28. SWinnerMatchType1 = "9" //中标单位(正则)匹配
  29. ProjectnameMatchTypes = "10" //采购意向名称匹配
  30. ProjectscopeMatchTypes = "11" //采购意向内容匹配
  31. BuyerMatchTypes = "12" //采购意向采购单位匹配
  32. TitleMatchTypeStr = "title" //标题匹配
  33. DetailMatchTypeStr = "detail" //全文匹配
  34. PurchasingMatchTypeStr = "purchasing" //标的物匹配
  35. AttachmentsMatchTypeStr = "filetext" //附件匹配
  36. ProjectnameMatchTypeStr = "projectname.pname" //项目名称匹配
  37. BuyerMatchTypeStr = "buyer" //项目名称匹配
  38. BuyerRegexpMatchTypeStr = "buyer.mbuyer" //采购单位正则匹配
  39. SWinnerMatchTypeStr = "s_winner" //项目名称匹配
  40. SWinnerRegexpMatchTypeStr = "s_winner.mwinner" //中标单位正则匹配
  41. TitleMatchTypeField = "title" //标题匹配
  42. DetailMatchTypeField = "detail" //全文匹配
  43. PurchasingMatchTypeField = "purchasing" //标的物匹配
  44. AttachmentsMatchTypeField = "filetext" //附件匹配
  45. ProjectnameMatchTypeField = "projectname" //项目名称匹配
  46. BuyerMatchTypeField = "buyer" //项目名称匹配
  47. SWinnerMatchTypeField = "s_winner" //项目名称匹配
  48. ProjectnameMatchTypesField = "procurementlist.projectname" //采购意向名称匹配
  49. ProjectscopeMatchTypesField = "procurementlist.projectscope" //采购意向内容匹配
  50. BuyerMatchTypesField = "procurementlist.buyer" //采购意向采购单位匹配
  51. MultiMatchType = "phrase"
  52. Standard = 1 //1:标准字段包,
  53. Advanced = 2 //2:高级字段包
  54. Standardstr = "standard" //1:标准字段包
  55. Advancedstr = "advanced" //2:高级字段包
  56. Url = "https://www.jianyu360.cn/article/content/%s.html"
  57. )
  58. func UtilEsSaveData(sdataid, index string, datas *[]map[string]interface{}) error {
  59. defer common.Catch()
  60. if !Mgo.Del("tagsdata", bson.M{"s_dataid": sdataid, "esIndex": index}) {
  61. return errors.New("mongo del err")
  62. }
  63. for _, v := range *datas {
  64. v["esIndex"] = index
  65. Mgo.Save("tagsdata", v)
  66. }
  67. return nil
  68. }
  69. // 客户规则
  70. func UtilEsFind1(tags map[string]interface{}, dataType, userId string) (error, string, int64) {
  71. defer common.Catch()
  72. sdataid := common.ObjToString(tags["s_dataid"])
  73. esquery := common.ObjToString(tags["s_esquery"])
  74. if IsNewSql != 0 {
  75. esquery = common.ObjToString(tags["s_esquery_search"])
  76. }
  77. if len(esquery) < 1 || len(sdataid) < 1 {
  78. return errors.New("s_esquery or s_dataid no found"), "", 0
  79. }
  80. i_maxnum := common.Int64All(tags["i_maxnum"])
  81. if i_maxnum <= 0 {
  82. i_maxnum = 100
  83. }
  84. maths := make([]map[string]string, 0)
  85. if orules, ok := tags["o_rules"].([]interface{}); ok {
  86. for _, v := range orules {
  87. orule, _ := v.(map[string]interface{})
  88. maths = append(maths, map[string]string{
  89. "s_matchkey": common.ObjToString(orule["s_matchkey"]),
  90. "s_keymatch": common.ObjToString(orule["s_keymatch"]),
  91. })
  92. }
  93. } else if orules, ok := tags["o_rules"].([]map[string]interface{}); ok {
  94. for _, v := range orules {
  95. maths = append(maths, map[string]string{
  96. "s_matchkey": common.ObjToString(v["s_matchkey"]),
  97. "s_keymatch": common.ObjToString(v["s_keymatch"]),
  98. })
  99. }
  100. } else {
  101. return errors.New("o_rules no found"), "", 0
  102. }
  103. index := EsIndex
  104. if strings.Contains(esquery, "site") { //选择网站名称,使用全量数据索引
  105. index = EsAllIndex
  106. }
  107. dataTypes := ""
  108. if dataType == "1" {
  109. index = InterimIndex
  110. dataTypes = "1"
  111. startTime := time.Now().Unix() - 86400*90
  112. endTime := time.Now().Unix()
  113. esquery, _ = FilterTimeSql(esquery, startTime, endTime)
  114. } else {
  115. index = TotalIndex
  116. dataTypes = "2"
  117. }
  118. // esquery = esquery[:len(esquery)-1] + `,"size":` + fmt.Sprintf("%d", i_maxnum) + `}`
  119. err, counts := searchDataArr(index, esquery, sdataid, userId, i_maxnum, tags, maths)
  120. if counts == 0 && err == nil && index == InterimIndex {
  121. err, counts = searchDataArr(TotalIndex, esquery, sdataid, userId, i_maxnum, tags, maths)
  122. if counts > 0 {
  123. dataTypes = "2"
  124. }
  125. }
  126. return err, dataTypes, counts
  127. }
  128. func searchDataArr(index, esquery, sdataid, userId string, i_maxnum int64, tags map[string]interface{}, maths []map[string]string) (error, int64) {
  129. var (
  130. err error
  131. counts = int64(0)
  132. // times = 0
  133. // times2 = 0
  134. )
  135. // for {
  136. // listLen := redis.GetInt("session", "es_status")
  137. // if listLen == 0 {
  138. // log.Println("es空闲!")
  139. // break
  140. // } else if times > 10 {
  141. // err = errors.New("系统繁忙,请稍后再试")
  142. // break
  143. // } else {
  144. // log.Println("es繁忙,", listLen)
  145. // }
  146. // times += 2
  147. // time.Sleep(2 * time.Second)
  148. // }
  149. // for {
  150. // listLens := int(redis.LLEN("datag", "jyqyfw_es_query"))
  151. // if listLens < 2 {
  152. // if isExists, _ := redis.Exists("datag", "jyqyfw_es_query_times_"+userId); isExists {
  153. // qt := redis.GetInt("datag", "jyqyfw_es_query_times_"+userId)
  154. // if qt > 2 {
  155. // log.Println("单个账号一分钟内超过3次 ", qt)
  156. // err = errors.New("系统繁忙,请稍后再试")
  157. // break
  158. // } else {
  159. // redis.Incr("datag", "jyqyfw_es_query_times_"+userId)
  160. // }
  161. // } else {
  162. // redis.Put("datag", "jyqyfw_es_query_times_"+userId, 1, 60)
  163. // }
  164. // log.Println("es空闲,当前进程数 ", listLens)
  165. // redis.RPUSH("datag", "jyqyfw_es_query", 1)
  166. err, counts = searchData(index, esquery, sdataid, i_maxnum, tags, maths)
  167. // redis.LPOP("datag", "jyqyfw_es_query")
  168. // break
  169. // } else if times2 > 10 {
  170. // err = errors.New("系统繁忙,请稍后再试")
  171. // break
  172. // } else {
  173. // log.Println("企业级服务es进程数过多,", listLens)
  174. // }
  175. // times2 += 2
  176. // time.Sleep(2 * time.Second)
  177. // }
  178. return err, counts
  179. }
  180. type MySource struct {
  181. Querys string
  182. }
  183. func (m *MySource) Source() (interface{}, error) {
  184. mp := make(map[string]interface{})
  185. json.Unmarshal([]byte(m.Querys), &mp)
  186. return mp["query"], nil
  187. }
  188. func searchData(index, esquery, sdataid string, i_maxnum int64, tags map[string]interface{}, maths []map[string]string) (error, int64) {
  189. esCon := elastic.VarEs.(*elastic.EsV7)
  190. client := esCon.GetEsConn()
  191. defer esCon.DestoryEsConn(client)
  192. ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute)
  193. cc := &MySource{
  194. Querys: esquery,
  195. }
  196. searchResult, err := client.Search(index).Query(cc).Size(int(i_maxnum)).Do(ctx)
  197. if err == nil && searchResult.Hits != nil {
  198. datas := make([]map[string]interface{}, 0)
  199. log.Info(fmt.Sprintf("es查询到的数量:%d", searchResult.Hits.TotalHits.Value))
  200. log.Info(fmt.Sprintf("es查询数据数量:%d", len(searchResult.Hits.Hits)))
  201. for _, v := range searchResult.Hits.Hits {
  202. item := make(map[string]interface{})
  203. if json.Unmarshal(v.Source, &item) == nil {
  204. delete(item, "_id")
  205. item["info_id"] = common.ObjToString(item["id"])
  206. item["s_dataid"] = sdataid
  207. item["s_jyhref"] = fmt.Sprintf(Url, encrypt.CommonEncodeArticle("content", common.ObjToString(item["id"])))
  208. item["i_createtime"] = time.Now().Unix()
  209. var d *DFA = &DFA{}
  210. var analyKeys []string //找到的关键词
  211. var matchType []string //匹配方式
  212. for _, math := range maths {
  213. fileds := strsToArr(math["s_keymatch"], "field")
  214. d.AddWord(strings.Split(math["s_matchkey"], ",")...)
  215. mkMap := make(map[string]interface{})
  216. tmpMap := make(map[string]interface{})
  217. for _, mk := range strings.Split(math["s_matchkey"], ",") {
  218. if strings.Contains(mk, "&&") {
  219. arr := strings.Split(mk, "&&")
  220. for _, s := range arr {
  221. if s != "" {
  222. tmpMap[s] = mk
  223. if b, _ := regexp.MatchString("[A-Z]", s); b {
  224. mkMap[strings.ToLower(s)] = s
  225. d.AddWord(strings.ToLower(s))
  226. } else {
  227. d.AddWord(s)
  228. }
  229. }
  230. }
  231. } else {
  232. if b, _ := regexp.MatchString("[A-Z]", mk); b {
  233. mkMap[strings.ToLower(mk)] = mk
  234. d.AddWord(strings.ToLower(mk))
  235. } else {
  236. d.AddWord(mk)
  237. }
  238. }
  239. }
  240. for _, filed := range fileds {
  241. filed1 := GetFieldData(item, filed)
  242. filed1 = ProcessData(filed1)
  243. ddds := d.Analy(filed1)
  244. analyKeys = append(analyKeys, ddds...)
  245. }
  246. if len(analyKeys) > 0 {
  247. matchType = append(matchType, strings.Join(fileds, ","))
  248. for k, v := range analyKeys {
  249. if tmpMap[v] != "" && tmpMap[v] != nil {
  250. analyKeys[k] = common.ObjToString(tmpMap[v])
  251. }
  252. }
  253. for _, v1 := range analyKeys {
  254. if mkMap[v1] != "" && mkMap[v1] != nil {
  255. analyKeys = deleteSlice(analyKeys, v1)
  256. analyKeys = append(analyKeys, common.ObjToString(mkMap[v1]))
  257. }
  258. }
  259. }
  260. d.Clear()
  261. }
  262. //去重
  263. ssavekey := make(map[string]bool)
  264. for _, v := range analyKeys {
  265. ssavekey[v] = true
  266. }
  267. ssavekeys := []string{}
  268. for k := range ssavekey {
  269. ssavekeys = append(ssavekeys, k)
  270. }
  271. item["s_matchkey"] = GetMactchKey(maths, item)
  272. item["s_matchtype"] = strings.Join(matchType, ",")
  273. findwinner := strings.TrimSpace(common.ObjToString(item["winner"]))
  274. if findwinner != "" {
  275. finddata, _ := MgoEnps.FindOne(EnpsColl, bson.M{"company_name": findwinner})
  276. if finddata != nil {
  277. if legal_person := common.ObjToString((*finddata)["legal_person"]); legal_person != "" {
  278. item["legal_person"] = legal_person
  279. }
  280. if email := common.ObjToString((*finddata)["company_email"]); email != "" {
  281. item["company_email"] = email
  282. }
  283. if phone := common.ObjToString((*finddata)["company_phone"]); phone != "" {
  284. item["company_phone"] = phone
  285. }
  286. item["qyk"] = finddata
  287. }
  288. }
  289. datas = append(datas, item)
  290. }
  291. }
  292. counts := Es.Count(index, Itype, esquery)
  293. log.Info(fmt.Sprintf("esCount查询的数据量 %d", counts))
  294. Mgo.Update("cuserdepartrule", bson.M{"_id": tags["_id"]}, bson.M{
  295. "$set": bson.M{
  296. "i_estotal": counts,
  297. }}, false, false)
  298. return UtilEsSaveData(sdataid, index, &datas), counts
  299. } else {
  300. log.Error("err", zap.Error(err))
  301. return err, 0
  302. }
  303. }
  304. func FilterTimeSql(esquery string, startTime, endTime int64) (string, error) {
  305. query := map[string]*sql.QueryObjecct{}
  306. if json.Unmarshal([]byte(esquery), &query) == nil {
  307. qb := query["query"]
  308. for i := 0; i <= 2; i++ {
  309. filter := qb.Bool
  310. if filter != nil { //有filter
  311. index := -1 //记录range的位置
  312. for i, m := range filter.Must {
  313. mMap := m.(map[string]interface{})
  314. if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range
  315. if esRange["publishtime"] != nil {
  316. index = i
  317. break
  318. }
  319. }
  320. }
  321. // 生成新的es语句
  322. tmpRange_ := bson.M{
  323. "range": bson.M{
  324. "publishtime": bson.M{
  325. "lte": endTime,
  326. "gt": startTime,
  327. },
  328. },
  329. }
  330. if index > -1 {
  331. filter.Must[index] = tmpRange_
  332. } else {
  333. filter.Must = append(filter.Must, tmpRange_)
  334. }
  335. tmpQuery := query
  336. strQuery, err := json.Marshal(tmpQuery)
  337. if err == nil {
  338. return string(strQuery), nil
  339. } else {
  340. log.Error("失败", zap.Error(err))
  341. return esquery, err
  342. }
  343. } else {
  344. // qb.Filter = &sql.Filter{}
  345. qb.Bool = &sql.BoolObject{}
  346. continue
  347. }
  348. }
  349. } else {
  350. log.Error("err", zap.Error(json.Unmarshal([]byte(esquery), &query)))
  351. return esquery, errors.New("err")
  352. }
  353. return esquery, nil
  354. }
  355. // 查询语句过长时根据开始结束时间分段查询数据
  356. func UtilEsFind2(tags map[string]interface{}, n int) (error, int64) {
  357. defer common.Catch()
  358. sdataid := common.ObjToString(tags["s_dataid"])
  359. esquery := common.ObjToString(tags["s_esquery"])
  360. if len(esquery) < 1 || len(sdataid) < 1 {
  361. return errors.New("s_esquery or s_dataid no found"), 0
  362. }
  363. i_maxnum := common.Int64All(tags["i_maxnum"])
  364. if i_maxnum <= 0 {
  365. i_maxnum = 100
  366. }
  367. maths := make([]map[string]string, 0)
  368. if orules, ok := tags["o_rules"].([]interface{}); ok {
  369. for _, v := range orules {
  370. orule, _ := v.(map[string]interface{})
  371. maths = append(maths, map[string]string{
  372. "s_matchkey": common.ObjToString(orule["s_matchkey"]),
  373. "s_keymatch": common.ObjToString(orule["s_keymatch"]),
  374. })
  375. }
  376. } else if orules, ok := tags["o_rules"].([]map[string]interface{}); ok {
  377. for _, v := range orules {
  378. maths = append(maths, map[string]string{
  379. "s_matchkey": common.ObjToString(v["s_matchkey"]),
  380. "s_keymatch": common.ObjToString(v["s_keymatch"]),
  381. })
  382. }
  383. } else {
  384. return errors.New("o_rules no found"), 0
  385. }
  386. esCon := elastic.VarEs.(*elastic.EsV7)
  387. client := esCon.GetEsConn()
  388. defer esCon.DestoryEsConn(client)
  389. ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
  390. //for _, k := range {
  391. ////todo 修改开始结束时间 重新生成es语句
  392. //}
  393. // 获取分隔区间
  394. timeList := GetTimeInterval(time.Unix(common.Int64All(tags["i_starttime"]), 0), time.Unix(common.Int64All(tags["i_endtime"]), 0), n)
  395. log.Info("分隔后的时间===================", zap.Any("", timeList))
  396. esQueryList := []string{}
  397. //esquery = esquery[:len(esquery)-1] + `,"size":` + fmt.Sprintf("%d", i_maxnum) + `}`
  398. // log.Println(esquery)
  399. query := map[string]*sql.QueryObjecct{}
  400. if json.Unmarshal([]byte(esquery), &query) == nil {
  401. qb := query["query"]
  402. filter := qb.Bool
  403. if filter != nil { //有filter
  404. index := -1 //记录range的位置
  405. for i, m := range filter.Must {
  406. mMap := m.(map[string]interface{})
  407. if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range
  408. if esRange["publishtime"] != nil {
  409. index = i
  410. break
  411. }
  412. }
  413. }
  414. log.Info("timeList================", zap.Any("", timeList))
  415. // 生成新的es语句
  416. for _, v := range timeList {
  417. tmpRange_ := bson.M{
  418. "range": bson.M{
  419. "publishtime": bson.M{
  420. "lte": v["endTime"],
  421. "gt": v["startTime"],
  422. },
  423. },
  424. }
  425. if index > -1 {
  426. filter.Must[index] = tmpRange_
  427. } else {
  428. filter.Must = append(filter.Must, tmpRange_)
  429. }
  430. tmpQuery := query
  431. strQuery, err := json.Marshal(tmpQuery)
  432. if err == nil {
  433. esQuery := string(strQuery)
  434. esQueryList = append(esQueryList, esQuery)
  435. } else {
  436. log.Error("失败", zap.Error(err))
  437. return err, 0
  438. }
  439. }
  440. }
  441. } else {
  442. log.Error("err", zap.Error(json.Unmarshal([]byte(esquery), &query)))
  443. }
  444. // for _, v := range esQueryList {
  445. // log.Println("es语句", v)
  446. // }
  447. log.Info("重新生成的es语句", zap.Any("", esQueryList))
  448. var totalCount int64
  449. datas := make([]map[string]interface{}, 0)
  450. for _, esSql := range esQueryList {
  451. // 查询
  452. esSql = esSql[:len(esSql)-1] + `,"size":` + fmt.Sprintf("%d", i_maxnum) + `}`
  453. log.Info("本次查询==" + esSql)
  454. index := EsIndex
  455. if strings.Contains(esSql, "site") { //选择网站名称,使用全量数据索引
  456. index = EsAllIndex
  457. }
  458. log.Info("es索引-------------------- " + index)
  459. cc := &MySource{
  460. Querys: esSql,
  461. }
  462. searchResult, err := client.Search(index).Query(cc).Do(ctx)
  463. if err == nil && searchResult.Hits != nil {
  464. log.Info("es查询到的数量:", zap.Int64("", searchResult.Hits.TotalHits.Value))
  465. log.Info("开始处理")
  466. for _, v := range searchResult.Hits.Hits {
  467. item := make(map[string]interface{})
  468. if json.Unmarshal(v.Source, &item) == nil {
  469. delete(item, "_id")
  470. item["info_id"] = common.ObjToString(item["id"])
  471. item["s_dataid"] = sdataid
  472. item["s_jyhref"] = fmt.Sprintf(Url, encrypt.CommonEncodeArticle("content", common.ObjToString(item["id"])))
  473. item["i_createtime"] = time.Now().Unix()
  474. var d *DFA = &DFA{}
  475. var analyKeys []string //找到的关键词
  476. var matchType []string //匹配方式
  477. for _, math := range maths {
  478. fileds := strsToArr(math["s_keymatch"], "field")
  479. d.AddWord(strings.Split(math["s_matchkey"], ",")...)
  480. mkMap := make(map[string]interface{})
  481. tmpMap := make(map[string]interface{})
  482. for _, mk := range strings.Split(math["s_matchkey"], ",") {
  483. if strings.Contains(mk, "&&") {
  484. arr := strings.Split(mk, "&&")
  485. for _, s := range arr {
  486. if s != "" {
  487. tmpMap[s] = mk
  488. if b, _ := regexp.MatchString("[A-Z]", s); b {
  489. mkMap[strings.ToLower(s)] = s
  490. d.AddWord(strings.ToLower(s))
  491. } else {
  492. d.AddWord(s)
  493. }
  494. }
  495. }
  496. } else {
  497. if b, _ := regexp.MatchString("[A-Z]", mk); b {
  498. mkMap[strings.ToLower(mk)] = mk
  499. d.AddWord(strings.ToLower(mk))
  500. } else {
  501. d.AddWord(mk)
  502. }
  503. }
  504. }
  505. for _, filed := range fileds {
  506. filed1 := strings.ToLower(common.ObjToString(item[filed]))
  507. ddds := d.Analy(filed1)
  508. analyKeys = append(analyKeys, ddds...)
  509. }
  510. if len(analyKeys) > 0 {
  511. matchType = append(matchType, strings.Join(fileds, ","))
  512. for k, v := range analyKeys {
  513. if tmpMap[v] != "" && tmpMap[v] != nil {
  514. analyKeys[k] = common.ObjToString(tmpMap[v])
  515. }
  516. }
  517. for _, v1 := range analyKeys {
  518. if mkMap[v1] != "" && mkMap[v1] != nil {
  519. analyKeys = deleteSlice(analyKeys, v1)
  520. analyKeys = append(analyKeys, common.ObjToString(mkMap[v1]))
  521. }
  522. }
  523. }
  524. d.Clear()
  525. }
  526. //去重
  527. ssavekey := make(map[string]bool)
  528. for _, v := range analyKeys {
  529. ssavekey[v] = true
  530. }
  531. ssavekeys := []string{}
  532. for k := range ssavekey {
  533. ssavekeys = append(ssavekeys, k)
  534. }
  535. item["s_matchkey"] = GetMactchKey(maths, item)
  536. item["s_matchtype"] = strings.Join(matchType, ",")
  537. findwinner := strings.TrimSpace(common.ObjToString(item["winner"]))
  538. if findwinner != "" {
  539. finddata, _ := MgoEnps.FindOne(EnpsColl, bson.M{"company_name": findwinner})
  540. if finddata != nil {
  541. if legal_person := common.ObjToString((*finddata)["legal_person"]); legal_person != "" {
  542. item["legal_person"] = legal_person
  543. }
  544. if email := common.ObjToString((*finddata)["company_email"]); email != "" {
  545. item["company_email"] = email
  546. }
  547. if phone := common.ObjToString((*finddata)["company_phone"]); phone != "" {
  548. item["company_phone"] = phone
  549. }
  550. //从最新年报中获取 中标单位联系电话、中标单位邮箱
  551. // if annual_reports, ok := finddata["annual_reports"].([]interface{}); ok && len(annual_reports) > 0 {
  552. // report_year := Sort_year_report(annual_reports) //最新年报
  553. // if len(report_year) > 0 {
  554. // if email := common.ObjToString(report_year["company_email"]); email != "" {
  555. // item["company_email"] = email
  556. // }
  557. // if phone := common.ObjToString(report_year["company_phone"]); phone != "" {
  558. // item["company_phone"] = phone
  559. // }
  560. // }
  561. // }
  562. item["qyk"] = finddata
  563. }
  564. }
  565. datas = append(datas, item)
  566. }
  567. }
  568. log.Info("处理完成", zap.Int("count", len(datas)))
  569. count := searchResult.Hits.TotalHits.Value
  570. totalCount += count
  571. } else {
  572. log.Error("err", zap.Error(err))
  573. return err, 0
  574. }
  575. }
  576. Mgo.Update("cuserdepartrule", bson.M{"_id": tags["_id"]}, bson.M{
  577. "$set": bson.M{
  578. "i_estotal": totalCount,
  579. }}, false, false)
  580. if int64(len(datas)) > i_maxnum {
  581. datas = datas[:i_maxnum]
  582. }
  583. return UtilEsSaveData(sdataid, "", &datas), totalCount
  584. }
  585. func GetTimeInterval(startTime, entTime time.Time, n int) []map[string]interface{} {
  586. timeList := []map[string]interface{}{}
  587. endTimeTmp := startTime
  588. for endTimeTmp.Unix() < entTime.Unix() {
  589. var endTime_ time.Time
  590. startTime = endTimeTmp
  591. endTimeTmp = startTime.AddDate(0, n, 0)
  592. if endTimeTmp.Unix() > entTime.Unix() {
  593. endTime_ = entTime
  594. timeList = append(timeList, map[string]interface{}{"startTime": startTime.Unix(), "endTime": endTime_.Unix()})
  595. break
  596. } else {
  597. endTime_ = endTimeTmp
  598. timeList = append(timeList, map[string]interface{}{"startTime": startTime.Unix(), "endTime": endTime_.Unix()})
  599. }
  600. }
  601. //log.Println(timeList)
  602. return timeList
  603. }
  604. func Utiltags(tag map[string]interface{}, id string) string {
  605. defer common.Catch()
  606. tmpbyte, _ := json.Marshal(tag)
  607. tab := models.Tag{}
  608. err := json.Unmarshal(tmpbyte, &tab)
  609. if err != nil {
  610. return "json err:" + err.Error()
  611. }
  612. //log.Println("前端数据", tab)
  613. QueryObjecct := sql.QueryObjecct{}
  614. ffBoolObject := sql.BoolObject{}
  615. adsBoolObect := sql.NewEsObject{}
  616. if tab.Sarea != "" {
  617. adsBoolObect.Bool.Should = append(adsBoolObect.Bool.Should, sql.AreaCityDistrictMust{AreaCityDistrict: &sql.AreaCityDistrict{Area: strings.Split(tab.Sarea, ",")}})
  618. }
  619. if tab.Scity != "" {
  620. adsBoolObect.Bool.Should = append(adsBoolObect.Bool.Should, sql.AreaCityDistrictMust{AreaCityDistrict: &sql.AreaCityDistrict{City: strings.Split(tab.Scity, ",")}})
  621. }
  622. if tab.Sdistrict != "" {
  623. //城市——区县
  624. cityds := strings.Split(tab.Sdistrict, ",")
  625. var ds []string
  626. for _, v := range cityds {
  627. ds = append(ds, strings.Split(v, "-")[1])
  628. }
  629. adsBoolObect.Bool.Should = append(adsBoolObect.Bool.Should, sql.AreaCityDistrictMust{AreaCityDistrict: &sql.AreaCityDistrict{District: ds}})
  630. }
  631. if len(adsBoolObect.Bool.Should) > 0 {
  632. ffBoolObject.Must = append(ffBoolObject.Must, adsBoolObect)
  633. }
  634. if tab.Stoptype != "" || tab.Ssubtype != "" {
  635. if len(tab.Stoptype) > 0 {
  636. toptypeSubtype := sql.ToptypeSubtype{}
  637. toptypeSubtype.Toptype = strings.Split(tab.Stoptype, ",")
  638. ffBoolObject.Must = append(ffBoolObject.Must, sql.ToptypeSubtypeMust{&toptypeSubtype})
  639. }
  640. if len(tab.Ssubtype) > 0 {
  641. toptypeSubtype := sql.ToptypeSubtype{}
  642. toptypeSubtype.Subtype = strings.Split(tab.Ssubtype, ",")
  643. ffBoolObject.Must = append(ffBoolObject.Must, sql.ToptypeSubtypeMust{&toptypeSubtype})
  644. }
  645. }
  646. if id == "67e64794756e2b3b3b6d5062" {
  647. siteArr := []string{"全军武器装备采购信息网", "军队采购网新网址"}
  648. ffBoolObject.Must = append(ffBoolObject.Must, sql.SiteMust{Site: &sql.Site{Site: siteArr}})
  649. } else {
  650. if tab.Site != "" {
  651. ffBoolObject.Must = append(ffBoolObject.Must, sql.SiteMust{Site: &sql.Site{Site: strings.Split(tab.Site, ",")}})
  652. }
  653. }
  654. if tab.Istarttime > 0 && tab.Iendtime > 0 {
  655. ffBoolObject.Must = append(ffBoolObject.Must, sql.PublishtimeMust{PublishtimeObject: &sql.PublishtimeObject{Publishtime: &sql.Publishtime{
  656. Gte: tab.Istarttime,
  657. Lt: tab.Iendtime,
  658. }}})
  659. } else if tab.Istarttime > 0 {
  660. ffBoolObject.Must = append(ffBoolObject.Must, sql.PublishtimeMust{PublishtimeObject: &sql.PublishtimeObject{Publishtime1: &sql.Publishtime1{
  661. Gte: tab.Istarttime,
  662. }}})
  663. } else if tab.Iendtime > 0 {
  664. ffBoolObject.Must = append(ffBoolObject.Must, sql.PublishtimeMust{PublishtimeObject: &sql.PublishtimeObject{Publishtime2: &sql.Publishtime2{
  665. Lt: tab.Iendtime,
  666. }}})
  667. }
  668. if tab.Sglobalbuyerclass != "" {
  669. if len(tab.Sglobalbuyerclass) > 0 {
  670. buyerclass := sql.BuyerclassObject{
  671. Terms: struct {
  672. Buyerclass []string `json:"buyerclass,omitempty"`
  673. }{
  674. Buyerclass: strings.Split(tab.Sglobalbuyerclass, ","),
  675. },
  676. }
  677. ffBoolObject.Must = append(ffBoolObject.Must, buyerclass)
  678. }
  679. }
  680. if tab.Sglobaltopscopeclass != "" || tab.Sglobalsubscopeclass != "" {
  681. if len(tab.Sglobaltopscopeclass) > 0 {
  682. topScopeclass := sql.Scopeclass{}
  683. topScopeclass.Globaltopscopeclass = strings.Split(tab.Sglobaltopscopeclass, ",")
  684. ffBoolObject.Must = append(ffBoolObject.Must, sql.ScopeclassMust{&topScopeclass})
  685. } else if len(tab.Sglobalsubscopeclass) > 0 {
  686. subScopeclass := sql.Scopeclass{}
  687. subScopeclass.Globalsubscopeclass = strings.Split(tab.Sglobalsubscopeclass, ",")
  688. ffBoolObject.Must = append(ffBoolObject.Must, sql.ScopeclassMust{&subScopeclass})
  689. }
  690. }
  691. fqBoolObject := sql.BoolObject{}
  692. if len(tab.Sexistfields) > 0 {
  693. tmpsfields := strings.Split(tab.Sexistfields, ",")
  694. for _, v := range tmpsfields {
  695. // fqBoolObject.MustNot = append(fqBoolObject.MustNot, sql.ExistfieldsObjectMust{
  696. // ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  697. // Missing struct {
  698. // Field string `json:"field,omitempty"`
  699. // } `json:"missing,omitempty"`
  700. // }{
  701. // Missing: struct {
  702. // Field string `json:"field,omitempty"`
  703. // }{Field: v},
  704. // },
  705. // },
  706. // })
  707. ffBoolObject.Must = append(ffBoolObject.Must, sql.ExistfieldsObjectMust{
  708. ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  709. Exists struct {
  710. Field string `json:"field,omitempty"`
  711. } `json:"exists,omitempty"`
  712. }{
  713. Exists: struct {
  714. Field string `json:"field,omitempty"`
  715. }{Field: v},
  716. },
  717. },
  718. })
  719. }
  720. }
  721. newEsObject := sql.NewEsObject{}
  722. if tab.Sglobaladdkey != "" && tab.Sglobaladdkeymatch != "" {
  723. if tmps := strsToArr(tab.Sglobaladdkeymatch, "str"); tmps != nil {
  724. newEsObject = method1(newEsObject, tab.Sglobaladdkey, tab.Sglobaladdkeymatch, tmps)
  725. }
  726. }
  727. if len(newEsObject.Bool.Should) > 0 || len(newEsObject.Bool.MustNot) > 0 || len(newEsObject.Bool.Must) > 0 {
  728. fqBoolObject.Must = append(fqBoolObject.Must, newEsObject)
  729. }
  730. nots := sql.NewEsObject{}
  731. //
  732. if tab.ExpurasingtimeStart > 0 && tab.ExpurasingtimeEnd > 0 {
  733. ffBoolObject.MustNot = append(ffBoolObject.MustNot, sql.ExpurasingtimeMust{ExpurasingtimeObject: &sql.ExpurasingtimeObject{Expurasingtime: &sql.Expurasingtime{
  734. Gte: tab.ExpurasingtimeStart,
  735. Lt: tab.ExpurasingtimeEnd,
  736. }}})
  737. } else if tab.ExpurasingtimeStart > 0 {
  738. ffBoolObject.MustNot = append(ffBoolObject.MustNot, sql.ExpurasingtimeMust{ExpurasingtimeObject: &sql.ExpurasingtimeObject{Expurasingtime1: &sql.Expurasingtime1{
  739. Gte: tab.ExpurasingtimeStart,
  740. }}})
  741. } else if tab.ExpurasingtimeEnd > 0 {
  742. ffBoolObject.MustNot = append(ffBoolObject.MustNot, sql.ExpurasingtimeMust{ExpurasingtimeObject: &sql.ExpurasingtimeObject{Expurasingtime2: &sql.Expurasingtime2{
  743. Lt: tab.ExpurasingtimeEnd,
  744. }}})
  745. }
  746. //
  747. if tab.Sglobalnotkey != "" && tab.Sglobalnotkeymatch != "" {
  748. if tmps := strsToArr(tab.Sglobalnotkeymatch, "str"); tmps != nil {
  749. nots = method1(nots, tab.Sglobalnotkey, tab.Sglobalnotkeymatch, tmps)
  750. }
  751. }
  752. if len(nots.Bool.Should) > 0 {
  753. fqBoolObject.MustNot = append(fqBoolObject.MustNot, nots)
  754. }
  755. torules := sql.BoolObject{}
  756. for _, v := range tab.Orules {
  757. tmpses := sql.NewEsObject{}
  758. if skeymatchs := strsToArr(v.Skeymatch, "str"); skeymatchs != nil {
  759. tmpnewEsObject := sql.NewEsObject{}
  760. tmpnewEsObject = method1(tmpnewEsObject, v.Smatchkey, v.Skeymatch, skeymatchs)
  761. if len(tmpnewEsObject.Bool.Should) > 0 || len(tmpnewEsObject.Bool.Must) > 0 || len(tmpnewEsObject.Bool.MustNot) > 0 {
  762. tmpses.Bool.Must = append(tmpses.Bool.Must, tmpnewEsObject)
  763. }
  764. }
  765. if saddkeymatchs := strsToArr(v.Saddkeymatch, "str"); saddkeymatchs != nil {
  766. addkeyarr := sql.NewEsObject{}
  767. addkeyarr = method1(addkeyarr, v.Saddkey, v.Saddkeymatch, saddkeymatchs)
  768. if len(addkeyarr.Bool.Should) > 0 {
  769. tmpses.Bool.Must = append(tmpses.Bool.Must, addkeyarr)
  770. }
  771. }
  772. if snotkeymatchs := strsToArr(v.Snotkeymatch, "str"); snotkeymatchs != nil {
  773. tmpses = method2(tmpses, v.Snotkey, v.Snotkeymatch, snotkeymatchs)
  774. }
  775. if len(v.Stopscopeclass) > 0 {
  776. sArr := strings.Split(v.Stopscopeclass, ",")
  777. tmpses.Bool.Must = append(tmpses.Bool.Must, sql.TopscopeclassObject{
  778. Terms: struct {
  779. Topscopeclass []string `json:"s_topscopeclass,omitempty"`
  780. }{
  781. Topscopeclass: sArr,
  782. },
  783. })
  784. } else if len(v.Ssubscopeclass) > 0 {
  785. tmpses.Bool.Must = append(tmpses.Bool.Must, sql.SubscopeclassObject{
  786. Terms: struct {
  787. Subscopeclass []string `json:"s_subscopeclass,omitempty"`
  788. }{
  789. Subscopeclass: strings.Split(v.Ssubscopeclass, ","),
  790. },
  791. })
  792. }
  793. if len(v.Sbuyerclass) > 0 {
  794. tmpses.Bool.Must = append(tmpses.Bool.Must, sql.BuyerclassObject{
  795. Terms: struct {
  796. Buyerclass []string `json:"buyerclass,omitempty"`
  797. }{
  798. Buyerclass: strings.Split(v.Sbuyerclass, ","),
  799. },
  800. })
  801. }
  802. torules.Should = append(torules.Should, tmpses)
  803. }
  804. bidamountSql := sql.NewEsObject{}
  805. if tab.Sbidamount != "" {
  806. if strings.Contains(tab.Sbidamount, "大于") && strings.Contains(tab.Sbidamount, "小于") {
  807. arr := strings.Split(tab.Sbidamount, ",")
  808. limit := &sql.BudgetOrBidamount{
  809. Gte: common.Float64All(strings.Replace(arr[0], "大于", "", -1)),
  810. Lt: common.Float64All(strings.Replace(arr[1], "小于", "", -1)),
  811. }
  812. bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}})
  813. } else if strings.Contains(tab.Sbidamount, "大于") {
  814. limit := &sql.BudgetOrBidamount{
  815. Gte: common.Float64All(strings.Replace(tab.Sbidamount, "大于", "", -1)),
  816. }
  817. bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}})
  818. } else if strings.Contains(tab.Sbidamount, "小于") {
  819. limit := &sql.BudgetOrBidamount{
  820. Lt: common.Float64All(strings.Replace(tab.Sbidamount, "小于", "", -1)),
  821. }
  822. bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}})
  823. }
  824. if tab.SbidamountFieldExist == 1 {
  825. // bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.ExistfieldsObjectMust{
  826. // ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  827. // Missing struct {
  828. // Field string `json:"field,omitempty"`
  829. // } `json:"missing,omitempty"`
  830. // }{
  831. // Missing: struct {
  832. // Field string `json:"field,omitempty"`
  833. // }{Field: "bidamount"},
  834. // },
  835. // },
  836. // })
  837. SbidamountFieldExistBool := sql.NewEsObject{}
  838. SbidamountFieldExistBool.Bool.MustNot = append(SbidamountFieldExistBool.Bool.MustNot, sql.ExistfieldsObjectMust{
  839. ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  840. Exists struct {
  841. Field string `json:"field,omitempty"`
  842. } `json:"exists,omitempty"`
  843. }{
  844. Exists: struct {
  845. Field string `json:"field,omitempty"`
  846. }{Field: "bidamount"},
  847. },
  848. },
  849. })
  850. bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, SbidamountFieldExistBool)
  851. }
  852. }
  853. budgetSql := sql.NewEsObject{}
  854. if tab.Sbudget != "" {
  855. if strings.Contains(tab.Sbudget, "大于") && strings.Contains(tab.Sbudget, "小于") {
  856. arr := strings.Split(tab.Sbudget, ",")
  857. limit := &sql.BudgetOrBidamount{
  858. Gte: common.Float64All(strings.Replace(arr[0], "大于", "", -1)),
  859. Lt: common.Float64All(strings.Replace(arr[1], "小于", "", -1)),
  860. }
  861. budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}})
  862. } else if strings.Contains(tab.Sbudget, "大于") {
  863. limit := &sql.BudgetOrBidamount{
  864. Gte: common.Float64All(strings.Replace(tab.Sbudget, "大于", "", -1)),
  865. }
  866. budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}})
  867. } else if strings.Contains(tab.Sbudget, "小于") {
  868. limit := &sql.BudgetOrBidamount{
  869. Lt: common.Float64All(strings.Replace(tab.Sbudget, "小于", "", -1)),
  870. }
  871. budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}})
  872. }
  873. if tab.SbudgetFieldExist == 1 {
  874. // budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.ExistfieldsObjectMust{
  875. // ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  876. // Missing struct {
  877. // Field string `json:"field,omitempty"`
  878. // } `json:"missing,omitempty"`
  879. // }{
  880. // Missing: struct {
  881. // Field string `json:"field,omitempty"`
  882. // }{Field: "budget"},
  883. // },
  884. // },
  885. // })
  886. SbudgetFieldExistBool := sql.NewEsObject{}
  887. SbudgetFieldExistBool.Bool.MustNot = append(SbudgetFieldExistBool.Bool.MustNot, sql.ExistfieldsObjectMust{
  888. ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
  889. Exists struct {
  890. Field string `json:"field,omitempty"`
  891. } `json:"exists,omitempty"`
  892. }{
  893. Exists: struct {
  894. Field string `json:"field,omitempty"`
  895. }{Field: "budget"},
  896. },
  897. },
  898. })
  899. budgetSql.Bool.Should = append(budgetSql.Bool.Should, SbudgetFieldExistBool)
  900. }
  901. }
  902. if len(ffBoolObject.Must) > 0 || len(ffBoolObject.MustNot) > 0 || len(ffBoolObject.Should) > 0 {
  903. // QueryObjecct.Filter = &sql.Filter{}
  904. QueryObjecct.Bool = &ffBoolObject
  905. }
  906. if len(fqBoolObject.Must) > 0 || len(fqBoolObject.MustNot) > 0 || len(fqBoolObject.Should) > 0 {
  907. if QueryObjecct.Bool == nil {
  908. QueryObjecct.Bool = &fqBoolObject
  909. } else {
  910. if fqBoolObject.Must != nil {
  911. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, fqBoolObject.Must...)
  912. }
  913. if fqBoolObject.Should != nil {
  914. QueryObjecct.Bool.Should = append(QueryObjecct.Bool.Should, fqBoolObject.Should...)
  915. }
  916. if fqBoolObject.MustNot != nil {
  917. QueryObjecct.Bool.MustNot = append(QueryObjecct.Bool.MustNot, fqBoolObject.MustNot...)
  918. }
  919. }
  920. if len(torules.Should) > 0 {
  921. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, map[string]interface{}{
  922. "bool": torules,
  923. })
  924. }
  925. } else if len(torules.Should) > 0 {
  926. if QueryObjecct.Bool == nil {
  927. // QueryObjecct.Query = &sql.Query{}
  928. QueryObjecct.Bool = &sql.BoolObject{}
  929. }
  930. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, map[string]interface{}{
  931. "bool": torules,
  932. })
  933. }
  934. if (len(budgetSql.Bool.Should) > 0 || len(bidamountSql.Bool.Should) > 0) && (len(fqBoolObject.Must) > 0 || len(fqBoolObject.MustNot) > 0 || len(fqBoolObject.Should) > 0) {
  935. if QueryObjecct.Bool == nil {
  936. QueryObjecct.Bool = &sql.BoolObject{}
  937. }
  938. if len(budgetSql.Bool.Should) > 0 {
  939. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, budgetSql)
  940. }
  941. if len(bidamountSql.Bool.Should) > 0 {
  942. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, bidamountSql)
  943. }
  944. } else {
  945. if QueryObjecct.Bool == nil {
  946. QueryObjecct.Bool = &sql.BoolObject{}
  947. }
  948. if len(budgetSql.Bool.Should) > 0 {
  949. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, budgetSql)
  950. }
  951. if len(bidamountSql.Bool.Should) > 0 {
  952. QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, bidamountSql)
  953. }
  954. }
  955. rdata := make(map[string]interface{})
  956. rdata["query"] = QueryObjecct
  957. esbytes, _ := json.Marshal(rdata)
  958. esStr := strings.Replace(string(esbytes), "regexp1", "regexp", -1)
  959. esStr = strings.Replace(esStr, "publishtime1", "publishtime", -1)
  960. esStr = strings.Replace(esStr, "publishtime2", "publishtime", -1)
  961. esStr = strings.Replace(esStr, "expurasingtime1", "expurasingtime", -1)
  962. esStr = strings.Replace(esStr, "expurasingtime2", "expurasingtime", -1)
  963. return strings.Replace(esStr, "regexp2", "regexp", -1)
  964. }
  965. func strsToArr(strs string, typestr string) []string {
  966. if len(strs) > 0 {
  967. tmps := strings.Split(strs, ",")
  968. switch typestr {
  969. case "str":
  970. for index, vint := range tmps {
  971. switch vint {
  972. case TitleMatchType:
  973. tmps[index] = TitleMatchTypeStr
  974. case DetailMatchType:
  975. tmps[index] = DetailMatchTypeStr
  976. case PurchasingMatchType:
  977. tmps[index] = PurchasingMatchTypeStr
  978. case AttachmentsMatchType:
  979. tmps[index] = AttachmentsMatchTypeStr
  980. case ProjectnameMatchType:
  981. tmps[index] = ProjectnameMatchTypeStr
  982. case BuyerMatchType:
  983. tmps[index] = BuyerMatchTypeStr
  984. case SWinnerMatchType:
  985. tmps[index] = SWinnerMatchTypeStr
  986. case BuyerMatchType1:
  987. tmps[index] = BuyerRegexpMatchTypeStr
  988. case SWinnerMatchType1:
  989. tmps[index] = SWinnerRegexpMatchTypeStr
  990. case ProjectnameMatchTypes:
  991. tmps[index] = ProjectnameMatchTypesField
  992. case ProjectscopeMatchTypes:
  993. tmps[index] = ProjectscopeMatchTypesField
  994. case BuyerMatchTypes:
  995. tmps[index] = BuyerMatchTypesField
  996. }
  997. }
  998. return tmps
  999. case "field":
  1000. for index, vint := range tmps {
  1001. switch vint {
  1002. case TitleMatchType:
  1003. tmps[index] = TitleMatchTypeField
  1004. case DetailMatchType:
  1005. tmps[index] = DetailMatchTypeField
  1006. case PurchasingMatchType:
  1007. tmps[index] = PurchasingMatchTypeField
  1008. case AttachmentsMatchType:
  1009. tmps[index] = AttachmentsMatchTypeField
  1010. case ProjectnameMatchType:
  1011. tmps[index] = ProjectnameMatchTypeField
  1012. case BuyerMatchType:
  1013. tmps[index] = BuyerMatchTypeField
  1014. case SWinnerMatchType:
  1015. tmps[index] = SWinnerMatchTypeField
  1016. case BuyerMatchType1:
  1017. tmps[index] = BuyerMatchTypeField
  1018. case SWinnerMatchType1:
  1019. tmps[index] = SWinnerMatchTypeField
  1020. case ProjectnameMatchTypes:
  1021. tmps[index] = ProjectnameMatchTypesField
  1022. case ProjectscopeMatchTypes:
  1023. tmps[index] = ProjectscopeMatchTypesField
  1024. case BuyerMatchTypes:
  1025. tmps[index] = BuyerMatchTypesField
  1026. }
  1027. }
  1028. return tmps
  1029. }
  1030. }
  1031. return nil
  1032. }
  1033. func init() {
  1034. EsIndex = common.ObjToString(Sysconfig["elasticsearch_index"])
  1035. EsType = common.ObjToString(Sysconfig["elasticsearch_type"])
  1036. }
  1037. func method1(newEsObject sql.NewEsObject, keyword, keymatch string, tmps []string) sql.NewEsObject {
  1038. keywordArr := []string{}
  1039. for _, vv := range strings.Split(keyword, ",") {
  1040. if vv == "" {
  1041. continue
  1042. }
  1043. //log.Println(vv, tmps, keymatch)
  1044. if len(tmps) == 1 && (strings.Contains(keymatch, "8") || strings.Contains(keymatch, "9")) {
  1045. //中标单位、采购单位、中标单位(正则)、采购单位(正则) 单选
  1046. strs := ""
  1047. if strings.Contains(vv, "&&") {
  1048. //strs = ".*" + strings.Replace(vv, "&&", ".*", -1) + ".*"
  1049. strs = strings.Replace(vv, "&&", ".*", -1)
  1050. } else {
  1051. strs = vv
  1052. }
  1053. if tmps[0] == "buyer.mbuyer" {
  1054. newEsObject.Bool.Should = append(newEsObject.Bool.Should, sql.ShouldObj{
  1055. MultiMatch: &sql.MultiMatch{
  1056. Query: strs,
  1057. Type: MultiMatchType,
  1058. Fields: tmps,
  1059. },
  1060. })
  1061. } else if tmps[0] == "s_winner.mwinner" {
  1062. newEsObject.Bool.Should = append(newEsObject.Bool.Should, sql.ShouldObj{
  1063. MultiMatch: &sql.MultiMatch{
  1064. Query: strs,
  1065. Type: MultiMatchType,
  1066. Fields: tmps,
  1067. },
  1068. })
  1069. }
  1070. } else if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) {
  1071. addkeylines := strings.Split(vv, "&&")
  1072. keywordArr = append(keywordArr, addkeylines[0])
  1073. } else {
  1074. addkeylines := strings.Split(vv, "&&")
  1075. //log.Println(addkeylines, "addkeylines")
  1076. if len(addkeylines) > 1 {
  1077. addkeyline := sql.NewEsObject{}
  1078. for _, vvv := range addkeylines {
  1079. if vvv == "" {
  1080. continue
  1081. }
  1082. addkeyline.Bool.Must = append(addkeyline.Bool.Must, sql.ShouldObj{
  1083. MultiMatch: &sql.MultiMatch{
  1084. Query: vvv,
  1085. Type: MultiMatchType,
  1086. Fields: tmps,
  1087. },
  1088. })
  1089. }
  1090. if len(addkeyline.Bool.Must) > 0 {
  1091. newEsObject.Bool.Should = append(newEsObject.Bool.Should, addkeyline)
  1092. }
  1093. } else {
  1094. newEsObject.Bool.Should = append(newEsObject.Bool.Should, sql.ShouldObj{
  1095. MultiMatch: &sql.MultiMatch{
  1096. Query: vv,
  1097. Type: MultiMatchType,
  1098. Fields: tmps,
  1099. },
  1100. })
  1101. }
  1102. }
  1103. }
  1104. if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) {
  1105. addkeyline := sql.NewEsObject{}
  1106. if tmps[0] == "buyer" {
  1107. addkeyline.Bool.Must = append(addkeyline.Bool.Must, sql.BuyerMatch{
  1108. Buyer: &sql.Buyer{Buyer: keywordArr},
  1109. })
  1110. } else if tmps[0] == "s_winner" {
  1111. addkeyline.Bool.Must = append(addkeyline.Bool.Must, sql.WinnerMatch{
  1112. Winner: &sql.Winner{Winner: keywordArr},
  1113. })
  1114. }
  1115. if len(addkeyline.Bool.Must) > 0 {
  1116. newEsObject.Bool.Should = append(newEsObject.Bool.Should, addkeyline)
  1117. }
  1118. }
  1119. return newEsObject
  1120. }
  1121. func method2(newEsObject sql.NewEsObject, keyword, keymatch string, tmps []string) sql.NewEsObject {
  1122. notKeyWordArr := []string{}
  1123. for _, vv := range strings.Split(keyword, ",") {
  1124. if vv == "" {
  1125. continue
  1126. }
  1127. if len(tmps) == 1 && (strings.Contains(keymatch, "8") || strings.Contains(keymatch, "9")) {
  1128. //中标单位、采购单位、中标单位(正则)、采购单位(正则) 单选
  1129. //strs := ""
  1130. //if strings.Contains(vv, "&&") {
  1131. // strs = ".*" + strings.Replace(vv, "&&", ".*", -1) + ".*"
  1132. //} else {
  1133. // strs = ".*" + vv + ".*"
  1134. //}
  1135. if tmps[0] == "buyer.mbuyer" {
  1136. newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, sql.ShouldObj{
  1137. MultiMatch: &sql.MultiMatch{
  1138. Query: vv,
  1139. Type: MultiMatchType,
  1140. Fields: tmps,
  1141. },
  1142. })
  1143. } else if tmps[0] == "s_winner.mwinner" {
  1144. newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, sql.ShouldObj{
  1145. MultiMatch: &sql.MultiMatch{
  1146. Query: vv,
  1147. Type: MultiMatchType,
  1148. Fields: tmps,
  1149. },
  1150. })
  1151. }
  1152. } else if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) {
  1153. notkeylines := strings.Split(vv, "&&")
  1154. notKeyWordArr = append(notKeyWordArr, notkeylines[0])
  1155. } else {
  1156. notkeylines := strings.Split(vv, "&&")
  1157. if len(notkeylines) > 1 {
  1158. notkeyline := sql.NewEsObject{}
  1159. for _, vvv := range notkeylines {
  1160. if vvv == "" {
  1161. continue
  1162. }
  1163. notkeyline.Bool.Must = append(notkeyline.Bool.Must, sql.ShouldObj{
  1164. MultiMatch: &sql.MultiMatch{
  1165. Query: vvv,
  1166. Type: MultiMatchType,
  1167. Fields: tmps,
  1168. },
  1169. })
  1170. }
  1171. if len(notkeyline.Bool.Must) > 0 {
  1172. newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, notkeyline)
  1173. }
  1174. } else {
  1175. newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, sql.ShouldObj{
  1176. MultiMatch: &sql.MultiMatch{
  1177. Query: vv,
  1178. Type: MultiMatchType,
  1179. Fields: tmps,
  1180. },
  1181. })
  1182. }
  1183. }
  1184. }
  1185. if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) {
  1186. var notkeyline interface{}
  1187. if tmps[0] == "buyer" {
  1188. notkeyline = sql.BuyerMatch{
  1189. Buyer: &sql.Buyer{Buyer: notKeyWordArr},
  1190. }
  1191. } else if tmps[0] == "s_winner" {
  1192. notkeyline = sql.WinnerMatch{
  1193. Winner: &sql.Winner{Winner: notKeyWordArr},
  1194. }
  1195. }
  1196. if notkeyline != nil {
  1197. newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, notkeyline)
  1198. }
  1199. }
  1200. return newEsObject
  1201. }
  1202. /**
  1203. * 一级公告行业处理 d、t、p
  1204. */
  1205. func method3(arr []string) []string {
  1206. var sArr []string
  1207. for _, v := range arr {
  1208. sArr = append(sArr, v+"d")
  1209. sArr = append(sArr, v+"t")
  1210. sArr = append(sArr, v+"p")
  1211. }
  1212. return sArr
  1213. }
  1214. func GetMactchKey(match []map[string]string, data map[string]interface{}) string {
  1215. keyWord := []string{}
  1216. for _, keys := range match {
  1217. types := keys["s_keymatch"]
  1218. key := keys["s_matchkey"]
  1219. if strings.Contains(types, "1") {
  1220. title := common.ObjToString(data["title"])
  1221. keyWord = KeyWordToData(types, title, key, keyWord)
  1222. }
  1223. if strings.Contains(types, "2") {
  1224. detail := common.ObjToString(data["detail"])
  1225. keyWord = KeyWordToData(types, detail, key, keyWord)
  1226. }
  1227. if strings.Contains(types, "3") {
  1228. purchasing := common.ObjToString(data["purchasing"])
  1229. keyWord = KeyWordToData(types, purchasing, key, keyWord)
  1230. }
  1231. if strings.Contains(types, "4") {
  1232. filetext := common.ObjToString(data["filetext"])
  1233. keyWord = KeyWordToData(types, filetext, key, keyWord)
  1234. }
  1235. if strings.Contains(types, "5") {
  1236. projectname := common.ObjToString(data["projectname"])
  1237. keyWord = KeyWordToData(types, projectname, key, keyWord)
  1238. }
  1239. if strings.Contains(types, "6") || strings.Contains(types, "8") {
  1240. buyer := common.ObjToString(data["buyer"])
  1241. keyWord = KeyWordToData(types, buyer, key, keyWord)
  1242. }
  1243. if strings.Contains(types, "7") || strings.Contains(types, "9") {
  1244. winner := common.ObjToString(data["s_winner"])
  1245. keyWord = KeyWordToData(types, winner, key, keyWord)
  1246. }
  1247. }
  1248. keyMap := map[string]bool{}
  1249. keyArr := []string{}
  1250. for _, key := range keyWord {
  1251. keyMap[key] = true
  1252. }
  1253. for k, _ := range keyMap {
  1254. keyArr = append(keyArr, k)
  1255. }
  1256. return strings.Join(keyArr, ",")
  1257. }
  1258. func KeyWordToData(types, item, key string, keyWord []string) []string {
  1259. for _, mk := range strings.Split(key, ",") {
  1260. if strings.Contains(mk, "&&") {
  1261. arr := strings.Split(mk, "&&")
  1262. isok := true
  1263. for _, s := range arr {
  1264. if s != "" {
  1265. if !strings.Contains(strings.ToUpper(item), strings.ToUpper(s)) {
  1266. isok = false
  1267. }
  1268. }
  1269. }
  1270. if isok {
  1271. keyWord = append(keyWord, mk)
  1272. }
  1273. } else {
  1274. spaceReg := regexp.MustCompile("[\\s\\n \u3000\u2003\u00a0]+")
  1275. itemUpper := strings.ToUpper(item)
  1276. mkUpper := strings.ToUpper(mk)
  1277. matchResult := strings.Contains(spaceReg.ReplaceAllString(itemUpper, ""), spaceReg.ReplaceAllString(mkUpper, ""))
  1278. if matchResult {
  1279. keyWord = append(keyWord, mk)
  1280. }
  1281. }
  1282. }
  1283. return keyWord
  1284. }