package util import ( "cmplatform/models" sql "cmplatform/util/sqlmodel" "context" "encoding/json" "errors" "fmt" "regexp" "strings" "time" "app.yhyue.com/moapp/jybase/common" "app.yhyue.com/moapp/jybase/encrypt" elastic "app.yhyue.com/moapp/jybase/es" "app.yhyue.com/moapp/jybase/log" "go.mongodb.org/mongo-driver/bson" "go.uber.org/zap" ) const ( TitleMatchType = "1" //标题匹配 DetailMatchType = "2" //全文匹配 PurchasingMatchType = "3" //标的物匹配 AttachmentsMatchType = "4" //附件匹配 ProjectnameMatchType = "5" //项目名称匹配 BuyerMatchType = "6" //采购单位匹配 SWinnerMatchType = "7" //中标单位匹配 BuyerMatchType1 = "8" //采购单位(正则)匹配 SWinnerMatchType1 = "9" //中标单位(正则)匹配 ProjectnameMatchTypes = "10" //采购意向名称匹配 ProjectscopeMatchTypes = "11" //采购意向内容匹配 BuyerMatchTypes = "12" //采购意向采购单位匹配 TitleMatchTypeStr = "title" //标题匹配 DetailMatchTypeStr = "detail" //全文匹配 PurchasingMatchTypeStr = "purchasing" //标的物匹配 AttachmentsMatchTypeStr = "filetext" //附件匹配 ProjectnameMatchTypeStr = "projectname.pname" //项目名称匹配 BuyerMatchTypeStr = "buyer" //项目名称匹配 BuyerRegexpMatchTypeStr = "buyer.mbuyer" //采购单位正则匹配 SWinnerMatchTypeStr = "s_winner" //项目名称匹配 SWinnerRegexpMatchTypeStr = "s_winner.mwinner" //中标单位正则匹配 TitleMatchTypeField = "title" //标题匹配 DetailMatchTypeField = "detail" //全文匹配 PurchasingMatchTypeField = "purchasing" //标的物匹配 AttachmentsMatchTypeField = "filetext" //附件匹配 ProjectnameMatchTypeField = "projectname" //项目名称匹配 BuyerMatchTypeField = "buyer" //项目名称匹配 SWinnerMatchTypeField = "s_winner" //项目名称匹配 ProjectnameMatchTypesField = "procurementlist.projectname" //采购意向名称匹配 ProjectscopeMatchTypesField = "procurementlist.projectscope" //采购意向内容匹配 BuyerMatchTypesField = "procurementlist.buyer" //采购意向采购单位匹配 MultiMatchType = "phrase" Standard = 1 //1:标准字段包, Advanced = 2 //2:高级字段包 Standardstr = "standard" //1:标准字段包 Advancedstr = "advanced" //2:高级字段包 Url = "https://www.jianyu360.cn/article/content/%s.html" ) func UtilEsSaveData(sdataid, index string, datas *[]map[string]interface{}) error { defer common.Catch() if !Mgo.Del("tagsdata", bson.M{"s_dataid": sdataid, "esIndex": index}) { return errors.New("mongo del err") } for _, v := range *datas { v["esIndex"] = index Mgo.Save("tagsdata", v) } return nil } // 客户规则 func UtilEsFind1(tags map[string]interface{}, dataType, userId string) (error, string, int64) { defer common.Catch() sdataid := common.ObjToString(tags["s_dataid"]) esquery := common.ObjToString(tags["s_esquery"]) if IsNewSql != 0 { esquery = common.ObjToString(tags["s_esquery_search"]) } if len(esquery) < 1 || len(sdataid) < 1 { return errors.New("s_esquery or s_dataid no found"), "", 0 } i_maxnum := common.Int64All(tags["i_maxnum"]) if i_maxnum <= 0 { i_maxnum = 100 } maths := make([]map[string]string, 0) if orules, ok := tags["o_rules"].([]interface{}); ok { for _, v := range orules { orule, _ := v.(map[string]interface{}) maths = append(maths, map[string]string{ "s_matchkey": common.ObjToString(orule["s_matchkey"]), "s_keymatch": common.ObjToString(orule["s_keymatch"]), }) } } else if orules, ok := tags["o_rules"].([]map[string]interface{}); ok { for _, v := range orules { maths = append(maths, map[string]string{ "s_matchkey": common.ObjToString(v["s_matchkey"]), "s_keymatch": common.ObjToString(v["s_keymatch"]), }) } } else { return errors.New("o_rules no found"), "", 0 } index := EsIndex if strings.Contains(esquery, "site") { //选择网站名称,使用全量数据索引 index = EsAllIndex } dataTypes := "" if dataType == "1" { index = InterimIndex dataTypes = "1" startTime := time.Now().Unix() - 86400*90 endTime := time.Now().Unix() esquery, _ = FilterTimeSql(esquery, startTime, endTime) } else { index = TotalIndex dataTypes = "2" } // esquery = esquery[:len(esquery)-1] + `,"size":` + fmt.Sprintf("%d", i_maxnum) + `}` err, counts := searchDataArr(index, esquery, sdataid, userId, i_maxnum, tags, maths) if counts == 0 && err == nil && index == InterimIndex { err, counts = searchDataArr(TotalIndex, esquery, sdataid, userId, i_maxnum, tags, maths) if counts > 0 { dataTypes = "2" } } return err, dataTypes, counts } func searchDataArr(index, esquery, sdataid, userId string, i_maxnum int64, tags map[string]interface{}, maths []map[string]string) (error, int64) { var ( err error counts = int64(0) // times = 0 // times2 = 0 ) // for { // listLen := redis.GetInt("session", "es_status") // if listLen == 0 { // log.Println("es空闲!") // break // } else if times > 10 { // err = errors.New("系统繁忙,请稍后再试") // break // } else { // log.Println("es繁忙,", listLen) // } // times += 2 // time.Sleep(2 * time.Second) // } // for { // listLens := int(redis.LLEN("datag", "jyqyfw_es_query")) // if listLens < 2 { // if isExists, _ := redis.Exists("datag", "jyqyfw_es_query_times_"+userId); isExists { // qt := redis.GetInt("datag", "jyqyfw_es_query_times_"+userId) // if qt > 2 { // log.Println("单个账号一分钟内超过3次 ", qt) // err = errors.New("系统繁忙,请稍后再试") // break // } else { // redis.Incr("datag", "jyqyfw_es_query_times_"+userId) // } // } else { // redis.Put("datag", "jyqyfw_es_query_times_"+userId, 1, 60) // } // log.Println("es空闲,当前进程数 ", listLens) // redis.RPUSH("datag", "jyqyfw_es_query", 1) err, counts = searchData(index, esquery, sdataid, i_maxnum, tags, maths) // redis.LPOP("datag", "jyqyfw_es_query") // break // } else if times2 > 10 { // err = errors.New("系统繁忙,请稍后再试") // break // } else { // log.Println("企业级服务es进程数过多,", listLens) // } // times2 += 2 // time.Sleep(2 * time.Second) // } return err, counts } type MySource struct { Querys string } func (m *MySource) Source() (interface{}, error) { mp := make(map[string]interface{}) json.Unmarshal([]byte(m.Querys), &mp) return mp["query"], nil } func searchData(index, esquery, sdataid string, i_maxnum int64, tags map[string]interface{}, maths []map[string]string) (error, int64) { esCon := elastic.VarEs.(*elastic.EsV7) client := esCon.GetEsConn() defer esCon.DestoryEsConn(client) ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute) cc := &MySource{ Querys: esquery, } searchResult, err := client.Search(index).Query(cc).Size(int(i_maxnum)).Do(ctx) if err == nil && searchResult.Hits != nil { datas := make([]map[string]interface{}, 0) log.Info(fmt.Sprintf("es查询到的数量:%d", searchResult.Hits.TotalHits.Value)) log.Info(fmt.Sprintf("es查询数据数量:%d", len(searchResult.Hits.Hits))) for _, v := range searchResult.Hits.Hits { item := make(map[string]interface{}) if json.Unmarshal(v.Source, &item) == nil { delete(item, "_id") item["info_id"] = common.ObjToString(item["id"]) item["s_dataid"] = sdataid item["s_jyhref"] = fmt.Sprintf(Url, encrypt.CommonEncodeArticle("content", common.ObjToString(item["id"]))) item["i_createtime"] = time.Now().Unix() var d *DFA = &DFA{} var analyKeys []string //找到的关键词 var matchType []string //匹配方式 for _, math := range maths { fileds := strsToArr(math["s_keymatch"], "field") d.AddWord(strings.Split(math["s_matchkey"], ",")...) mkMap := make(map[string]interface{}) tmpMap := make(map[string]interface{}) for _, mk := range strings.Split(math["s_matchkey"], ",") { if strings.Contains(mk, "&&") { arr := strings.Split(mk, "&&") for _, s := range arr { if s != "" { tmpMap[s] = mk if b, _ := regexp.MatchString("[A-Z]", s); b { mkMap[strings.ToLower(s)] = s d.AddWord(strings.ToLower(s)) } else { d.AddWord(s) } } } } else { if b, _ := regexp.MatchString("[A-Z]", mk); b { mkMap[strings.ToLower(mk)] = mk d.AddWord(strings.ToLower(mk)) } else { d.AddWord(mk) } } } for _, filed := range fileds { filed1 := GetFieldData(item, filed) filed1 = ProcessData(filed1) ddds := d.Analy(filed1) analyKeys = append(analyKeys, ddds...) } if len(analyKeys) > 0 { matchType = append(matchType, strings.Join(fileds, ",")) for k, v := range analyKeys { if tmpMap[v] != "" && tmpMap[v] != nil { analyKeys[k] = common.ObjToString(tmpMap[v]) } } for _, v1 := range analyKeys { if mkMap[v1] != "" && mkMap[v1] != nil { analyKeys = deleteSlice(analyKeys, v1) analyKeys = append(analyKeys, common.ObjToString(mkMap[v1])) } } } d.Clear() } //去重 ssavekey := make(map[string]bool) for _, v := range analyKeys { ssavekey[v] = true } ssavekeys := []string{} for k := range ssavekey { ssavekeys = append(ssavekeys, k) } item["s_matchkey"] = GetMactchKey(maths, item) item["s_matchtype"] = strings.Join(matchType, ",") findwinner := strings.TrimSpace(common.ObjToString(item["winner"])) if findwinner != "" { finddata, _ := MgoEnps.FindOne(EnpsColl, bson.M{"company_name": findwinner}) if finddata != nil { if legal_person := common.ObjToString((*finddata)["legal_person"]); legal_person != "" { item["legal_person"] = legal_person } if email := common.ObjToString((*finddata)["company_email"]); email != "" { item["company_email"] = email } if phone := common.ObjToString((*finddata)["company_phone"]); phone != "" { item["company_phone"] = phone } item["qyk"] = finddata } } datas = append(datas, item) } } counts := Es.Count(index, Itype, esquery) log.Info(fmt.Sprintf("esCount查询的数据量 %d", counts)) Mgo.Update("cuserdepartrule", bson.M{"_id": tags["_id"]}, bson.M{ "$set": bson.M{ "i_estotal": counts, }}, false, false) return UtilEsSaveData(sdataid, index, &datas), counts } else { log.Error("err", zap.Error(err)) return err, 0 } } func FilterTimeSql(esquery string, startTime, endTime int64) (string, error) { query := map[string]*sql.QueryObjecct{} if json.Unmarshal([]byte(esquery), &query) == nil { qb := query["query"] for i := 0; i <= 2; i++ { filter := qb.Bool if filter != nil { //有filter index := -1 //记录range的位置 for i, m := range filter.Must { mMap := m.(map[string]interface{}) if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range if esRange["publishtime"] != nil { index = i break } } } // 生成新的es语句 tmpRange_ := bson.M{ "range": bson.M{ "publishtime": bson.M{ "lte": endTime, "gt": startTime, }, }, } if index > -1 { filter.Must[index] = tmpRange_ } else { filter.Must = append(filter.Must, tmpRange_) } tmpQuery := query strQuery, err := json.Marshal(tmpQuery) if err == nil { return string(strQuery), nil } else { log.Error("失败", zap.Error(err)) return esquery, err } } else { // qb.Filter = &sql.Filter{} qb.Bool = &sql.BoolObject{} continue } } } else { log.Error("err", zap.Error(json.Unmarshal([]byte(esquery), &query))) return esquery, errors.New("err") } return esquery, nil } // 查询语句过长时根据开始结束时间分段查询数据 func UtilEsFind2(tags map[string]interface{}, n int) (error, int64) { defer common.Catch() sdataid := common.ObjToString(tags["s_dataid"]) esquery := common.ObjToString(tags["s_esquery"]) if len(esquery) < 1 || len(sdataid) < 1 { return errors.New("s_esquery or s_dataid no found"), 0 } i_maxnum := common.Int64All(tags["i_maxnum"]) if i_maxnum <= 0 { i_maxnum = 100 } maths := make([]map[string]string, 0) if orules, ok := tags["o_rules"].([]interface{}); ok { for _, v := range orules { orule, _ := v.(map[string]interface{}) maths = append(maths, map[string]string{ "s_matchkey": common.ObjToString(orule["s_matchkey"]), "s_keymatch": common.ObjToString(orule["s_keymatch"]), }) } } else if orules, ok := tags["o_rules"].([]map[string]interface{}); ok { for _, v := range orules { maths = append(maths, map[string]string{ "s_matchkey": common.ObjToString(v["s_matchkey"]), "s_keymatch": common.ObjToString(v["s_keymatch"]), }) } } else { return errors.New("o_rules no found"), 0 } esCon := elastic.VarEs.(*elastic.EsV7) client := esCon.GetEsConn() defer esCon.DestoryEsConn(client) ctx, _ := context.WithTimeout(context.Background(), 30*time.Second) //for _, k := range { ////todo 修改开始结束时间 重新生成es语句 //} // 获取分隔区间 timeList := GetTimeInterval(time.Unix(common.Int64All(tags["i_starttime"]), 0), time.Unix(common.Int64All(tags["i_endtime"]), 0), n) log.Info("分隔后的时间===================", zap.Any("", timeList)) esQueryList := []string{} //esquery = esquery[:len(esquery)-1] + `,"size":` + fmt.Sprintf("%d", i_maxnum) + `}` // log.Println(esquery) query := map[string]*sql.QueryObjecct{} if json.Unmarshal([]byte(esquery), &query) == nil { qb := query["query"] filter := qb.Bool if filter != nil { //有filter index := -1 //记录range的位置 for i, m := range filter.Must { mMap := m.(map[string]interface{}) if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range if esRange["publishtime"] != nil { index = i break } } } log.Info("timeList================", zap.Any("", timeList)) // 生成新的es语句 for _, v := range timeList { tmpRange_ := bson.M{ "range": bson.M{ "publishtime": bson.M{ "lte": v["endTime"], "gt": v["startTime"], }, }, } if index > -1 { filter.Must[index] = tmpRange_ } else { filter.Must = append(filter.Must, tmpRange_) } tmpQuery := query strQuery, err := json.Marshal(tmpQuery) if err == nil { esQuery := string(strQuery) esQueryList = append(esQueryList, esQuery) } else { log.Error("失败", zap.Error(err)) return err, 0 } } } } else { log.Error("err", zap.Error(json.Unmarshal([]byte(esquery), &query))) } // for _, v := range esQueryList { // log.Println("es语句", v) // } log.Info("重新生成的es语句", zap.Any("", esQueryList)) var totalCount int64 datas := make([]map[string]interface{}, 0) for _, esSql := range esQueryList { // 查询 esSql = esSql[:len(esSql)-1] + `,"size":` + fmt.Sprintf("%d", i_maxnum) + `}` log.Info("本次查询==" + esSql) index := EsIndex if strings.Contains(esSql, "site") { //选择网站名称,使用全量数据索引 index = EsAllIndex } log.Info("es索引-------------------- " + index) cc := &MySource{ Querys: esSql, } searchResult, err := client.Search(index).Query(cc).Do(ctx) if err == nil && searchResult.Hits != nil { log.Info("es查询到的数量:", zap.Int64("", searchResult.Hits.TotalHits.Value)) log.Info("开始处理") for _, v := range searchResult.Hits.Hits { item := make(map[string]interface{}) if json.Unmarshal(v.Source, &item) == nil { delete(item, "_id") item["info_id"] = common.ObjToString(item["id"]) item["s_dataid"] = sdataid item["s_jyhref"] = fmt.Sprintf(Url, encrypt.CommonEncodeArticle("content", common.ObjToString(item["id"]))) item["i_createtime"] = time.Now().Unix() var d *DFA = &DFA{} var analyKeys []string //找到的关键词 var matchType []string //匹配方式 for _, math := range maths { fileds := strsToArr(math["s_keymatch"], "field") d.AddWord(strings.Split(math["s_matchkey"], ",")...) mkMap := make(map[string]interface{}) tmpMap := make(map[string]interface{}) for _, mk := range strings.Split(math["s_matchkey"], ",") { if strings.Contains(mk, "&&") { arr := strings.Split(mk, "&&") for _, s := range arr { if s != "" { tmpMap[s] = mk if b, _ := regexp.MatchString("[A-Z]", s); b { mkMap[strings.ToLower(s)] = s d.AddWord(strings.ToLower(s)) } else { d.AddWord(s) } } } } else { if b, _ := regexp.MatchString("[A-Z]", mk); b { mkMap[strings.ToLower(mk)] = mk d.AddWord(strings.ToLower(mk)) } else { d.AddWord(mk) } } } for _, filed := range fileds { filed1 := strings.ToLower(common.ObjToString(item[filed])) ddds := d.Analy(filed1) analyKeys = append(analyKeys, ddds...) } if len(analyKeys) > 0 { matchType = append(matchType, strings.Join(fileds, ",")) for k, v := range analyKeys { if tmpMap[v] != "" && tmpMap[v] != nil { analyKeys[k] = common.ObjToString(tmpMap[v]) } } for _, v1 := range analyKeys { if mkMap[v1] != "" && mkMap[v1] != nil { analyKeys = deleteSlice(analyKeys, v1) analyKeys = append(analyKeys, common.ObjToString(mkMap[v1])) } } } d.Clear() } //去重 ssavekey := make(map[string]bool) for _, v := range analyKeys { ssavekey[v] = true } ssavekeys := []string{} for k := range ssavekey { ssavekeys = append(ssavekeys, k) } item["s_matchkey"] = GetMactchKey(maths, item) item["s_matchtype"] = strings.Join(matchType, ",") findwinner := strings.TrimSpace(common.ObjToString(item["winner"])) if findwinner != "" { finddata, _ := MgoEnps.FindOne(EnpsColl, bson.M{"company_name": findwinner}) if finddata != nil { if legal_person := common.ObjToString((*finddata)["legal_person"]); legal_person != "" { item["legal_person"] = legal_person } if email := common.ObjToString((*finddata)["company_email"]); email != "" { item["company_email"] = email } if phone := common.ObjToString((*finddata)["company_phone"]); phone != "" { item["company_phone"] = phone } //从最新年报中获取 中标单位联系电话、中标单位邮箱 // if annual_reports, ok := finddata["annual_reports"].([]interface{}); ok && len(annual_reports) > 0 { // report_year := Sort_year_report(annual_reports) //最新年报 // if len(report_year) > 0 { // if email := common.ObjToString(report_year["company_email"]); email != "" { // item["company_email"] = email // } // if phone := common.ObjToString(report_year["company_phone"]); phone != "" { // item["company_phone"] = phone // } // } // } item["qyk"] = finddata } } datas = append(datas, item) } } log.Info("处理完成", zap.Int("count", len(datas))) count := searchResult.Hits.TotalHits.Value totalCount += count } else { log.Error("err", zap.Error(err)) return err, 0 } } Mgo.Update("cuserdepartrule", bson.M{"_id": tags["_id"]}, bson.M{ "$set": bson.M{ "i_estotal": totalCount, }}, false, false) if int64(len(datas)) > i_maxnum { datas = datas[:i_maxnum] } return UtilEsSaveData(sdataid, "", &datas), totalCount } func GetTimeInterval(startTime, entTime time.Time, n int) []map[string]interface{} { timeList := []map[string]interface{}{} endTimeTmp := startTime for endTimeTmp.Unix() < entTime.Unix() { var endTime_ time.Time startTime = endTimeTmp endTimeTmp = startTime.AddDate(0, n, 0) if endTimeTmp.Unix() > entTime.Unix() { endTime_ = entTime timeList = append(timeList, map[string]interface{}{"startTime": startTime.Unix(), "endTime": endTime_.Unix()}) break } else { endTime_ = endTimeTmp timeList = append(timeList, map[string]interface{}{"startTime": startTime.Unix(), "endTime": endTime_.Unix()}) } } //log.Println(timeList) return timeList } func Utiltags(tag map[string]interface{}, id string) string { defer common.Catch() tmpbyte, _ := json.Marshal(tag) tab := models.Tag{} err := json.Unmarshal(tmpbyte, &tab) if err != nil { return "json err:" + err.Error() } //log.Println("前端数据", tab) QueryObjecct := sql.QueryObjecct{} ffBoolObject := sql.BoolObject{} adsBoolObect := sql.NewEsObject{} if tab.Sarea != "" { adsBoolObect.Bool.Should = append(adsBoolObect.Bool.Should, sql.AreaCityDistrictMust{AreaCityDistrict: &sql.AreaCityDistrict{Area: strings.Split(tab.Sarea, ",")}}) } if tab.Scity != "" { adsBoolObect.Bool.Should = append(adsBoolObect.Bool.Should, sql.AreaCityDistrictMust{AreaCityDistrict: &sql.AreaCityDistrict{City: strings.Split(tab.Scity, ",")}}) } if tab.Sdistrict != "" { //城市——区县 cityds := strings.Split(tab.Sdistrict, ",") var ds []string for _, v := range cityds { ds = append(ds, strings.Split(v, "-")[1]) } adsBoolObect.Bool.Should = append(adsBoolObect.Bool.Should, sql.AreaCityDistrictMust{AreaCityDistrict: &sql.AreaCityDistrict{District: ds}}) } if len(adsBoolObect.Bool.Should) > 0 { ffBoolObject.Must = append(ffBoolObject.Must, adsBoolObect) } if tab.Stoptype != "" || tab.Ssubtype != "" { if len(tab.Stoptype) > 0 { toptypeSubtype := sql.ToptypeSubtype{} toptypeSubtype.Toptype = strings.Split(tab.Stoptype, ",") ffBoolObject.Must = append(ffBoolObject.Must, sql.ToptypeSubtypeMust{&toptypeSubtype}) } if len(tab.Ssubtype) > 0 { toptypeSubtype := sql.ToptypeSubtype{} toptypeSubtype.Subtype = strings.Split(tab.Ssubtype, ",") ffBoolObject.Must = append(ffBoolObject.Must, sql.ToptypeSubtypeMust{&toptypeSubtype}) } } if id == "67e64794756e2b3b3b6d5062" { siteArr := []string{"全军武器装备采购信息网", "军队采购网新网址"} ffBoolObject.Must = append(ffBoolObject.Must, sql.SiteMust{Site: &sql.Site{Site: siteArr}}) } else { if tab.Site != "" { ffBoolObject.Must = append(ffBoolObject.Must, sql.SiteMust{Site: &sql.Site{Site: strings.Split(tab.Site, ",")}}) } } if tab.Istarttime > 0 && tab.Iendtime > 0 { ffBoolObject.Must = append(ffBoolObject.Must, sql.PublishtimeMust{PublishtimeObject: &sql.PublishtimeObject{Publishtime: &sql.Publishtime{ Gte: tab.Istarttime, Lt: tab.Iendtime, }}}) } else if tab.Istarttime > 0 { ffBoolObject.Must = append(ffBoolObject.Must, sql.PublishtimeMust{PublishtimeObject: &sql.PublishtimeObject{Publishtime1: &sql.Publishtime1{ Gte: tab.Istarttime, }}}) } else if tab.Iendtime > 0 { ffBoolObject.Must = append(ffBoolObject.Must, sql.PublishtimeMust{PublishtimeObject: &sql.PublishtimeObject{Publishtime2: &sql.Publishtime2{ Lt: tab.Iendtime, }}}) } if tab.Sglobalbuyerclass != "" { if len(tab.Sglobalbuyerclass) > 0 { buyerclass := sql.BuyerclassObject{ Terms: struct { Buyerclass []string `json:"buyerclass,omitempty"` }{ Buyerclass: strings.Split(tab.Sglobalbuyerclass, ","), }, } ffBoolObject.Must = append(ffBoolObject.Must, buyerclass) } } if tab.Sglobaltopscopeclass != "" || tab.Sglobalsubscopeclass != "" { if len(tab.Sglobaltopscopeclass) > 0 { topScopeclass := sql.Scopeclass{} topScopeclass.Globaltopscopeclass = strings.Split(tab.Sglobaltopscopeclass, ",") ffBoolObject.Must = append(ffBoolObject.Must, sql.ScopeclassMust{&topScopeclass}) } else if len(tab.Sglobalsubscopeclass) > 0 { subScopeclass := sql.Scopeclass{} subScopeclass.Globalsubscopeclass = strings.Split(tab.Sglobalsubscopeclass, ",") ffBoolObject.Must = append(ffBoolObject.Must, sql.ScopeclassMust{&subScopeclass}) } } fqBoolObject := sql.BoolObject{} if len(tab.Sexistfields) > 0 { tmpsfields := strings.Split(tab.Sexistfields, ",") for _, v := range tmpsfields { // fqBoolObject.MustNot = append(fqBoolObject.MustNot, sql.ExistfieldsObjectMust{ // ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct { // Missing struct { // Field string `json:"field,omitempty"` // } `json:"missing,omitempty"` // }{ // Missing: struct { // Field string `json:"field,omitempty"` // }{Field: v}, // }, // }, // }) ffBoolObject.Must = append(ffBoolObject.Must, sql.ExistfieldsObjectMust{ ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct { Exists struct { Field string `json:"field,omitempty"` } `json:"exists,omitempty"` }{ Exists: struct { Field string `json:"field,omitempty"` }{Field: v}, }, }, }) } } newEsObject := sql.NewEsObject{} if tab.Sglobaladdkey != "" && tab.Sglobaladdkeymatch != "" { if tmps := strsToArr(tab.Sglobaladdkeymatch, "str"); tmps != nil { newEsObject = method1(newEsObject, tab.Sglobaladdkey, tab.Sglobaladdkeymatch, tmps) } } if len(newEsObject.Bool.Should) > 0 || len(newEsObject.Bool.MustNot) > 0 || len(newEsObject.Bool.Must) > 0 { fqBoolObject.Must = append(fqBoolObject.Must, newEsObject) } nots := sql.NewEsObject{} // if tab.ExpurasingtimeStart > 0 && tab.ExpurasingtimeEnd > 0 { ffBoolObject.MustNot = append(ffBoolObject.MustNot, sql.ExpurasingtimeMust{ExpurasingtimeObject: &sql.ExpurasingtimeObject{Expurasingtime: &sql.Expurasingtime{ Gte: tab.ExpurasingtimeStart, Lt: tab.ExpurasingtimeEnd, }}}) } else if tab.ExpurasingtimeStart > 0 { ffBoolObject.MustNot = append(ffBoolObject.MustNot, sql.ExpurasingtimeMust{ExpurasingtimeObject: &sql.ExpurasingtimeObject{Expurasingtime1: &sql.Expurasingtime1{ Gte: tab.ExpurasingtimeStart, }}}) } else if tab.ExpurasingtimeEnd > 0 { ffBoolObject.MustNot = append(ffBoolObject.MustNot, sql.ExpurasingtimeMust{ExpurasingtimeObject: &sql.ExpurasingtimeObject{Expurasingtime2: &sql.Expurasingtime2{ Lt: tab.ExpurasingtimeEnd, }}}) } // if tab.Sglobalnotkey != "" && tab.Sglobalnotkeymatch != "" { if tmps := strsToArr(tab.Sglobalnotkeymatch, "str"); tmps != nil { nots = method1(nots, tab.Sglobalnotkey, tab.Sglobalnotkeymatch, tmps) } } if len(nots.Bool.Should) > 0 { fqBoolObject.MustNot = append(fqBoolObject.MustNot, nots) } torules := sql.BoolObject{} for _, v := range tab.Orules { tmpses := sql.NewEsObject{} if skeymatchs := strsToArr(v.Skeymatch, "str"); skeymatchs != nil { tmpnewEsObject := sql.NewEsObject{} tmpnewEsObject = method1(tmpnewEsObject, v.Smatchkey, v.Skeymatch, skeymatchs) if len(tmpnewEsObject.Bool.Should) > 0 || len(tmpnewEsObject.Bool.Must) > 0 || len(tmpnewEsObject.Bool.MustNot) > 0 { tmpses.Bool.Must = append(tmpses.Bool.Must, tmpnewEsObject) } } if saddkeymatchs := strsToArr(v.Saddkeymatch, "str"); saddkeymatchs != nil { addkeyarr := sql.NewEsObject{} addkeyarr = method1(addkeyarr, v.Saddkey, v.Saddkeymatch, saddkeymatchs) if len(addkeyarr.Bool.Should) > 0 { tmpses.Bool.Must = append(tmpses.Bool.Must, addkeyarr) } } if snotkeymatchs := strsToArr(v.Snotkeymatch, "str"); snotkeymatchs != nil { tmpses = method2(tmpses, v.Snotkey, v.Snotkeymatch, snotkeymatchs) } if len(v.Stopscopeclass) > 0 { sArr := strings.Split(v.Stopscopeclass, ",") tmpses.Bool.Must = append(tmpses.Bool.Must, sql.TopscopeclassObject{ Terms: struct { Topscopeclass []string `json:"s_topscopeclass,omitempty"` }{ Topscopeclass: sArr, }, }) } else if len(v.Ssubscopeclass) > 0 { tmpses.Bool.Must = append(tmpses.Bool.Must, sql.SubscopeclassObject{ Terms: struct { Subscopeclass []string `json:"s_subscopeclass,omitempty"` }{ Subscopeclass: strings.Split(v.Ssubscopeclass, ","), }, }) } if len(v.Sbuyerclass) > 0 { tmpses.Bool.Must = append(tmpses.Bool.Must, sql.BuyerclassObject{ Terms: struct { Buyerclass []string `json:"buyerclass,omitempty"` }{ Buyerclass: strings.Split(v.Sbuyerclass, ","), }, }) } torules.Should = append(torules.Should, tmpses) } bidamountSql := sql.NewEsObject{} if tab.Sbidamount != "" { if strings.Contains(tab.Sbidamount, "大于") && strings.Contains(tab.Sbidamount, "小于") { arr := strings.Split(tab.Sbidamount, ",") limit := &sql.BudgetOrBidamount{ Gte: common.Float64All(strings.Replace(arr[0], "大于", "", -1)), Lt: common.Float64All(strings.Replace(arr[1], "小于", "", -1)), } bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}}) } else if strings.Contains(tab.Sbidamount, "大于") { limit := &sql.BudgetOrBidamount{ Gte: common.Float64All(strings.Replace(tab.Sbidamount, "大于", "", -1)), } bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}}) } else if strings.Contains(tab.Sbidamount, "小于") { limit := &sql.BudgetOrBidamount{ Lt: common.Float64All(strings.Replace(tab.Sbidamount, "小于", "", -1)), } bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}}) } if tab.SbidamountFieldExist == 1 { // bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.ExistfieldsObjectMust{ // ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct { // Missing struct { // Field string `json:"field,omitempty"` // } `json:"missing,omitempty"` // }{ // Missing: struct { // Field string `json:"field,omitempty"` // }{Field: "bidamount"}, // }, // }, // }) SbidamountFieldExistBool := sql.NewEsObject{} SbidamountFieldExistBool.Bool.MustNot = append(SbidamountFieldExistBool.Bool.MustNot, sql.ExistfieldsObjectMust{ ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct { Exists struct { Field string `json:"field,omitempty"` } `json:"exists,omitempty"` }{ Exists: struct { Field string `json:"field,omitempty"` }{Field: "bidamount"}, }, }, }) bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, SbidamountFieldExistBool) } } budgetSql := sql.NewEsObject{} if tab.Sbudget != "" { if strings.Contains(tab.Sbudget, "大于") && strings.Contains(tab.Sbudget, "小于") { arr := strings.Split(tab.Sbudget, ",") limit := &sql.BudgetOrBidamount{ Gte: common.Float64All(strings.Replace(arr[0], "大于", "", -1)), Lt: common.Float64All(strings.Replace(arr[1], "小于", "", -1)), } budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}}) } else if strings.Contains(tab.Sbudget, "大于") { limit := &sql.BudgetOrBidamount{ Gte: common.Float64All(strings.Replace(tab.Sbudget, "大于", "", -1)), } budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}}) } else if strings.Contains(tab.Sbudget, "小于") { limit := &sql.BudgetOrBidamount{ Lt: common.Float64All(strings.Replace(tab.Sbudget, "小于", "", -1)), } budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}}) } if tab.SbudgetFieldExist == 1 { // budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.ExistfieldsObjectMust{ // ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct { // Missing struct { // Field string `json:"field,omitempty"` // } `json:"missing,omitempty"` // }{ // Missing: struct { // Field string `json:"field,omitempty"` // }{Field: "budget"}, // }, // }, // }) SbudgetFieldExistBool := sql.NewEsObject{} SbudgetFieldExistBool.Bool.MustNot = append(SbudgetFieldExistBool.Bool.MustNot, sql.ExistfieldsObjectMust{ ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct { Exists struct { Field string `json:"field,omitempty"` } `json:"exists,omitempty"` }{ Exists: struct { Field string `json:"field,omitempty"` }{Field: "budget"}, }, }, }) budgetSql.Bool.Should = append(budgetSql.Bool.Should, SbudgetFieldExistBool) } } if len(ffBoolObject.Must) > 0 || len(ffBoolObject.MustNot) > 0 || len(ffBoolObject.Should) > 0 { // QueryObjecct.Filter = &sql.Filter{} QueryObjecct.Bool = &ffBoolObject } if len(fqBoolObject.Must) > 0 || len(fqBoolObject.MustNot) > 0 || len(fqBoolObject.Should) > 0 { if QueryObjecct.Bool == nil { QueryObjecct.Bool = &fqBoolObject } else { if fqBoolObject.Must != nil { QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, fqBoolObject.Must...) } if fqBoolObject.Should != nil { QueryObjecct.Bool.Should = append(QueryObjecct.Bool.Should, fqBoolObject.Should...) } if fqBoolObject.MustNot != nil { QueryObjecct.Bool.MustNot = append(QueryObjecct.Bool.MustNot, fqBoolObject.MustNot...) } } if len(torules.Should) > 0 { QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, map[string]interface{}{ "bool": torules, }) } } else if len(torules.Should) > 0 { if QueryObjecct.Bool == nil { // QueryObjecct.Query = &sql.Query{} QueryObjecct.Bool = &sql.BoolObject{} } QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, map[string]interface{}{ "bool": torules, }) } if (len(budgetSql.Bool.Should) > 0 || len(bidamountSql.Bool.Should) > 0) && (len(fqBoolObject.Must) > 0 || len(fqBoolObject.MustNot) > 0 || len(fqBoolObject.Should) > 0) { if QueryObjecct.Bool == nil { QueryObjecct.Bool = &sql.BoolObject{} } if len(budgetSql.Bool.Should) > 0 { QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, budgetSql) } if len(bidamountSql.Bool.Should) > 0 { QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, bidamountSql) } } else { if QueryObjecct.Bool == nil { QueryObjecct.Bool = &sql.BoolObject{} } if len(budgetSql.Bool.Should) > 0 { QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, budgetSql) } if len(bidamountSql.Bool.Should) > 0 { QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, bidamountSql) } } rdata := make(map[string]interface{}) rdata["query"] = QueryObjecct esbytes, _ := json.Marshal(rdata) esStr := strings.Replace(string(esbytes), "regexp1", "regexp", -1) esStr = strings.Replace(esStr, "publishtime1", "publishtime", -1) esStr = strings.Replace(esStr, "publishtime2", "publishtime", -1) esStr = strings.Replace(esStr, "expurasingtime1", "expurasingtime", -1) esStr = strings.Replace(esStr, "expurasingtime2", "expurasingtime", -1) return strings.Replace(esStr, "regexp2", "regexp", -1) } func strsToArr(strs string, typestr string) []string { if len(strs) > 0 { tmps := strings.Split(strs, ",") switch typestr { case "str": for index, vint := range tmps { switch vint { case TitleMatchType: tmps[index] = TitleMatchTypeStr case DetailMatchType: tmps[index] = DetailMatchTypeStr case PurchasingMatchType: tmps[index] = PurchasingMatchTypeStr case AttachmentsMatchType: tmps[index] = AttachmentsMatchTypeStr case ProjectnameMatchType: tmps[index] = ProjectnameMatchTypeStr case BuyerMatchType: tmps[index] = BuyerMatchTypeStr case SWinnerMatchType: tmps[index] = SWinnerMatchTypeStr case BuyerMatchType1: tmps[index] = BuyerRegexpMatchTypeStr case SWinnerMatchType1: tmps[index] = SWinnerRegexpMatchTypeStr case ProjectnameMatchTypes: tmps[index] = ProjectnameMatchTypesField case ProjectscopeMatchTypes: tmps[index] = ProjectscopeMatchTypesField case BuyerMatchTypes: tmps[index] = BuyerMatchTypesField } } return tmps case "field": for index, vint := range tmps { switch vint { case TitleMatchType: tmps[index] = TitleMatchTypeField case DetailMatchType: tmps[index] = DetailMatchTypeField case PurchasingMatchType: tmps[index] = PurchasingMatchTypeField case AttachmentsMatchType: tmps[index] = AttachmentsMatchTypeField case ProjectnameMatchType: tmps[index] = ProjectnameMatchTypeField case BuyerMatchType: tmps[index] = BuyerMatchTypeField case SWinnerMatchType: tmps[index] = SWinnerMatchTypeField case BuyerMatchType1: tmps[index] = BuyerMatchTypeField case SWinnerMatchType1: tmps[index] = SWinnerMatchTypeField case ProjectnameMatchTypes: tmps[index] = ProjectnameMatchTypesField case ProjectscopeMatchTypes: tmps[index] = ProjectscopeMatchTypesField case BuyerMatchTypes: tmps[index] = BuyerMatchTypesField } } return tmps } } return nil } func init() { EsIndex = common.ObjToString(Sysconfig["elasticsearch_index"]) EsType = common.ObjToString(Sysconfig["elasticsearch_type"]) } func method1(newEsObject sql.NewEsObject, keyword, keymatch string, tmps []string) sql.NewEsObject { keywordArr := []string{} for _, vv := range strings.Split(keyword, ",") { if vv == "" { continue } //log.Println(vv, tmps, keymatch) if len(tmps) == 1 && (strings.Contains(keymatch, "8") || strings.Contains(keymatch, "9")) { //中标单位、采购单位、中标单位(正则)、采购单位(正则) 单选 strs := "" if strings.Contains(vv, "&&") { //strs = ".*" + strings.Replace(vv, "&&", ".*", -1) + ".*" strs = strings.Replace(vv, "&&", ".*", -1) } else { strs = vv } if tmps[0] == "buyer.mbuyer" { newEsObject.Bool.Should = append(newEsObject.Bool.Should, sql.ShouldObj{ MultiMatch: &sql.MultiMatch{ Query: strs, Type: MultiMatchType, Fields: tmps, }, }) } else if tmps[0] == "s_winner.mwinner" { newEsObject.Bool.Should = append(newEsObject.Bool.Should, sql.ShouldObj{ MultiMatch: &sql.MultiMatch{ Query: strs, Type: MultiMatchType, Fields: tmps, }, }) } } else if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) { addkeylines := strings.Split(vv, "&&") keywordArr = append(keywordArr, addkeylines[0]) } else { addkeylines := strings.Split(vv, "&&") //log.Println(addkeylines, "addkeylines") if len(addkeylines) > 1 { addkeyline := sql.NewEsObject{} for _, vvv := range addkeylines { if vvv == "" { continue } addkeyline.Bool.Must = append(addkeyline.Bool.Must, sql.ShouldObj{ MultiMatch: &sql.MultiMatch{ Query: vvv, Type: MultiMatchType, Fields: tmps, }, }) } if len(addkeyline.Bool.Must) > 0 { newEsObject.Bool.Should = append(newEsObject.Bool.Should, addkeyline) } } else { newEsObject.Bool.Should = append(newEsObject.Bool.Should, sql.ShouldObj{ MultiMatch: &sql.MultiMatch{ Query: vv, Type: MultiMatchType, Fields: tmps, }, }) } } } if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) { addkeyline := sql.NewEsObject{} if tmps[0] == "buyer" { addkeyline.Bool.Must = append(addkeyline.Bool.Must, sql.BuyerMatch{ Buyer: &sql.Buyer{Buyer: keywordArr}, }) } else if tmps[0] == "s_winner" { addkeyline.Bool.Must = append(addkeyline.Bool.Must, sql.WinnerMatch{ Winner: &sql.Winner{Winner: keywordArr}, }) } if len(addkeyline.Bool.Must) > 0 { newEsObject.Bool.Should = append(newEsObject.Bool.Should, addkeyline) } } return newEsObject } func method2(newEsObject sql.NewEsObject, keyword, keymatch string, tmps []string) sql.NewEsObject { notKeyWordArr := []string{} for _, vv := range strings.Split(keyword, ",") { if vv == "" { continue } if len(tmps) == 1 && (strings.Contains(keymatch, "8") || strings.Contains(keymatch, "9")) { //中标单位、采购单位、中标单位(正则)、采购单位(正则) 单选 //strs := "" //if strings.Contains(vv, "&&") { // strs = ".*" + strings.Replace(vv, "&&", ".*", -1) + ".*" //} else { // strs = ".*" + vv + ".*" //} if tmps[0] == "buyer.mbuyer" { newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, sql.ShouldObj{ MultiMatch: &sql.MultiMatch{ Query: vv, Type: MultiMatchType, Fields: tmps, }, }) } else if tmps[0] == "s_winner.mwinner" { newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, sql.ShouldObj{ MultiMatch: &sql.MultiMatch{ Query: vv, Type: MultiMatchType, Fields: tmps, }, }) } } else if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) { notkeylines := strings.Split(vv, "&&") notKeyWordArr = append(notKeyWordArr, notkeylines[0]) } else { notkeylines := strings.Split(vv, "&&") if len(notkeylines) > 1 { notkeyline := sql.NewEsObject{} for _, vvv := range notkeylines { if vvv == "" { continue } notkeyline.Bool.Must = append(notkeyline.Bool.Must, sql.ShouldObj{ MultiMatch: &sql.MultiMatch{ Query: vvv, Type: MultiMatchType, Fields: tmps, }, }) } if len(notkeyline.Bool.Must) > 0 { newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, notkeyline) } } else { newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, sql.ShouldObj{ MultiMatch: &sql.MultiMatch{ Query: vv, Type: MultiMatchType, Fields: tmps, }, }) } } } if len(tmps) == 1 && (strings.Contains(keymatch, "6") || strings.Contains(keymatch, "7")) { var notkeyline interface{} if tmps[0] == "buyer" { notkeyline = sql.BuyerMatch{ Buyer: &sql.Buyer{Buyer: notKeyWordArr}, } } else if tmps[0] == "s_winner" { notkeyline = sql.WinnerMatch{ Winner: &sql.Winner{Winner: notKeyWordArr}, } } if notkeyline != nil { newEsObject.Bool.MustNot = append(newEsObject.Bool.MustNot, notkeyline) } } return newEsObject } /** * 一级公告行业处理 d、t、p */ func method3(arr []string) []string { var sArr []string for _, v := range arr { sArr = append(sArr, v+"d") sArr = append(sArr, v+"t") sArr = append(sArr, v+"p") } return sArr } func GetMactchKey(match []map[string]string, data map[string]interface{}) string { keyWord := []string{} for _, keys := range match { types := keys["s_keymatch"] key := keys["s_matchkey"] if strings.Contains(types, "1") { title := common.ObjToString(data["title"]) keyWord = KeyWordToData(types, title, key, keyWord) } if strings.Contains(types, "2") { detail := common.ObjToString(data["detail"]) keyWord = KeyWordToData(types, detail, key, keyWord) } if strings.Contains(types, "3") { purchasing := common.ObjToString(data["purchasing"]) keyWord = KeyWordToData(types, purchasing, key, keyWord) } if strings.Contains(types, "4") { filetext := common.ObjToString(data["filetext"]) keyWord = KeyWordToData(types, filetext, key, keyWord) } if strings.Contains(types, "5") { projectname := common.ObjToString(data["projectname"]) keyWord = KeyWordToData(types, projectname, key, keyWord) } if strings.Contains(types, "6") || strings.Contains(types, "8") { buyer := common.ObjToString(data["buyer"]) keyWord = KeyWordToData(types, buyer, key, keyWord) } if strings.Contains(types, "7") || strings.Contains(types, "9") { winner := common.ObjToString(data["s_winner"]) keyWord = KeyWordToData(types, winner, key, keyWord) } } keyMap := map[string]bool{} keyArr := []string{} for _, key := range keyWord { keyMap[key] = true } for k, _ := range keyMap { keyArr = append(keyArr, k) } return strings.Join(keyArr, ",") } func KeyWordToData(types, item, key string, keyWord []string) []string { for _, mk := range strings.Split(key, ",") { if strings.Contains(mk, "&&") { arr := strings.Split(mk, "&&") isok := true for _, s := range arr { if s != "" { if !strings.Contains(strings.ToUpper(item), strings.ToUpper(s)) { isok = false } } } if isok { keyWord = append(keyWord, mk) } } else { spaceReg := regexp.MustCompile("[\\s\\n \u3000\u2003\u00a0]+") itemUpper := strings.ToUpper(item) mkUpper := strings.ToUpper(mk) matchResult := strings.Contains(spaceReg.ReplaceAllString(itemUpper, ""), spaceReg.ReplaceAllString(mkUpper, "")) if matchResult { keyWord = append(keyWord, mk) } } } return keyWord }