|
|
@@ -5,7 +5,6 @@ import (
|
|
|
"encoding/json"
|
|
|
"errors"
|
|
|
"fmt"
|
|
|
- "go.mongodb.org/mongo-driver/bson"
|
|
|
"log"
|
|
|
"math"
|
|
|
mg "mongodb"
|
|
|
@@ -47,8 +46,8 @@ type SieveCondition struct {
|
|
|
MaxPrice string `json:"maxprice"` //金额——最多
|
|
|
SelectType string `json:"selectType"` //筛选（正文 or 标题）
|
|
|
Subtype string `json:"subtype"` //信息类型
|
|
|
+ SelectIds []string `json:"selectId"` //选择信息导出
|
|
|
Comeinfrom string `json:"comeinfrom"` //查询来源
|
|
|
- DisWord string `json:"disWord"` //分销系统 口令
|
|
|
}
|
|
|
|
|
|
const (
|
|
|
@@ -65,6 +64,11 @@ var searchPool = make(chan bool, 8)
|
|
|
|
|
|
//获取数据导出查询语句
|
|
|
func getDataExportSql(scd *SieveCondition) string {
|
|
|
+ if len(scd.SelectIds) > 0 {
|
|
|
+ query := `{"query":{"bool":{"must":[%s]}}}`
|
|
|
+ doSearchSql := fmt.Sprintf(`{"terms":{"_id":[%s]}}`, `"`+strings.Join(scd.SelectIds, `","`)+`"`)
|
|
|
+ return fmt.Sprintf(query, doSearchSql)
|
|
|
+ }
|
|
|
multi_match := `{"multi_match": {"query": %s,"type": "phrase", "fields": [%s]}}`
|
|
|
query := `{"query":{"bool":{"must":[%s],"must_not":[%s],"should":[%s],"minimum_should_match": %d}}}`
|
|
|
query_bool_should := `{"bool":{"should":[%s],"minimum_should_match": 1}}`
|
|
|
@@ -296,12 +300,16 @@ func GetSqlObjFromId(mongo mg.MongodbSim, _id string) *SieveCondition {
|
|
|
HasWinnerTel: qutil.ObjToString((*query)["hasWinnertel"]),
|
|
|
ComeInTime: qutil.Int64All((*query)["comeintime"]),
|
|
|
Comeinfrom: qutil.ObjToString((*query)["comeinfrom"]),
|
|
|
+ SelectIds: getStringArrFromDbResult((*query)["selectIds"]),
|
|
|
}
|
|
|
}
|
|
|
|
|
|
//数据导出-查询结果数量
|
|
|
func GetDataExportSearchCountByScdId(sim mg.MongodbSim, elasticAddress, id string) (count int) {
|
|
|
scd := GetSqlObjFromId(sim, id) //用户筛选条件
|
|
|
+ if scd.SelectIds != nil {
|
|
|
+ return len(scd.SelectIds)
|
|
|
+ }
|
|
|
return GetDataExportSearchCountBySieveCondition(scd, elasticAddress)
|
|
|
}
|
|
|
|
|
|
@@ -379,6 +387,9 @@ func isNullSearch(scd *SieveCondition) (isNull bool) {
|
|
|
|
|
|
func GetDataExportSearchResultByScdId(sim mg.MongodbSim, elasticAddress, id, dataType string, checkCount int) (*[]map[string]interface{}, error) {
|
|
|
scd := GetSqlObjFromId(sim, id)
|
|
|
+ //if scd.SelectIds != nil {
|
|
|
+ // return GetDataExportSelectResult(scd, dataType, checkCount)
|
|
|
+ //}
|
|
|
return GetDataExportSearchResult(elasticAddress, scd, dataType, checkCount)
|
|
|
}
|
|
|
|
|
|
@@ -387,16 +398,48 @@ func GetDataExportSearchResultByScdId(sim mg.MongodbSim, elasticAddress, id, dat
|
|
|
//checkCount -1 预览500条
|
|
|
func GetDataExportSearchResult(elasticAddress string, scd *SieveCondition, dataType string, checkCount int) (*[]map[string]interface{}, error) {
|
|
|
defer qutil.Catch()
|
|
|
- var res []map[string]interface{}
|
|
|
- //获取查询语句
|
|
|
if scd == nil {
|
|
|
return nil, errors.New("GetDataExportSearchResult-获取查询条件")
|
|
|
}
|
|
|
+ //获取查询语句
|
|
|
qstr := getDataExportSql(scd)
|
|
|
log.Printf("GetDataExportSearchResult-%s-sql:%s\n", scd.Id, qstr)
|
|
|
+ log.Println("len(scd.SelectIds)=", len(scd.SelectIds))
|
|
|
//数据导出数据查询
|
|
|
- if checkCount > onceSearchCount { //分批次查询
|
|
|
- batchNum := qutil.IntAll(math.Ceil(float64(checkCount) / float64(onceSearchCount)))
|
|
|
+ res := doSearchByBatch(qstr, dataType, checkCount, fmt.Sprintf("%s-%s", "GetDataExportSearchResult", scd.Id))
|
|
|
+ //超级搜索一致的检索（防止数据导出和超级搜索数据量不一致）
|
|
|
+ if scd.Comeinfrom == "supersearchPage" && (len(scd.Keyword) != 0 || len(scd.Industry) != 0) && len(scd.SelectIds) == 0 {
|
|
|
+ if len(scd.Keyword) != 0 {
|
|
|
+ num := len(res)
|
|
|
+ searchTextSize := 0
|
|
|
+ if len(scd.Keyword) > 0 {
|
|
|
+ searchTextSize = len([]rune(scd.Keyword[0].Keyword))
|
|
|
+ }
|
|
|
+ if searchTextSize > 3 && num < 50 {
|
|
|
+ secondKWS := jy.HttpEs(scd.Keyword[0].Keyword, "ik_smart", elasticAddress)
|
|
|
+ scd.Keyword[0].Keyword = secondKWS
|
|
|
+ scd.SelectType = "title"
|
|
|
+ qstr = getDataExportSql(scd)
|
|
|
+ log.Printf("GetDataExportSearchResult-%s-分词查询-sql:%s\n", scd.Id, qstr)
|
|
|
+ res2 := doSearch(qstr, 0, 100, "")
|
|
|
+ res = *delRepeatMapArr(&res, res2)
|
|
|
+ if len(res) > 100 {
|
|
|
+ res = res[:100]
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //校验数量
|
|
|
+ if checkCount != len(res) && checkCount != -1 {
|
|
|
+ return nil, fmt.Errorf("GetDataExportSearchResult-%s-数据总量校验异常，期望:%d,实际:%d", scd.Id, checkCount, len(res))
|
|
|
+ //发邮件
|
|
|
+ }
|
|
|
+ return &res, nil
|
|
|
+}
|
|
|
+
|
|
|
+func doSearchByBatch(query, dataType string, searchCount int, flag string) (res []map[string]interface{}) {
|
|
|
+ if searchCount > onceSearchCount { //分批次查询
|
|
|
+ batchNum := qutil.IntAll(math.Ceil(float64(searchCount) / float64(onceSearchCount)))
|
|
|
var searchWaitGroup = &sync.WaitGroup{}
|
|
|
var lock sync.Mutex
|
|
|
for n := 0; n < batchNum; n++ {
|
|
|
@@ -409,27 +452,27 @@ func GetDataExportSearchResult(elasticAddress string, scd *SieveCondition, dataT
|
|
|
}()
|
|
|
checkNum, checkOk := onceSearchCount, false
|
|
|
if start == (batchNum - 1) {
|
|
|
- if checkCount%onceSearchCount != 0 {
|
|
|
- checkNum = checkCount % onceSearchCount
|
|
|
+ if searchCount%onceSearchCount != 0 {
|
|
|
+ checkNum = searchCount % onceSearchCount
|
|
|
}
|
|
|
}
|
|
|
|
|
|
var tmp *[]map[string]interface{}
|
|
|
for i := 0; i < 3; i++ {
|
|
|
- tmp = doSearch(qstr, start*onceSearchCount, onceSearchCount, dataType)
|
|
|
+ tmp = doSearch(query, start*onceSearchCount, onceSearchCount, dataType)
|
|
|
if tmp != nil && (len(*tmp) == checkNum) { //校验数据量是否够
|
|
|
checkOk = true
|
|
|
break
|
|
|
}
|
|
|
}
|
|
|
if tmp == nil {
|
|
|
- log.Printf("GetDataExportSearchResult-%s-第%d页数据查询结果为空\n", scd.Id, start+1)
|
|
|
+ log.Printf("%s-第%d页数据查询结果为空\n", flag, start+1)
|
|
|
return
|
|
|
}
|
|
|
if checkOk {
|
|
|
- log.Printf("GetDataExportSearchResult-%s-第%d页数据加载完成,共%d条\n", scd.Id, start+1, len(*tmp))
|
|
|
+ log.Printf("%s-第%d页数据加载完成,共%d条\n", flag, start+1, len(*tmp))
|
|
|
} else {
|
|
|
- log.Printf("GetDataExportSearchResult-%s-第%d页数据加载异常,共%d条,预期%d条\n", scd.Id, start+1, len(*tmp), checkNum)
|
|
|
+ log.Printf("%s-第%d页数据加载异常,共%d条,预期%d条\n", flag, start+1, len(*tmp), checkNum)
|
|
|
}
|
|
|
lock.Lock()
|
|
|
res = append(res, *tmp...)
|
|
|
@@ -437,100 +480,22 @@ func GetDataExportSearchResult(elasticAddress string, scd *SieveCondition, dataT
|
|
|
}(n)
|
|
|
}
|
|
|
searchWaitGroup.Wait()
|
|
|
- log.Printf("GetDataExportSearchResult-%s-分批次加载数据总量为%d\n", scd.Id, len(res))
|
|
|
+ log.Printf("%s-分批次加载数据总量为%d\n", flag, len(res))
|
|
|
} else {
|
|
|
- queryCount := qutil.If(checkCount == -1, 500, checkCount).(int)
|
|
|
- tmp := doSearch(qstr, 0, queryCount, dataType)
|
|
|
+ queryCount := qutil.If(searchCount == -1, onceSearchCount, searchCount).(int)
|
|
|
+ searchPool <- true
|
|
|
+ tmp := doSearch(query, 0, queryCount, dataType)
|
|
|
+ <-searchPool
|
|
|
if tmp == nil || len(*tmp) == 0 {
|
|
|
- log.Printf("GetDataExportSearchResult-%s-一次性加载数据异常\n", scd.Id)
|
|
|
+ log.Printf("%s-一次性加载数据异常\n", flag)
|
|
|
} else {
|
|
|
res = *tmp
|
|
|
- log.Printf("GetDataExportSearchResult-%s-一次性加载数据总量为%d\n", scd.Id, len(res))
|
|
|
+ log.Printf("%s-一次性加载数据总量为%d\n", flag, len(res))
|
|
|
}
|
|
|
}
|
|
|
- //超级搜索一致的检索（防止数据导出和超级搜索数据量不一致）
|
|
|
- if scd.Comeinfrom == "supersearchPage" && (len(scd.Keyword) != 0 || len(scd.Industry) != 0) {
|
|
|
- if len(scd.Keyword) != 0 {
|
|
|
- num := len(res)
|
|
|
- searchTextSize := 0
|
|
|
- if len(scd.Keyword) > 0 {
|
|
|
- searchTextSize = len([]rune(scd.Keyword[0].Keyword))
|
|
|
- }
|
|
|
- if searchTextSize > 3 && num < 50 {
|
|
|
- secondKWS := jy.HttpEs(scd.Keyword[0].Keyword, "ik_smart", elasticAddress)
|
|
|
- scd.Keyword[0].Keyword = secondKWS
|
|
|
- scd.SelectType = "title"
|
|
|
- qstr = getDataExportSql(scd)
|
|
|
- log.Printf("GetDataExportSearchResult-%s-分词查询-sql:%s\n", scd.Id, qstr)
|
|
|
- res2 := doSearch(qstr, 0, 100, "")
|
|
|
- res = *delRepeatMapArr(&res, res2)
|
|
|
- if len(res) > 100 {
|
|
|
- res = res[:100]
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- //校验数量
|
|
|
- if checkCount != len(res) && checkCount != -1 {
|
|
|
- return nil, fmt.Errorf("GetDataExportSearchResult-%s-数据总量校验异常，期望:%d,实际:%d", scd.Id, checkCount, len(res))
|
|
|
- //发邮件
|
|
|
- }
|
|
|
- return &res, nil
|
|
|
+ return
|
|
|
}
|
|
|
|
|
|
-//数据导出预览
|
|
|
-//func GetDataExportSearchResultPreview(sim mg.MongodbSim, elasticAddress string, _id, dataType string) (*[]map[string]interface{}, []KeyWord) {
|
|
|
-// defer qutil.Catch()
|
|
|
-// var res []map[string]interface{}
|
|
|
-// var kws []KeyWord
|
|
|
-// scd := GetSqlObjFromId(sim, _id)
|
|
|
-// //获取查询语句
|
|
|
-// qstr := getDataExportSql(scd)
|
|
|
-// log.Printf("GetDataExportSearchResultUseId-%s-sql:%s\n", scd.Id, qstr)
|
|
|
-// kws = scd.Keyword
|
|
|
-// //数据预览数据查询
|
|
|
-// if scd.Comeinfrom == "supersearchPage" && len(scd.Keyword) == 0 && len(scd.Industry) == 0 {
|
|
|
-// //空查询
|
|
|
-// obj := redis.Get("other", "export_news")
|
|
|
-// if obj != nil {
|
|
|
-// res = qutil.ObjArrToMapArr(obj.([]interface{}))
|
|
|
-// } else {
|
|
|
-// res = *doSearch(qstr, 0, 500, "2")
|
|
|
-// if len(res) > 0 {
|
|
|
-// redis.Put("other", "export_news", res, 7200)
|
|
|
-// }
|
|
|
-// }
|
|
|
-// return &res, kws
|
|
|
-// } else if scd.Comeinfrom == "supersearchPage" {
|
|
|
-// //超级搜索非空查询
|
|
|
-// count := int(elastic.Count(INDEX, TYPE, qstr))
|
|
|
-// searchTextSize := 0
|
|
|
-// if len(scd.Keyword) > 0 {
|
|
|
-// searchTextSize = len([]rune(scd.Keyword[0].Keyword))
|
|
|
-// }
|
|
|
-// if searchTextSize > 3 && count < 50 {
|
|
|
-// var res *[]map[string]interface{}
|
|
|
-// if count > 0 {
|
|
|
-// res = doSearch(qstr, 0, count, "")
|
|
|
-// }
|
|
|
-// secondKWS := jy.HttpEs(scd.Keyword[0].Keyword, "ik_smart", elasticAddress)
|
|
|
-// scd.Keyword[0].Keyword = secondKWS
|
|
|
-// scd.SelectType = "title"
|
|
|
-// qstr = getDataExportSql(scd)
|
|
|
-// log.Printf("GetDataExportSearchResultUseId-%s-分词-sql:%s\n", scd.Id, qstr)
|
|
|
-// res2 := doSearch(qstr, 0, 100, "")
|
|
|
-// if len(*res2) > 100 {
|
|
|
-// res2_temp := (*res2)[:100]
|
|
|
-// res2 = &res2_temp
|
|
|
-// }
|
|
|
-// return delRepeatMapArr(res, res2), kws
|
|
|
-// }
|
|
|
-// }
|
|
|
-// //非空查询
|
|
|
-// res = *doSearch(qstr, 0, 500, dataType)
|
|
|
-// return &res, kws
|
|
|
-//}
|
|
|
-
|
|
|
func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdomain string, dataType string, encry ...bool) *[]map[string]interface{} {
|
|
|
//格式化输出
|
|
|
isEncry := false
|
|
|
@@ -543,7 +508,7 @@ func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdo
|
|
|
//查询企业公示 法人 公司电话 公司邮箱地址
|
|
|
s_winner, ok := v["s_winner"].(string) //改为entidlist？
|
|
|
if ok && s_winner != "" {
|
|
|
- if entData, ok := entmg.Find("winner_enterprise", bson.M{"company_name": s_winner}, nil, `{"company_name":1,"company_email":1,"legal_person":1,"company_phone":1}`, false, -1, -1); ok {
|
|
|
+ if entData, ok := entmg.Find("winner_enterprise", map[string]interface{}{"company_name": s_winner}, nil, `{"company_name":1,"company_email":1,"legal_person":1,"company_phone":1}`, false, -1, -1); ok {
|
|
|
if entData != nil && *entData != nil && len(*entData) > 0 {
|
|
|
for _, ev := range *entData {
|
|
|
if v["s_winner"] == ev["company_name"] {
|
wangkaiyue