|
|
@@ -1,1001 +1,1002 @@
|
|
|
package public
|
|
|
|
|
|
-import (
|
|
|
- "encoding/json"
|
|
|
- "errors"
|
|
|
- "fmt"
|
|
|
- "log"
|
|
|
- "math"
|
|
|
- "qfw/util"
|
|
|
- "qfw/util/elastic"
|
|
|
- "qfw/util/jy"
|
|
|
- "qfw/util/redis"
|
|
|
- "sort"
|
|
|
- "strconv"
|
|
|
- "strings"
|
|
|
- "sync"
|
|
|
- "time"
|
|
|
-
|
|
|
- "go.mongodb.org/mongo-driver/bson"
|
|
|
-)
|
|
|
-
|
|
|
-/*筛选条件--关键词*/
|
|
|
-type KeyWord struct {
|
|
|
- Keyword string `json:"keyword"` //关键词
|
|
|
- Appended []string `json:"appended"` //附加词
|
|
|
- Exclude []string `json:"exclude"` //排除词
|
|
|
-}
|
|
|
-
|
|
|
-/*筛选条件*/
|
|
|
-type SieveCondition struct {
|
|
|
- Id string `json:"id"`
|
|
|
- PublishTime string `json:"publishtime"` //发布时间
|
|
|
- Area []string `json:"area"` //地区-省份
|
|
|
- City []string `json:"city"` //地区-城市
|
|
|
- Region []string `json:"region"` //地区-省份+城市
|
|
|
- Industry []string `json:"industry"` //行业
|
|
|
- Keyword []KeyWord `json:"keywords"` //关键词
|
|
|
- Buyer []string `json:"buyer"` //招标单位(采购单位)
|
|
|
- Buyerclass []string `json:"buyerclass"` //采购单位类型
|
|
|
- Winner []string `json:"winner"` //中标单位
|
|
|
- ComeInTime int64 `json:"comeintime"` //入库时间(秒)
|
|
|
- OpenId string `json:"openid"` //用户openid
|
|
|
- MinPrice string `json:"minprice"` //金额——最少
|
|
|
- MaxPrice string `json:"maxprice"` //金额——最多
|
|
|
- SelectType string `json:"selectType"` //筛选（正文 or 标题）
|
|
|
- Subtype string `json:"subtype"` //信息类型
|
|
|
- Comeinfrom string `json:"comeinfrom"` //查询来源
|
|
|
-}
|
|
|
-
|
|
|
-const (
|
|
|
- INDEX = "bidding"
|
|
|
- TYPE = "bidding"
|
|
|
- bidSearch_sort = `{"publishtime":-1}`
|
|
|
-)
|
|
|
-
|
|
|
-var ExportTable string = "export_search"
|
|
|
-
|
|
|
-//获取数据导出查询语句
|
|
|
-func getDataExportSql(scd *SieveCondition) string {
|
|
|
- multi_match := `{"multi_match": {"query": %s,"type": "phrase", "fields": [%s]}}`
|
|
|
- query := `{"query":{"bool":{"must":[%s],"should":[%s],"minimum_should_match": %d}}}`
|
|
|
- query_bool_should := `{"bool":{"should":[%s],"minimum_should_match": 1}}`
|
|
|
- query_price := `{"bool":{"must":[{"range":{"bidamount":{%s}}}]}},{"bool":{"must":[{"range":{"budget":{%s}}}],"must_not":[{"range":{"bidamount":{"gte":-1}}}]}}`
|
|
|
- query_bool_must := `{"terms":{"%s":[%s]}}`
|
|
|
- query_bool_must_and := `{"bool":{"must":[%s]%s}}`
|
|
|
- gte := `"gte": %s`
|
|
|
- lte := `"lte": %s`
|
|
|
-
|
|
|
- bools := []string{}
|
|
|
- musts := []string{fmt.Sprintf(`{"range":{"comeintime":{"lt":%d}}}`, scd.ComeInTime)}
|
|
|
- //省份
|
|
|
- areaCity := []string{}
|
|
|
- if len(scd.Area) > 0 {
|
|
|
- areaquery := `{"terms":{"area":[`
|
|
|
- for k, v := range scd.Area {
|
|
|
- if k > 0 {
|
|
|
- areaquery += `,`
|
|
|
- }
|
|
|
- areaquery += `"` + v + `"`
|
|
|
- }
|
|
|
- areaquery += `]}}`
|
|
|
- areaCity = append(areaCity, areaquery)
|
|
|
- }
|
|
|
- //城市
|
|
|
- if len(scd.City) > 0 {
|
|
|
- areaquery := `{"terms":{"city":[`
|
|
|
- for k, v := range scd.City {
|
|
|
- if k > 0 {
|
|
|
- areaquery += `,`
|
|
|
- }
|
|
|
- areaquery += `"` + v + `"`
|
|
|
- }
|
|
|
- areaquery += `]}}`
|
|
|
- areaCity = append(areaCity, areaquery)
|
|
|
- }
|
|
|
- if len(areaCity) > 0 {
|
|
|
- musts = append(musts, fmt.Sprintf(query_bool_should, strings.Join(areaCity, ",")))
|
|
|
- }
|
|
|
- //检索日期
|
|
|
-
|
|
|
- starttime := ""
|
|
|
- now := time.Unix(scd.ComeInTime, 0)
|
|
|
- endtime := fmt.Sprintf("%d", now.Unix())
|
|
|
- if scd.PublishTime == "lately-7" { //最近7天
|
|
|
- starttime = fmt.Sprint(time.Date(now.Year(), now.Month(), now.Day()-7, 0, 0, 0, 0, time.Local).Unix())
|
|
|
- } else if scd.PublishTime == "lately-30" { //最近30天
|
|
|
- starttime = fmt.Sprint(time.Date(now.Year(), now.Month(), now.Day()-30, 0, 0, 0, 0, time.Local).Unix())
|
|
|
- } else if scd.PublishTime == "thisyear" { //去年
|
|
|
- starttime = fmt.Sprint(time.Date(now.Year()-1, 1, 1, 0, 0, 0, 0, time.Local).Unix())
|
|
|
- endtime = fmt.Sprint(time.Date(now.Year()-1, 12, 31, 23, 59, 59, 0, time.Local).Unix())
|
|
|
- } else if strings.Contains(scd.PublishTime, "_") { //设置检索日期
|
|
|
- starttime = strings.Split(scd.PublishTime, "_")[0]
|
|
|
- endTime_tmp := now
|
|
|
- if etime := strings.Split(scd.PublishTime, "_")[1]; etime != "" {
|
|
|
- etTime := time.Unix(util.Int64All(etime), 0)
|
|
|
- endTime_tmp = time.Date(etTime.Year(), etTime.Month(), etTime.Day()+1, 0, 0, 0, 0, time.Local)
|
|
|
- }
|
|
|
- //结束时间必须小于筛选时间
|
|
|
- if endTime_tmp.After(now) {
|
|
|
- endTime_tmp = now
|
|
|
- }
|
|
|
- endtime = fmt.Sprintf("%d", endTime_tmp.Unix())
|
|
|
- }
|
|
|
- timequery := `{"range":{"publishtime":{`
|
|
|
- if starttime != "" {
|
|
|
- timequery += `"gte":` + starttime
|
|
|
- }
|
|
|
- if starttime != "" && endtime != "" {
|
|
|
- timequery += `,`
|
|
|
- }
|
|
|
- if endtime != "" {
|
|
|
- timequery += `"lt":` + endtime
|
|
|
- }
|
|
|
- timequery += `}}}`
|
|
|
- musts = append(musts, timequery)
|
|
|
-
|
|
|
- if scd.Subtype != "" {
|
|
|
- subquery := `{"terms":{"subtype":[`
|
|
|
- for k, v := range strings.Split(scd.Subtype, ",") {
|
|
|
- if k > 0 {
|
|
|
- subquery += `,`
|
|
|
- }
|
|
|
- subquery += `"` + v + `"`
|
|
|
- }
|
|
|
- subquery += `]}}`
|
|
|
- musts = append(musts, subquery)
|
|
|
- }
|
|
|
- if len(scd.Industry) > 0 {
|
|
|
- musts = append(musts, fmt.Sprintf(query_bool_must, "s_subscopeclass", `"`+strings.Join(scd.Industry, `","`)+`"`))
|
|
|
- }
|
|
|
- if len(scd.Buyer) > 0 {
|
|
|
- musts = append(musts, fmt.Sprintf(query_bool_must, "buyer", `"`+strings.Join(scd.Buyer, `","`)+`"`))
|
|
|
- }
|
|
|
- if len(scd.Buyerclass) > 0 {
|
|
|
- musts = append(musts, fmt.Sprintf(query_bool_must, "buyerclass", `"`+strings.Join(scd.Buyerclass, `","`)+`"`))
|
|
|
- }
|
|
|
- if len(scd.Winner) > 0 {
|
|
|
- musts = append(musts, fmt.Sprintf(query_bool_must, "s_winner", `"`+strings.Join(scd.Winner, `","`)+`"`))
|
|
|
- }
|
|
|
- _minPrice := ""
|
|
|
- _maxPrice := ""
|
|
|
- if scd.MinPrice != "" || scd.MaxPrice != "" {
|
|
|
- sq := ``
|
|
|
- if scd.MinPrice != "" {
|
|
|
- min, _ := strconv.ParseFloat(scd.MinPrice, 64)
|
|
|
- _minPrice = fmt.Sprintf("%.0f", min*10000)
|
|
|
- if _minPrice == "0" {
|
|
|
- _minPrice = ""
|
|
|
- }
|
|
|
- }
|
|
|
- if scd.MaxPrice != "" {
|
|
|
- max, _ := strconv.ParseFloat(scd.MaxPrice, 64)
|
|
|
- _maxPrice = fmt.Sprintf("%.0f", max*10000)
|
|
|
- if _maxPrice == "0" {
|
|
|
- _maxPrice = ""
|
|
|
- }
|
|
|
- }
|
|
|
- if _minPrice != "" {
|
|
|
- sq += fmt.Sprintf(gte, _minPrice)
|
|
|
- }
|
|
|
- if _minPrice != "" && _maxPrice != "" {
|
|
|
- sq += `,`
|
|
|
- }
|
|
|
- if _maxPrice != "" {
|
|
|
- sq += fmt.Sprintf(lte, _maxPrice)
|
|
|
- }
|
|
|
- if _minPrice != "" || _maxPrice != "" {
|
|
|
- query_price := fmt.Sprintf(query_bool_should, fmt.Sprintf(query_price, sq, sq))
|
|
|
- musts = append(musts, query_price)
|
|
|
- }
|
|
|
- }
|
|
|
- boolsNum := 0
|
|
|
- //should
|
|
|
- if len(scd.Keyword) > 0 {
|
|
|
- boolsNum = 1
|
|
|
- if scd.SelectType == "" || scd.SelectType == "all" {
|
|
|
- scd.SelectType = "detail\", \"title"
|
|
|
- }
|
|
|
- multi_match = fmt.Sprintf(multi_match, "%s", "\""+scd.SelectType+"\"")
|
|
|
-
|
|
|
- if scd.Comeinfrom == "supersearchPage" {
|
|
|
- var keywordArr []string
|
|
|
- if strings.Contains(scd.Keyword[0].Keyword, "+") {
|
|
|
- keywordArr = strings.Split(scd.Keyword[0].Keyword, "+")
|
|
|
- } else if strings.Contains(scd.Keyword[0].Keyword, " ") {
|
|
|
- keywordArr = strings.Split(scd.Keyword[0].Keyword, " ")
|
|
|
- }
|
|
|
- if len(keywordArr) > 1 {
|
|
|
- KeyWordSearch := KeyWord{}
|
|
|
- for _, v := range keywordArr {
|
|
|
- KeyWordSearch.Appended = append(KeyWordSearch.Appended, v)
|
|
|
- }
|
|
|
- scd.Keyword = []KeyWord{KeyWordSearch}
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- for _, v := range scd.Keyword {
|
|
|
- shoulds := []string{}
|
|
|
- must_not := []string{}
|
|
|
- //附加词
|
|
|
- if v.Keyword != "" {
|
|
|
- shoulds = append(shoulds, fmt.Sprintf(multi_match, "\""+v.Keyword+"\""))
|
|
|
- }
|
|
|
-
|
|
|
- for _, vv := range v.Appended {
|
|
|
- shoulds = append(shoulds, fmt.Sprintf(multi_match, "\""+vv+"\""))
|
|
|
- }
|
|
|
-
|
|
|
- //排除词
|
|
|
- for _, vv := range v.Exclude {
|
|
|
- must_not = append(must_not, fmt.Sprintf(multi_match, "\""+vv+"\""))
|
|
|
- }
|
|
|
-
|
|
|
- //添加
|
|
|
- if len(shoulds) > 0 {
|
|
|
- notStr := ""
|
|
|
- if len(must_not) > 0 {
|
|
|
- notStr = fmt.Sprintf(`,"must_not":[%s]`, strings.Join(must_not, ","))
|
|
|
- }
|
|
|
- bools = append(bools, fmt.Sprintf(query_bool_must_and, strings.Join(shoulds, ","), notStr))
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- qstr := fmt.Sprintf(query, strings.Join(musts, ","), strings.Join(bools, ","), boolsNum)
|
|
|
- return qstr
|
|
|
-}
|
|
|
-
|
|
|
-func getSqlObjFromId(_id string) *SieveCondition {
|
|
|
- var (
|
|
|
- query *map[string]interface{}
|
|
|
- ok bool
|
|
|
- )
|
|
|
- if query, ok = MQFW.FindById(ExportTable, _id, nil); !ok {
|
|
|
- return nil
|
|
|
- }
|
|
|
- return &SieveCondition{
|
|
|
- Id: _id,
|
|
|
- Keyword: getKeyWordArrFromDbResult((*query)["keywords"]),
|
|
|
- Industry: getStringArrFromDbResult((*query)["industry"]),
|
|
|
- MinPrice: util.ObjToString((*query)["minprice"]),
|
|
|
- MaxPrice: util.ObjToString((*query)["maxprice"]),
|
|
|
- Subtype: util.ObjToString((*query)["subtype"]),
|
|
|
- Area: getStringArrFromDbResult((*query)["area"]),
|
|
|
- City: getStringArrFromDbResult((*query)["city"]),
|
|
|
- SelectType: util.ObjToString((*query)["selectType"]),
|
|
|
- PublishTime: util.ObjToString((*query)["publishtime"]),
|
|
|
- Buyer: getStringArrFromDbResult((*query)["buyer"]),
|
|
|
- Buyerclass: getStringArrFromDbResult((*query)["buyerclass"]),
|
|
|
- Winner: getStringArrFromDbResult((*query)["winner"]),
|
|
|
- ComeInTime: util.Int64All((*query)["comeintime"]),
|
|
|
- Comeinfrom: util.ObjToString((*query)["comeinfrom"]),
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-//数据导出-查询结果数量
|
|
|
-func GetDataExportSearchCountUseId(_id string) (count int) {
|
|
|
- scd := getSqlObjFromId(_id)
|
|
|
- qstr := getDataExportSql(scd)
|
|
|
- log.Printf("GetDataExportSearchCountUseId-%s-count:%d-sql:%s\n", _id, count, qstr)
|
|
|
- if isNullSearch(scd) {
|
|
|
- return ExConf.MsgMaxCount
|
|
|
- }
|
|
|
- count = int(elastic.Count(INDEX, TYPE, qstr))
|
|
|
- //超级搜索一致的检索（防止数据导出和超级搜索数据量不一致）
|
|
|
- if scd.Comeinfrom == "supersearchPage" && (len(scd.Keyword) != 0 || len(scd.Industry) != 0) {
|
|
|
- if len(scd.Keyword) != 0 {
|
|
|
- searchTextSize := 0
|
|
|
- if len(scd.Keyword) > 0 {
|
|
|
- searchTextSize = len([]rune(scd.Keyword[0].Keyword))
|
|
|
- }
|
|
|
- if searchTextSize > 3 && count < 50 {
|
|
|
- var res *[]map[string]interface{}
|
|
|
- if count > 0 {
|
|
|
- res = doSearch(qstr, 0, count, "")
|
|
|
- }
|
|
|
- secondKWS := jy.HttpEs(scd.Keyword[0].Keyword, "ik_smart", DbConf.Elasticsearch.Main.Address)
|
|
|
- scd.Keyword[0].Keyword = secondKWS
|
|
|
- scd.SelectType = "title"
|
|
|
- qstr = getDataExportSql(scd)
|
|
|
- res2 := doSearch(qstr, 0, 100, "")
|
|
|
- result := len(*delRepeatMapArr(res, res2))
|
|
|
- if result > 100 {
|
|
|
- result = 100
|
|
|
- }
|
|
|
- log.Printf("GetDataExportSearchCountUseId-%s-count:%d-分词-sql:%s\n", _id, result, qstr)
|
|
|
- return result
|
|
|
- }
|
|
|
- return
|
|
|
- }
|
|
|
- }
|
|
|
- log.Printf("GetDataExportSearchCountUseId-%s-count:%d\n", _id, count)
|
|
|
- return
|
|
|
-}
|
|
|
-
|
|
|
-//合并map数据，去重
|
|
|
-func delRepeatMapArr(res *[]map[string]interface{}, res2 *[]map[string]interface{}) *[]map[string]interface{} {
|
|
|
- if res != nil {
|
|
|
- for _, v := range *res {
|
|
|
- for n, m := range *res2 {
|
|
|
- if util.ObjToString(v["_id"]) == util.ObjToString(m["_id"]) {
|
|
|
- *res2 = append((*res2)[0:n], (*res2)[n+1:]...)
|
|
|
- break
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- *res = append(*res, *res2...)
|
|
|
- } else {
|
|
|
- res = res2
|
|
|
- }
|
|
|
- return res
|
|
|
-}
|
|
|
-
|
|
|
-//查询条件是否为空
|
|
|
-func isNullSearch(scd *SieveCondition) (isNull bool) {
|
|
|
- if scd.PublishTime == "" && len(scd.Area) == 0 && len(scd.Industry) == 0 && len(scd.Keyword) == 0 && len(scd.Buyer) == 0 && len(scd.Winner) == 0 && scd.MinPrice == "" && scd.MaxPrice == "" && scd.Subtype == "" && len(scd.City) == 0 {
|
|
|
- isNull = true
|
|
|
- }
|
|
|
- return isNull
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * 数据导出 查询结果
|
|
|
- * _id 数据库查询条件记录id
|
|
|
- * dataType 1-普通字段 2-高级字段
|
|
|
- * webdomain 三级页域名
|
|
|
- * count 返回数量 (-1:预览数据查询)
|
|
|
- */
|
|
|
-var EntTable = "winner_enterprise"
|
|
|
-
|
|
|
-func GetDataExportSearchResultUseId(_id, dataType string, count int) (*[]map[string]interface{}, []KeyWord) {
|
|
|
- defer util.Catch()
|
|
|
- var res []map[string]interface{}
|
|
|
- var kws []KeyWord
|
|
|
- scd := getSqlObjFromId(_id)
|
|
|
- //获取查询语句
|
|
|
- qstr := getDataExportSql(scd)
|
|
|
- log.Printf("GetDataExportSearchResultUseId-%s-sql:%s\n", scd.Id, qstr)
|
|
|
- kws = scd.Keyword
|
|
|
- //数据预览数据查询
|
|
|
- if scd.Comeinfrom == "supersearchPage" && len(scd.Keyword) == 0 && len(scd.Industry) == 0 {
|
|
|
- //空查询
|
|
|
- obj := redis.Get("other", "export_news")
|
|
|
- if obj != nil {
|
|
|
- res = util.ObjArrToMapArr(obj.([]interface{}))
|
|
|
- } else {
|
|
|
- res = *doSearch(qstr, 0, 500, "2")
|
|
|
- if len(res) > 0 {
|
|
|
- redis.Put("other", "export_news", res, 7200)
|
|
|
- }
|
|
|
- }
|
|
|
- return &res, kws
|
|
|
- } else if scd.Comeinfrom == "supersearchPage" {
|
|
|
- //超级搜索非空查询
|
|
|
- count = int(elastic.Count(INDEX, TYPE, qstr))
|
|
|
- searchTextSize := 0
|
|
|
- if len(scd.Keyword) > 0 {
|
|
|
- searchTextSize = len([]rune(scd.Keyword[0].Keyword))
|
|
|
- }
|
|
|
- if searchTextSize > 3 && count < 50 {
|
|
|
- var res *[]map[string]interface{}
|
|
|
- if count > 0 {
|
|
|
- res = doSearch(qstr, 0, count, "")
|
|
|
- }
|
|
|
- secondKWS := jy.HttpEs(scd.Keyword[0].Keyword, "ik_smart", DbConf.Elasticsearch.Main.Address)
|
|
|
- scd.Keyword[0].Keyword = secondKWS
|
|
|
- scd.SelectType = "title"
|
|
|
- qstr = getDataExportSql(scd)
|
|
|
- log.Printf("GetDataExportSearchResultUseId-%s-分词-sql:%s\n", scd.Id, qstr)
|
|
|
- res2 := doSearch(qstr, 0, 100, "")
|
|
|
- if len(*res2) > 100 {
|
|
|
- res2_temp := (*res2)[:100]
|
|
|
- res2 = &res2_temp
|
|
|
- }
|
|
|
- return delRepeatMapArr(res, res2), kws
|
|
|
- }
|
|
|
- }
|
|
|
- //非空查询
|
|
|
- res = *doSearch(qstr, 0, 500, dataType)
|
|
|
- return &res, kws
|
|
|
-}
|
|
|
-
|
|
|
-func FormatExportData(data *[]map[string]interface{}, webdomain string, dataType string, EntArr []string) *[]map[string]interface{} {
|
|
|
- //格式化输出
|
|
|
- for _, v := range *data {
|
|
|
- //有中标企业 且 高级字段查询
|
|
|
- if len(EntArr) > 0 && dataType == "2" {
|
|
|
- //查询企业公示 法人 公司电话 公司邮箱地址
|
|
|
- query := bson.M{"company_name": bson.M{"$in": EntArr}} //
|
|
|
- if entData, ok := Mgo_Ent.Find(EntTable, query, nil, `{"company_name":1,"company_email":1,"legal_person":1,"company_phone":1}`, false, -1, -1); ok {
|
|
|
- if entData != nil && *entData != nil && len(*entData) > 0 {
|
|
|
- for _, ev := range *entData {
|
|
|
- if v["s_winner"] == ev["company_name"] {
|
|
|
- legal_person := ""
|
|
|
- if ev["legal_person"] != nil && ev["legal_person"].(string) != "" {
|
|
|
- legal_person = ev["legal_person"].(string)
|
|
|
- var xx = "*"
|
|
|
- switch len([]rune(legal_person)) {
|
|
|
- case 3:
|
|
|
- xx = "**"
|
|
|
- case 4:
|
|
|
- xx = "***"
|
|
|
- }
|
|
|
- legal_person = string([]rune(legal_person)[:1]) + xx
|
|
|
- }
|
|
|
- company_phone := ""
|
|
|
- if ev["company_phone"] != nil && ev["company_phone"].(string) != "" {
|
|
|
- company_phone = ev["company_phone"].(string)
|
|
|
- if len([]rune(company_phone)) > 7 {
|
|
|
- company_phone = company_phone[:7] + "****"
|
|
|
- } else {
|
|
|
- company_phone = "****"
|
|
|
- }
|
|
|
- }
|
|
|
- company_email := ""
|
|
|
- if ev["company_email"] != nil && ev["company_email"].(string) != "无" {
|
|
|
- company_email = ev["company_email"].(string)
|
|
|
- if len(strings.Split(company_email, "@")) > 1 {
|
|
|
- company_email = "******" + "@" + strings.Split(company_email, "@")[1]
|
|
|
- }
|
|
|
- }
|
|
|
- v["legal_person"] = legal_person
|
|
|
- v["company_phone"] = company_phone
|
|
|
- v["company_email"] = company_email
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- //====================字段补漏=========================
|
|
|
- if v["toptype"] == "结果" && dataType == "2" && !(v["agency"] != nil && v["budget"] != nil && v["buyerperson"] != nil && v["buyertel"] != nil) {
|
|
|
- r := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query":{"term":{"list.infoid":"%s"}},"_source": ["list"]}`, v["_id"]))
|
|
|
- if len(*r) > 0 {
|
|
|
- MsgList := (*r)[0]["list"]
|
|
|
- if MsgList != nil {
|
|
|
- list := util.ObjArrToMapArr(MsgList.([]interface{}))
|
|
|
- for _, vv := range list {
|
|
|
- if vv["subtype"] == "招标" {
|
|
|
- if v["agency"] == nil && vv["agency"] != nil {
|
|
|
- v["agency"] = vv["agency"]
|
|
|
- }
|
|
|
- if v["budget"] == nil && vv["budget"] != nil {
|
|
|
- v["budget"] = vv["budget"]
|
|
|
- }
|
|
|
- if v["buyerperson"] == nil && vv["buyerperson"] != nil {
|
|
|
- v["buyerperson"] = vv["buyerperson"]
|
|
|
- }
|
|
|
- if v["buyertel"] == nil && vv["buyertel"] != nil {
|
|
|
- v["buyertel"] = vv["buyertel"]
|
|
|
- }
|
|
|
- break
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- if v["area"] == "A" {
|
|
|
- v["area"] = "全国"
|
|
|
- }
|
|
|
- if v["bidamount"] != nil {
|
|
|
- v["bidamount"] = formatFloat(util.Float64All(v["bidamount"]))
|
|
|
- }
|
|
|
- if v["budget"] != nil {
|
|
|
- v["budget"] = formatFloat(util.Float64All(v["budget"]))
|
|
|
- }
|
|
|
- if v["publishtime"] != nil {
|
|
|
- date := v["publishtime"]
|
|
|
- v["publishtime"] = util.FormatDateWithObj(&date, util.Date_Short_Layout)
|
|
|
- }
|
|
|
- if v["bidopentime"] != nil {
|
|
|
- date := v["bidopentime"]
|
|
|
- v["bidopentime"] = util.FormatDateWithObj(&date, util.Date_Short_Layout)
|
|
|
- }
|
|
|
- if v["_id"] != nil {
|
|
|
- v["url"] = webdomain + "/article/content/" + util.CommonEncodeArticle("content", v["_id"].(string)) + ".html"
|
|
|
- }
|
|
|
- if v["currency"] == "" || v["currency"] == nil {
|
|
|
- v["currency"] = "人民币"
|
|
|
- }
|
|
|
- if v["projectscope"] != "" && v["projectscope"] != nil {
|
|
|
- str := ClearHtml.ReplaceAllString(v["projectscope"].(string), "")
|
|
|
- str = ClearOther.ReplaceAllString(str, "")
|
|
|
- str = strings.Replace(str, " ", "", -1)
|
|
|
- if len([]rune(str)) > 100 {
|
|
|
- str = util.SubString(str, 0, 100) + "..."
|
|
|
- }
|
|
|
- v["projectscope"] = str
|
|
|
- }
|
|
|
- if v["detail"] != "" && v["detail"] != nil {
|
|
|
- str := ClearHtml.ReplaceAllString(v["detail"].(string), "")
|
|
|
- str = ClearOther.ReplaceAllString(str, "")
|
|
|
- str = strings.Replace(str, " ", "", -1)
|
|
|
- if len([]rune(str)) > 100 {
|
|
|
- str = util.SubString(str, 0, 100) + "..."
|
|
|
- }
|
|
|
- v["detail"] = str
|
|
|
- }
|
|
|
- if v["title"] != "" && v["title"] != nil {
|
|
|
- str := ClearHtml.ReplaceAllString(v["title"].(string), "")
|
|
|
- str = ClearOther.ReplaceAllString(str, "")
|
|
|
- str = strings.Replace(str, " ", "", -1)
|
|
|
- if len([]rune(str)) > 100 {
|
|
|
- str = util.SubString(str, 0, 100) + "..."
|
|
|
- }
|
|
|
- v["title"] = str
|
|
|
- }
|
|
|
- if v["subtype"] == nil && v["toptype"] != nil {
|
|
|
- v["subtype"] = v["toptype"]
|
|
|
- }
|
|
|
- }
|
|
|
- return data
|
|
|
-}
|
|
|
-
|
|
|
-//保留到0.01分
|
|
|
-func formatFloat(value float64) string {
|
|
|
- str := strings.TrimRight(fmt.Sprintf("%.7f", value*10000/100000000), "0")
|
|
|
- if str[len(str)-1:] == "." {
|
|
|
- return str[:len(str)-1]
|
|
|
- }
|
|
|
- return str
|
|
|
-}
|
|
|
-
|
|
|
-func FormatExportDatas(data *[]map[string]interface{}, webdomain string, dataType string, entId int) *[]map[string]interface{} {
|
|
|
- //格式化输出
|
|
|
- var (
|
|
|
- entexportPool = make(chan bool, 20)
|
|
|
- entexportWaitGroup = &sync.WaitGroup{}
|
|
|
- // newsDatalen = make(chan bool, 20000)
|
|
|
- // datalock = sync.RWMutex{}
|
|
|
- )
|
|
|
- log.Println("补充信息开始")
|
|
|
- for _, v := range *data {
|
|
|
- entexportWaitGroup.Add(1)
|
|
|
- entexportPool <- true
|
|
|
- go func(v map[string]interface{}) {
|
|
|
- defer func() {
|
|
|
- entexportWaitGroup.Done()
|
|
|
- <-entexportPool
|
|
|
- // datalock.Unlock()
|
|
|
- }()
|
|
|
- // datalock.Lock()
|
|
|
- //二次验证
|
|
|
- // id := util.ObjToString(v["_id"])
|
|
|
- // isExist, err := redis.Exists("other", "entexportdata_"+id+"_"+fmt.Sprintln(entId))
|
|
|
- // if err != nil {
|
|
|
- // log.Println("企业搜索数据导出redis判重失败")
|
|
|
- // } else if !isExist {
|
|
|
- // newsDatalen <- true
|
|
|
- // }
|
|
|
- //
|
|
|
- //有中标企业 且 高级字段查询
|
|
|
- if dataType == "2" {
|
|
|
- //查询企业公示 法人 公司电话 公司邮箱地址
|
|
|
- // query := bson.M{"company_name": bson.M{"$in": EntArr}} //
|
|
|
- s_winner := strings.Split(util.ObjToString(v["s_winner"]), ",")[0]
|
|
|
- query := bson.M{"company_name": s_winner} //
|
|
|
- if entData, ok := Mgo_Ent.Find(EntTable, query, nil, `{"company_name":1,"company_email":1,"legal_person":1,"company_phone":1}`, false, -1, -1); ok {
|
|
|
- if entData != nil && *entData != nil && len(*entData) > 0 {
|
|
|
- for _, ev := range *entData {
|
|
|
- if v["s_winner"] == ev["company_name"] {
|
|
|
- legal_person := ""
|
|
|
- if ev["legal_person"] != nil {
|
|
|
- legal_person = ev["legal_person"].(string)
|
|
|
- }
|
|
|
- company_phone := ""
|
|
|
- if ev["company_phone"] != nil {
|
|
|
- company_phone = ev["company_phone"].(string)
|
|
|
- }
|
|
|
- company_email := ""
|
|
|
- if ev["company_email"] != nil && ev["company_email"] != "无" {
|
|
|
- company_email = ev["company_email"].(string)
|
|
|
- }
|
|
|
- v["legal_person"] = legal_person
|
|
|
- v["company_phone"] = company_phone
|
|
|
- v["company_email"] = company_email
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- //====================字段补漏=========================
|
|
|
- if v["toptype"] == "结果" && dataType == "2" && !(v["agency"] != nil && v["budget"] != nil && v["buyerperson"] != nil && v["buyertel"] != nil) {
|
|
|
- r := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query":{"term":{"list.infoid":"%s"}},"_source": ["list"]}`, v["_id"]))
|
|
|
- if len(*r) > 0 {
|
|
|
- MsgList := (*r)[0]["list"]
|
|
|
- if MsgList != nil {
|
|
|
- list := util.ObjArrToMapArr(MsgList.([]interface{}))
|
|
|
- for _, vv := range list {
|
|
|
- if vv["subtype"] == "招标" {
|
|
|
- if v["agency"] == nil && vv["agency"] != nil {
|
|
|
- v["agency"] = vv["agency"]
|
|
|
- }
|
|
|
- if v["budget"] == nil && vv["budget"] != nil {
|
|
|
- v["budget"] = vv["budget"]
|
|
|
- }
|
|
|
- if v["buyerperson"] == nil && vv["buyerperson"] != nil {
|
|
|
- v["buyerperson"] = vv["buyerperson"]
|
|
|
- }
|
|
|
- if v["buyertel"] == nil && vv["buyertel"] != nil {
|
|
|
- v["buyertel"] = vv["buyertel"]
|
|
|
- }
|
|
|
- break
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- if v["area"] == "A" {
|
|
|
- v["area"] = "全国"
|
|
|
- }
|
|
|
- // if v["bidamount"] != nil {
|
|
|
- // v["bidamount"] = fmt.Sprintf("%g", util.Float64All(v["bidamount"]))
|
|
|
- // }
|
|
|
- // if v["budget"] != nil {
|
|
|
- // v["budget"] = fmt.Sprintf("%g", util.Float64All(v["budget"]))
|
|
|
- // }
|
|
|
- if v["publishtime"] != nil {
|
|
|
- date := v["publishtime"]
|
|
|
- v["publishtime"] = util.FormatDateWithObj(&date, util.Date_Short_Layout)
|
|
|
- }
|
|
|
- if v["bidopentime"] != nil {
|
|
|
- date := v["bidopentime"]
|
|
|
- v["bidopentime"] = util.FormatDateWithObj(&date, util.Date_Short_Layout)
|
|
|
- }
|
|
|
- if v["currency"] == "" || v["currency"] == nil {
|
|
|
- v["currency"] = "人民币"
|
|
|
- }
|
|
|
- if v["subtype"] == nil && v["toptype"] != nil {
|
|
|
- v["subtype"] = v["toptype"]
|
|
|
- }
|
|
|
- if v["detail"] != "" && v["detail"] != nil {
|
|
|
- str := ClearHtml.ReplaceAllString(v["detail"].(string), "")
|
|
|
- str = ClearOther.ReplaceAllString(str, "")
|
|
|
- str = strings.Replace(str, " ", "", -1)
|
|
|
- v["detail"] = str
|
|
|
- }
|
|
|
- if v["_id"] != nil {
|
|
|
- v["url"] = webdomain + "/article/content/" + util.CommonEncodeArticle("content", v["_id"].(string)) + ".html"
|
|
|
- }
|
|
|
- }(v)
|
|
|
- }
|
|
|
- entexportWaitGroup.Wait()
|
|
|
- log.Println("补充信息结束")
|
|
|
- return data
|
|
|
-}
|
|
|
-
|
|
|
-func doSearch(sql string, start, count int, dataType string) *[]map[string]interface{} {
|
|
|
- if sql != "" {
|
|
|
- //筛选字段
|
|
|
- if dataType != "" {
|
|
|
- dataexport_field := `"_id","title","detail","area","city","publishtime","projectname","buyer","s_winner","bidamount","subtype","toptype"`
|
|
|
- if dataType == "2" {
|
|
|
- dataexport_field += `,"href","projectcode","buyerperson","buyertel","budget","bidopentime","agency","projectscope","winnerperson","winnertel"`
|
|
|
- }
|
|
|
- sql = sql[:len(sql)-1] + `,"_source":[` + dataexport_field + "]}"
|
|
|
- }
|
|
|
- //分页排序
|
|
|
- sql = sql[:len(sql)-1] + `,"sort": {"publishtime":"desc"},"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(count) + "}"
|
|
|
- }
|
|
|
- return elastic.Get(INDEX, TYPE, sql)
|
|
|
-}
|
|
|
-
|
|
|
-func getKeyWordArrFromDbResult(k interface{}) (arr []KeyWord) {
|
|
|
- if k == nil {
|
|
|
- return
|
|
|
- }
|
|
|
- kArr := k.([]interface{})
|
|
|
- for _, v := range kArr {
|
|
|
- kw := KeyWord{}
|
|
|
- b, e := json.Marshal(v)
|
|
|
- if e != nil {
|
|
|
- log.Println(e.Error())
|
|
|
- }
|
|
|
- json.Unmarshal(b, &kw)
|
|
|
- arr = append(arr, kw)
|
|
|
- }
|
|
|
- return
|
|
|
-}
|
|
|
-
|
|
|
-func getStringArrFromDbResult(c interface{}) (arr []string) {
|
|
|
- if c != nil {
|
|
|
- cArr := c.([]interface{})
|
|
|
- arr = util.ObjArrToStringArr(cArr)
|
|
|
- }
|
|
|
- return
|
|
|
-}
|
|
|
-
|
|
|
-//获取结果,空字段最少的数据
|
|
|
-func ScreenData(arr *[]map[string]interface{}, dataType string, resultNum int, kws []KeyWord) (res []map[string]interface{}) {
|
|
|
- AllMap := map[int][]map[string]interface{}{}
|
|
|
-
|
|
|
- NoKwsMap := map[int][]map[string]interface{}{}
|
|
|
- lastNum := resultNum
|
|
|
- for _, v := range *arr {
|
|
|
- emptyNum := countOfTheEmpty(v, dataType)
|
|
|
- if emptyNum == -1 {
|
|
|
- continue
|
|
|
- }
|
|
|
- if len(kws) > 0 && kws[0].Keyword != "" {
|
|
|
- var kwsFlag = true
|
|
|
- for _, vk := range kws {
|
|
|
- if strings.Contains(util.ObjToString(v["title"]), strings.Replace(vk.Keyword, "+", "", -1)) {
|
|
|
- kwsFlag = false
|
|
|
- continue
|
|
|
- }
|
|
|
- }
|
|
|
- if kwsFlag {
|
|
|
- if NoKwsMap[emptyNum] == nil {
|
|
|
- NoKwsMap[emptyNum] = []map[string]interface{}{v}
|
|
|
- } else {
|
|
|
- NoKwsMap[emptyNum] = append(NoKwsMap[emptyNum], v)
|
|
|
- }
|
|
|
- continue
|
|
|
- }
|
|
|
- }
|
|
|
- if AllMap[emptyNum] == nil {
|
|
|
- AllMap[emptyNum] = []map[string]interface{}{v}
|
|
|
- continue
|
|
|
- }
|
|
|
- AllMap[emptyNum] = append(AllMap[emptyNum], v)
|
|
|
- }
|
|
|
- //获取key
|
|
|
- keys := []int{}
|
|
|
- for k, _ := range AllMap {
|
|
|
- keys = append(keys, k)
|
|
|
- }
|
|
|
- sort.Ints(keys)
|
|
|
- log.Println("空字段数量", keys)
|
|
|
- //选取结果
|
|
|
- for _, v := range keys {
|
|
|
- if len(AllMap[v]) >= resultNum {
|
|
|
- return append(res, AllMap[v][:resultNum]...)
|
|
|
- } else {
|
|
|
- resultNum = resultNum - len(AllMap[v])
|
|
|
- tmp := append(res, AllMap[v][:len(AllMap[v])]...)
|
|
|
- res = tmp
|
|
|
- }
|
|
|
- }
|
|
|
- if len(res) < lastNum {
|
|
|
- resultNum = lastNum - len(res)
|
|
|
- //获取key
|
|
|
- Nokeys := []int{}
|
|
|
- for k, _ := range NoKwsMap {
|
|
|
- Nokeys = append(Nokeys, k)
|
|
|
- }
|
|
|
- sort.Ints(Nokeys)
|
|
|
- log.Println("没关键词的空字段数量", Nokeys)
|
|
|
- //选取结果
|
|
|
- for _, v := range Nokeys {
|
|
|
- if len(NoKwsMap[v]) >= resultNum {
|
|
|
- return append(res, NoKwsMap[v][:resultNum]...)
|
|
|
- } else {
|
|
|
- resultNum = resultNum - len(NoKwsMap[v])
|
|
|
- tmp := append(res, NoKwsMap[v][:len(NoKwsMap[v])]...)
|
|
|
- res = tmp
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- return res
|
|
|
-}
|
|
|
-func countOfTheEmpty(m map[string]interface{}, dataType string) int {
|
|
|
- MsgType := m["subtype"]
|
|
|
- // if MsgType == "拟建" {
|
|
|
- // return -1
|
|
|
- // }
|
|
|
- //计算空字段数量
|
|
|
- var count int = 0
|
|
|
- //高级字段包
|
|
|
- if dataType == "2" {
|
|
|
- if m["href"] == "" || m["href"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["projectcode"] == "" || m["projectcode"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["buyerperson"] == "" || m["buyerperson"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["buyertel"] == "" || m["buyertel"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["budget"] == "" || m["budget"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["bidopentime"] == "" || m["bidopentime"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["agency"] == "" || m["agency"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["projectscope"] == "" || m["projectscope"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- }
|
|
|
- if m["city"] == "" || m["city"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["publishtime"] == "" || m["publishtime"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["projectname"] == "" || m["projectname"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["buyer"] == "" || m["buyer"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- if m["s_winner"] == "" || m["s_winner"] == nil {
|
|
|
- if MsgType != "招标" {
|
|
|
- count++
|
|
|
- }
|
|
|
- }
|
|
|
- if m["bidamount"] == "" || m["bidamount"] == nil {
|
|
|
- if MsgType != "招标" {
|
|
|
- count++
|
|
|
- }
|
|
|
- }
|
|
|
- if m["subtype"] == "" || m["subtype"] == nil {
|
|
|
- count++
|
|
|
- }
|
|
|
- return count
|
|
|
-}
|
|
|
-
|
|
|
-func GetEntDataExportCount(_id string, entId, entUserId, limitNum, current int, isFirst bool) (count, newCount int, data *[]map[string]interface{}) {
|
|
|
- defer util.Catch()
|
|
|
- var (
|
|
|
- searchsWaitGroup = &sync.WaitGroup{}
|
|
|
- searchsPool = make(chan bool, 20)
|
|
|
- // res = &[]map[string]interface{}{}
|
|
|
- newCountPool = make(chan bool, 20000)
|
|
|
- )
|
|
|
- count = GetDataExportSearchCountUseId(_id)
|
|
|
- log.Println("count", count)
|
|
|
- if count > ExConf.MsgMaxCount {
|
|
|
- count = ExConf.MsgMaxCount
|
|
|
- }
|
|
|
- dataType := "2"
|
|
|
- //数据导出数据查询
|
|
|
-
|
|
|
- res, err := GetDataExportSearchResult(_id, dataType, count)
|
|
|
- if err != nil {
|
|
|
- log.Println("企业数据导出错误 ", err)
|
|
|
- return 0, 0, nil
|
|
|
- }
|
|
|
- // secondCount := 0
|
|
|
-
|
|
|
- // isOK := true
|
|
|
- // if secondCount > current {
|
|
|
- // isOK = false
|
|
|
- // }
|
|
|
- for _, v := range *res {
|
|
|
- searchsWaitGroup.Add(1)
|
|
|
- searchsPool <- true
|
|
|
- id := util.ObjToString(v["_id"])
|
|
|
- go func(id string) {
|
|
|
- defer func() {
|
|
|
- searchsWaitGroup.Done()
|
|
|
- <-searchsPool
|
|
|
- }()
|
|
|
- isExist, err := redis.Exists("other", "entexportdata_"+id+"_"+fmt.Sprintln(entId))
|
|
|
- if err != nil {
|
|
|
- log.Println("企业搜索数据导出redis判重失败")
|
|
|
- } else if isExist {
|
|
|
- log.Println("数据重复，id ", id, "entid ", entId, "userid ", entUserId)
|
|
|
- return
|
|
|
- }
|
|
|
- newCountPool <- true
|
|
|
- if !isFirst {
|
|
|
- redis.Put("other", "entexportdata_"+id+"_"+fmt.Sprintln(entId), 1, -1)
|
|
|
- }
|
|
|
- }(id)
|
|
|
- if !isFirst {
|
|
|
- delete(v, "_id")
|
|
|
- v["entid"] = entId
|
|
|
- v["userid"] = entUserId
|
|
|
- v["infoid"] = id
|
|
|
- v["createtime"] = time.Now().Unix()
|
|
|
- }
|
|
|
- }
|
|
|
- searchsWaitGroup.Wait()
|
|
|
- log.Println("企业数据导出--数据遍历完成")
|
|
|
- newCount = len(newCountPool)
|
|
|
- log.Println("new", newCount)
|
|
|
- data = res
|
|
|
- return
|
|
|
-}
|
|
|
-
|
|
|
-func GetDataExportSearchResult(id string, dataType string, checkCount int) (*[]map[string]interface{}, error) {
|
|
|
- defer util.Catch()
|
|
|
- var (
|
|
|
- onceSearchCount = 500
|
|
|
- searchPool = make(chan bool, 20)
|
|
|
- res []map[string]interface{}
|
|
|
- )
|
|
|
- //获取查询语句
|
|
|
- scd := getSqlObjFromId(id)
|
|
|
- if scd == nil {
|
|
|
- return nil, errors.New("GetDataExportSearchResult-获取查询条件")
|
|
|
- }
|
|
|
- qstr := getDataExportSql(scd)
|
|
|
- log.Printf("GetDataExportSearchResult-%s-sql:%s\n", scd.Id, qstr)
|
|
|
- //数据导出数据查询
|
|
|
- if checkCount > onceSearchCount { //分批次查询
|
|
|
- batchNum := util.IntAll(math.Ceil(float64(checkCount) / float64(onceSearchCount)))
|
|
|
- var searchWaitGroup = &sync.WaitGroup{}
|
|
|
- var lock sync.Mutex
|
|
|
- for n := 0; n < batchNum; n++ {
|
|
|
- searchWaitGroup.Add(1)
|
|
|
- searchPool <- true
|
|
|
- go func(start int) {
|
|
|
- defer func() {
|
|
|
- searchWaitGroup.Done()
|
|
|
- <-searchPool
|
|
|
- }()
|
|
|
- checkNum, checkOk := onceSearchCount, false
|
|
|
- if start == (batchNum - 1) {
|
|
|
- if checkCount%onceSearchCount != 0 {
|
|
|
- checkNum = checkCount % onceSearchCount
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- var tmp *[]map[string]interface{}
|
|
|
- for i := 0; i < 3; i++ {
|
|
|
- tmp = doSearch(qstr, start*onceSearchCount, onceSearchCount, dataType)
|
|
|
- if tmp != nil && (len(*tmp) == checkNum) { //校验数据量是否够
|
|
|
- checkOk = true
|
|
|
- break
|
|
|
- }
|
|
|
- }
|
|
|
- if tmp == nil {
|
|
|
- log.Printf("GetDataExportSearchResult-%s-第%d页数据查询结果为空\n", scd.Id, start+1)
|
|
|
- return
|
|
|
- }
|
|
|
- if checkOk {
|
|
|
- log.Printf("GetDataExportSearchResult-%s-第%d页数据加载完成,共%d条\n", scd.Id, start+1, len(*tmp))
|
|
|
- } else {
|
|
|
- log.Printf("GetDataExportSearchResult-%s-第%d页数据加载异常,共%d条,预期%d条\n", scd.Id, start+1, len(*tmp), checkNum)
|
|
|
- }
|
|
|
- lock.Lock()
|
|
|
- res = append(res, *tmp...)
|
|
|
- lock.Unlock()
|
|
|
- }(n)
|
|
|
- }
|
|
|
- searchWaitGroup.Wait()
|
|
|
- log.Printf("GetDataExportSearchResult-%s-分批次加载数据总量为%d\n", scd.Id, len(res))
|
|
|
- } else {
|
|
|
- tmp := doSearch(qstr, 0, checkCount, dataType)
|
|
|
- if tmp == nil || len(*tmp) == 0 {
|
|
|
- log.Printf("GetDataExportSearchResult-%s-一次性加载数据异常\n", scd.Id)
|
|
|
- } else {
|
|
|
- res = *tmp
|
|
|
- log.Printf("GetDataExportSearchResult-%s-一次性加载数据总量为%d\n", scd.Id, len(res))
|
|
|
- }
|
|
|
- }
|
|
|
- //超级搜索一致的检索（防止数据导出和超级搜索数据量不一致）
|
|
|
- if scd.Comeinfrom == "supersearchPage" && (len(scd.Keyword) != 0 || len(scd.Industry) != 0) {
|
|
|
- if len(scd.Keyword) != 0 {
|
|
|
- num := len(res)
|
|
|
- searchTextSize := 0
|
|
|
- if len(scd.Keyword) > 0 {
|
|
|
- searchTextSize = len([]rune(scd.Keyword[0].Keyword))
|
|
|
- }
|
|
|
- if searchTextSize > 3 && num < 50 {
|
|
|
- secondKWS := jy.HttpEs(scd.Keyword[0].Keyword, "ik_smart", DbConf.Elasticsearch.Main.Address)
|
|
|
- scd.Keyword[0].Keyword = secondKWS
|
|
|
- scd.SelectType = "title"
|
|
|
- qstr = getDataExportSql(scd)
|
|
|
- log.Printf("GetDataExportSearchResult-%s-分词查询-sql:%s\n", scd.Id, qstr)
|
|
|
- res2 := doSearch(qstr, 0, 100, "")
|
|
|
- res = *delRepeatMapArr(&res, res2)
|
|
|
- if len(res) > 100 {
|
|
|
- res = res[:100]
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- //校验数量
|
|
|
- if checkCount != len(res) {
|
|
|
- return nil, fmt.Errorf("GetDataExportSearchResult-%s-数据总量校验异常，期望:%d,实际:%d", scd.Id, checkCount, len(res))
|
|
|
- //发邮件
|
|
|
- }
|
|
|
- return &res, nil
|
|
|
-}
|
|
|
+//
|
|
|
+//import (
|
|
|
+// "encoding/json"
|
|
|
+// "errors"
|
|
|
+// "fmt"
|
|
|
+// "log"
|
|
|
+// "math"
|
|
|
+// "qfw/util"
|
|
|
+// "qfw/util/elastic"
|
|
|
+// "qfw/util/jy"
|
|
|
+// "qfw/util/redis"
|
|
|
+// "sort"
|
|
|
+// "strconv"
|
|
|
+// "strings"
|
|
|
+// "sync"
|
|
|
+// "time"
|
|
|
+//
|
|
|
+// "go.mongodb.org/mongo-driver/bson"
|
|
|
+//)
|
|
|
+//
|
|
|
+///*筛选条件--关键词*/
|
|
|
+//type KeyWord struct {
|
|
|
+// Keyword string `json:"keyword"` //关键词
|
|
|
+// Appended []string `json:"appended"` //附加词
|
|
|
+// Exclude []string `json:"exclude"` //排除词
|
|
|
+//}
|
|
|
+//
|
|
|
+///*筛选条件*/
|
|
|
+//type SieveCondition struct {
|
|
|
+// Id string `json:"id"`
|
|
|
+// PublishTime string `json:"publishtime"` //发布时间
|
|
|
+// Area []string `json:"area"` //地区-省份
|
|
|
+// City []string `json:"city"` //地区-城市
|
|
|
+// Region []string `json:"region"` //地区-省份+城市
|
|
|
+// Industry []string `json:"industry"` //行业
|
|
|
+// Keyword []KeyWord `json:"keywords"` //关键词
|
|
|
+// Buyer []string `json:"buyer"` //招标单位(采购单位)
|
|
|
+// Buyerclass []string `json:"buyerclass"` //采购单位类型
|
|
|
+// Winner []string `json:"winner"` //中标单位
|
|
|
+// ComeInTime int64 `json:"comeintime"` //入库时间(秒)
|
|
|
+// OpenId string `json:"openid"` //用户openid
|
|
|
+// MinPrice string `json:"minprice"` //金额——最少
|
|
|
+// MaxPrice string `json:"maxprice"` //金额——最多
|
|
|
+// SelectType string `json:"selectType"` //筛选（正文 or 标题）
|
|
|
+// Subtype string `json:"subtype"` //信息类型
|
|
|
+// Comeinfrom string `json:"comeinfrom"` //查询来源
|
|
|
+//}
|
|
|
+//
|
|
|
+//const (
|
|
|
+// INDEX = "bidding"
|
|
|
+// TYPE = "bidding"
|
|
|
+// bidSearch_sort = `{"publishtime":-1}`
|
|
|
+//)
|
|
|
+//
|
|
|
+//var ExportTable string = "export_search"
|
|
|
+//
|
|
|
+////获取数据导出查询语句
|
|
|
+//func getDataExportSql(scd *SieveCondition) string {
|
|
|
+// multi_match := `{"multi_match": {"query": %s,"type": "phrase", "fields": [%s]}}`
|
|
|
+// query := `{"query":{"bool":{"must":[%s],"should":[%s],"minimum_should_match": %d}}}`
|
|
|
+// query_bool_should := `{"bool":{"should":[%s],"minimum_should_match": 1}}`
|
|
|
+// query_price := `{"bool":{"must":[{"range":{"bidamount":{%s}}}]}},{"bool":{"must":[{"range":{"budget":{%s}}}],"must_not":[{"range":{"bidamount":{"gte":-1}}}]}}`
|
|
|
+// query_bool_must := `{"terms":{"%s":[%s]}}`
|
|
|
+// query_bool_must_and := `{"bool":{"must":[%s]%s}}`
|
|
|
+// gte := `"gte": %s`
|
|
|
+// lte := `"lte": %s`
|
|
|
+//
|
|
|
+// bools := []string{}
|
|
|
+// musts := []string{fmt.Sprintf(`{"range":{"comeintime":{"lt":%d}}}`, scd.ComeInTime)}
|
|
|
+// //省份
|
|
|
+// areaCity := []string{}
|
|
|
+// if len(scd.Area) > 0 {
|
|
|
+// areaquery := `{"terms":{"area":[`
|
|
|
+// for k, v := range scd.Area {
|
|
|
+// if k > 0 {
|
|
|
+// areaquery += `,`
|
|
|
+// }
|
|
|
+// areaquery += `"` + v + `"`
|
|
|
+// }
|
|
|
+// areaquery += `]}}`
|
|
|
+// areaCity = append(areaCity, areaquery)
|
|
|
+// }
|
|
|
+// //城市
|
|
|
+// if len(scd.City) > 0 {
|
|
|
+// areaquery := `{"terms":{"city":[`
|
|
|
+// for k, v := range scd.City {
|
|
|
+// if k > 0 {
|
|
|
+// areaquery += `,`
|
|
|
+// }
|
|
|
+// areaquery += `"` + v + `"`
|
|
|
+// }
|
|
|
+// areaquery += `]}}`
|
|
|
+// areaCity = append(areaCity, areaquery)
|
|
|
+// }
|
|
|
+// if len(areaCity) > 0 {
|
|
|
+// musts = append(musts, fmt.Sprintf(query_bool_should, strings.Join(areaCity, ",")))
|
|
|
+// }
|
|
|
+// //检索日期
|
|
|
+//
|
|
|
+// starttime := ""
|
|
|
+// now := time.Unix(scd.ComeInTime, 0)
|
|
|
+// endtime := fmt.Sprintf("%d", now.Unix())
|
|
|
+// if scd.PublishTime == "lately-7" { //最近7天
|
|
|
+// starttime = fmt.Sprint(time.Date(now.Year(), now.Month(), now.Day()-7, 0, 0, 0, 0, time.Local).Unix())
|
|
|
+// } else if scd.PublishTime == "lately-30" { //最近30天
|
|
|
+// starttime = fmt.Sprint(time.Date(now.Year(), now.Month(), now.Day()-30, 0, 0, 0, 0, time.Local).Unix())
|
|
|
+// } else if scd.PublishTime == "thisyear" { //去年
|
|
|
+// starttime = fmt.Sprint(time.Date(now.Year()-1, 1, 1, 0, 0, 0, 0, time.Local).Unix())
|
|
|
+// endtime = fmt.Sprint(time.Date(now.Year()-1, 12, 31, 23, 59, 59, 0, time.Local).Unix())
|
|
|
+// } else if strings.Contains(scd.PublishTime, "_") { //设置检索日期
|
|
|
+// starttime = strings.Split(scd.PublishTime, "_")[0]
|
|
|
+// endTime_tmp := now
|
|
|
+// if etime := strings.Split(scd.PublishTime, "_")[1]; etime != "" {
|
|
|
+// etTime := time.Unix(util.Int64All(etime), 0)
|
|
|
+// endTime_tmp = time.Date(etTime.Year(), etTime.Month(), etTime.Day()+1, 0, 0, 0, 0, time.Local)
|
|
|
+// }
|
|
|
+// //结束时间必须小于筛选时间
|
|
|
+// if endTime_tmp.After(now) {
|
|
|
+// endTime_tmp = now
|
|
|
+// }
|
|
|
+// endtime = fmt.Sprintf("%d", endTime_tmp.Unix())
|
|
|
+// }
|
|
|
+// timequery := `{"range":{"publishtime":{`
|
|
|
+// if starttime != "" {
|
|
|
+// timequery += `"gte":` + starttime
|
|
|
+// }
|
|
|
+// if starttime != "" && endtime != "" {
|
|
|
+// timequery += `,`
|
|
|
+// }
|
|
|
+// if endtime != "" {
|
|
|
+// timequery += `"lt":` + endtime
|
|
|
+// }
|
|
|
+// timequery += `}}}`
|
|
|
+// musts = append(musts, timequery)
|
|
|
+//
|
|
|
+// if scd.Subtype != "" {
|
|
|
+// subquery := `{"terms":{"subtype":[`
|
|
|
+// for k, v := range strings.Split(scd.Subtype, ",") {
|
|
|
+// if k > 0 {
|
|
|
+// subquery += `,`
|
|
|
+// }
|
|
|
+// subquery += `"` + v + `"`
|
|
|
+// }
|
|
|
+// subquery += `]}}`
|
|
|
+// musts = append(musts, subquery)
|
|
|
+// }
|
|
|
+// if len(scd.Industry) > 0 {
|
|
|
+// musts = append(musts, fmt.Sprintf(query_bool_must, "s_subscopeclass", `"`+strings.Join(scd.Industry, `","`)+`"`))
|
|
|
+// }
|
|
|
+// if len(scd.Buyer) > 0 {
|
|
|
+// musts = append(musts, fmt.Sprintf(query_bool_must, "buyer", `"`+strings.Join(scd.Buyer, `","`)+`"`))
|
|
|
+// }
|
|
|
+// if len(scd.Buyerclass) > 0 {
|
|
|
+// musts = append(musts, fmt.Sprintf(query_bool_must, "buyerclass", `"`+strings.Join(scd.Buyerclass, `","`)+`"`))
|
|
|
+// }
|
|
|
+// if len(scd.Winner) > 0 {
|
|
|
+// musts = append(musts, fmt.Sprintf(query_bool_must, "s_winner", `"`+strings.Join(scd.Winner, `","`)+`"`))
|
|
|
+// }
|
|
|
+// _minPrice := ""
|
|
|
+// _maxPrice := ""
|
|
|
+// if scd.MinPrice != "" || scd.MaxPrice != "" {
|
|
|
+// sq := ``
|
|
|
+// if scd.MinPrice != "" {
|
|
|
+// min, _ := strconv.ParseFloat(scd.MinPrice, 64)
|
|
|
+// _minPrice = fmt.Sprintf("%.0f", min*10000)
|
|
|
+// if _minPrice == "0" {
|
|
|
+// _minPrice = ""
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if scd.MaxPrice != "" {
|
|
|
+// max, _ := strconv.ParseFloat(scd.MaxPrice, 64)
|
|
|
+// _maxPrice = fmt.Sprintf("%.0f", max*10000)
|
|
|
+// if _maxPrice == "0" {
|
|
|
+// _maxPrice = ""
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if _minPrice != "" {
|
|
|
+// sq += fmt.Sprintf(gte, _minPrice)
|
|
|
+// }
|
|
|
+// if _minPrice != "" && _maxPrice != "" {
|
|
|
+// sq += `,`
|
|
|
+// }
|
|
|
+// if _maxPrice != "" {
|
|
|
+// sq += fmt.Sprintf(lte, _maxPrice)
|
|
|
+// }
|
|
|
+// if _minPrice != "" || _maxPrice != "" {
|
|
|
+// query_price := fmt.Sprintf(query_bool_should, fmt.Sprintf(query_price, sq, sq))
|
|
|
+// musts = append(musts, query_price)
|
|
|
+// }
|
|
|
+// }
|
|
|
+// boolsNum := 0
|
|
|
+// //should
|
|
|
+// if len(scd.Keyword) > 0 {
|
|
|
+// boolsNum = 1
|
|
|
+// if scd.SelectType == "" || scd.SelectType == "all" {
|
|
|
+// scd.SelectType = "detail\", \"title"
|
|
|
+// }
|
|
|
+// multi_match = fmt.Sprintf(multi_match, "%s", "\""+scd.SelectType+"\"")
|
|
|
+//
|
|
|
+// if scd.Comeinfrom == "supersearchPage" {
|
|
|
+// var keywordArr []string
|
|
|
+// if strings.Contains(scd.Keyword[0].Keyword, "+") {
|
|
|
+// keywordArr = strings.Split(scd.Keyword[0].Keyword, "+")
|
|
|
+// } else if strings.Contains(scd.Keyword[0].Keyword, " ") {
|
|
|
+// keywordArr = strings.Split(scd.Keyword[0].Keyword, " ")
|
|
|
+// }
|
|
|
+// if len(keywordArr) > 1 {
|
|
|
+// KeyWordSearch := KeyWord{}
|
|
|
+// for _, v := range keywordArr {
|
|
|
+// KeyWordSearch.Appended = append(KeyWordSearch.Appended, v)
|
|
|
+// }
|
|
|
+// scd.Keyword = []KeyWord{KeyWordSearch}
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// for _, v := range scd.Keyword {
|
|
|
+// shoulds := []string{}
|
|
|
+// must_not := []string{}
|
|
|
+// //附加词
|
|
|
+// if v.Keyword != "" {
|
|
|
+// shoulds = append(shoulds, fmt.Sprintf(multi_match, "\""+v.Keyword+"\""))
|
|
|
+// }
|
|
|
+//
|
|
|
+// for _, vv := range v.Appended {
|
|
|
+// shoulds = append(shoulds, fmt.Sprintf(multi_match, "\""+vv+"\""))
|
|
|
+// }
|
|
|
+//
|
|
|
+// //排除词
|
|
|
+// for _, vv := range v.Exclude {
|
|
|
+// must_not = append(must_not, fmt.Sprintf(multi_match, "\""+vv+"\""))
|
|
|
+// }
|
|
|
+//
|
|
|
+// //添加
|
|
|
+// if len(shoulds) > 0 {
|
|
|
+// notStr := ""
|
|
|
+// if len(must_not) > 0 {
|
|
|
+// notStr = fmt.Sprintf(`,"must_not":[%s]`, strings.Join(must_not, ","))
|
|
|
+// }
|
|
|
+// bools = append(bools, fmt.Sprintf(query_bool_must_and, strings.Join(shoulds, ","), notStr))
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// qstr := fmt.Sprintf(query, strings.Join(musts, ","), strings.Join(bools, ","), boolsNum)
|
|
|
+// return qstr
|
|
|
+//}
|
|
|
+//
|
|
|
+//func getSqlObjFromId(_id string) *SieveCondition {
|
|
|
+// var (
|
|
|
+// query *map[string]interface{}
|
|
|
+// ok bool
|
|
|
+// )
|
|
|
+// if query, ok = MQFW.FindById(ExportTable, _id, nil); !ok {
|
|
|
+// return nil
|
|
|
+// }
|
|
|
+// return &SieveCondition{
|
|
|
+// Id: _id,
|
|
|
+// Keyword: getKeyWordArrFromDbResult((*query)["keywords"]),
|
|
|
+// Industry: getStringArrFromDbResult((*query)["industry"]),
|
|
|
+// MinPrice: util.ObjToString((*query)["minprice"]),
|
|
|
+// MaxPrice: util.ObjToString((*query)["maxprice"]),
|
|
|
+// Subtype: util.ObjToString((*query)["subtype"]),
|
|
|
+// Area: getStringArrFromDbResult((*query)["area"]),
|
|
|
+// City: getStringArrFromDbResult((*query)["city"]),
|
|
|
+// SelectType: util.ObjToString((*query)["selectType"]),
|
|
|
+// PublishTime: util.ObjToString((*query)["publishtime"]),
|
|
|
+// Buyer: getStringArrFromDbResult((*query)["buyer"]),
|
|
|
+// Buyerclass: getStringArrFromDbResult((*query)["buyerclass"]),
|
|
|
+// Winner: getStringArrFromDbResult((*query)["winner"]),
|
|
|
+// ComeInTime: util.Int64All((*query)["comeintime"]),
|
|
|
+// Comeinfrom: util.ObjToString((*query)["comeinfrom"]),
|
|
|
+// }
|
|
|
+//}
|
|
|
+//
|
|
|
+////数据导出-查询结果数量
|
|
|
+//func GetDataExportSearchCountUseId(_id string) (count int) {
|
|
|
+// scd := getSqlObjFromId(_id)
|
|
|
+// qstr := getDataExportSql(scd)
|
|
|
+// log.Printf("GetDataExportSearchCountUseId-%s-count:%d-sql:%s\n", _id, count, qstr)
|
|
|
+// if isNullSearch(scd) {
|
|
|
+// return ExConf.MsgMaxCount
|
|
|
+// }
|
|
|
+// count = int(elastic.Count(INDEX, TYPE, qstr))
|
|
|
+// //超级搜索一致的检索（防止数据导出和超级搜索数据量不一致）
|
|
|
+// if scd.Comeinfrom == "supersearchPage" && (len(scd.Keyword) != 0 || len(scd.Industry) != 0) {
|
|
|
+// if len(scd.Keyword) != 0 {
|
|
|
+// searchTextSize := 0
|
|
|
+// if len(scd.Keyword) > 0 {
|
|
|
+// searchTextSize = len([]rune(scd.Keyword[0].Keyword))
|
|
|
+// }
|
|
|
+// if searchTextSize > 3 && count < 50 {
|
|
|
+// var res *[]map[string]interface{}
|
|
|
+// if count > 0 {
|
|
|
+// res = doSearch(qstr, 0, count, "")
|
|
|
+// }
|
|
|
+// secondKWS := jy.HttpEs(scd.Keyword[0].Keyword, "ik_smart", DbConf.Elasticsearch.Main.Address)
|
|
|
+// scd.Keyword[0].Keyword = secondKWS
|
|
|
+// scd.SelectType = "title"
|
|
|
+// qstr = getDataExportSql(scd)
|
|
|
+// res2 := doSearch(qstr, 0, 100, "")
|
|
|
+// result := len(*delRepeatMapArr(res, res2))
|
|
|
+// if result > 100 {
|
|
|
+// result = 100
|
|
|
+// }
|
|
|
+// log.Printf("GetDataExportSearchCountUseId-%s-count:%d-分词-sql:%s\n", _id, result, qstr)
|
|
|
+// return result
|
|
|
+// }
|
|
|
+// return
|
|
|
+// }
|
|
|
+// }
|
|
|
+// log.Printf("GetDataExportSearchCountUseId-%s-count:%d\n", _id, count)
|
|
|
+// return
|
|
|
+//}
|
|
|
+//
|
|
|
+////合并map数据，去重
|
|
|
+//func delRepeatMapArr(res *[]map[string]interface{}, res2 *[]map[string]interface{}) *[]map[string]interface{} {
|
|
|
+// if res != nil {
|
|
|
+// for _, v := range *res {
|
|
|
+// for n, m := range *res2 {
|
|
|
+// if util.ObjToString(v["_id"]) == util.ObjToString(m["_id"]) {
|
|
|
+// *res2 = append((*res2)[0:n], (*res2)[n+1:]...)
|
|
|
+// break
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// *res = append(*res, *res2...)
|
|
|
+// } else {
|
|
|
+// res = res2
|
|
|
+// }
|
|
|
+// return res
|
|
|
+//}
|
|
|
+//
|
|
|
+////查询条件是否为空
|
|
|
+//func isNullSearch(scd *SieveCondition) (isNull bool) {
|
|
|
+// if scd.PublishTime == "" && len(scd.Area) == 0 && len(scd.Industry) == 0 && len(scd.Keyword) == 0 && len(scd.Buyer) == 0 && len(scd.Winner) == 0 && scd.MinPrice == "" && scd.MaxPrice == "" && scd.Subtype == "" && len(scd.City) == 0 {
|
|
|
+// isNull = true
|
|
|
+// }
|
|
|
+// return isNull
|
|
|
+//}
|
|
|
+//
|
|
|
+///*
|
|
|
+// * 数据导出 查询结果
|
|
|
+// * _id 数据库查询条件记录id
|
|
|
+// * dataType 1-普通字段 2-高级字段
|
|
|
+// * webdomain 三级页域名
|
|
|
+// * count 返回数量 (-1:预览数据查询)
|
|
|
+// */
|
|
|
+//var EntTable = "winner_enterprise"
|
|
|
+//
|
|
|
+//func GetDataExportSearchResultUseId(_id, dataType string, count int) (*[]map[string]interface{}, []KeyWord) {
|
|
|
+// defer util.Catch()
|
|
|
+// var res []map[string]interface{}
|
|
|
+// var kws []KeyWord
|
|
|
+// scd := getSqlObjFromId(_id)
|
|
|
+// //获取查询语句
|
|
|
+// qstr := getDataExportSql(scd)
|
|
|
+// log.Printf("GetDataExportSearchResultUseId-%s-sql:%s\n", scd.Id, qstr)
|
|
|
+// kws = scd.Keyword
|
|
|
+// //数据预览数据查询
|
|
|
+// if scd.Comeinfrom == "supersearchPage" && len(scd.Keyword) == 0 && len(scd.Industry) == 0 {
|
|
|
+// //空查询
|
|
|
+// obj := redis.Get("other", "export_news")
|
|
|
+// if obj != nil {
|
|
|
+// res = util.ObjArrToMapArr(obj.([]interface{}))
|
|
|
+// } else {
|
|
|
+// res = *doSearch(qstr, 0, 500, "2")
|
|
|
+// if len(res) > 0 {
|
|
|
+// redis.Put("other", "export_news", res, 7200)
|
|
|
+// }
|
|
|
+// }
|
|
|
+// return &res, kws
|
|
|
+// } else if scd.Comeinfrom == "supersearchPage" {
|
|
|
+// //超级搜索非空查询
|
|
|
+// count = int(elastic.Count(INDEX, TYPE, qstr))
|
|
|
+// searchTextSize := 0
|
|
|
+// if len(scd.Keyword) > 0 {
|
|
|
+// searchTextSize = len([]rune(scd.Keyword[0].Keyword))
|
|
|
+// }
|
|
|
+// if searchTextSize > 3 && count < 50 {
|
|
|
+// var res *[]map[string]interface{}
|
|
|
+// if count > 0 {
|
|
|
+// res = doSearch(qstr, 0, count, "")
|
|
|
+// }
|
|
|
+// secondKWS := jy.HttpEs(scd.Keyword[0].Keyword, "ik_smart", DbConf.Elasticsearch.Main.Address)
|
|
|
+// scd.Keyword[0].Keyword = secondKWS
|
|
|
+// scd.SelectType = "title"
|
|
|
+// qstr = getDataExportSql(scd)
|
|
|
+// log.Printf("GetDataExportSearchResultUseId-%s-分词-sql:%s\n", scd.Id, qstr)
|
|
|
+// res2 := doSearch(qstr, 0, 100, "")
|
|
|
+// if len(*res2) > 100 {
|
|
|
+// res2_temp := (*res2)[:100]
|
|
|
+// res2 = &res2_temp
|
|
|
+// }
|
|
|
+// return delRepeatMapArr(res, res2), kws
|
|
|
+// }
|
|
|
+// }
|
|
|
+// //非空查询
|
|
|
+// res = *doSearch(qstr, 0, 500, dataType)
|
|
|
+// return &res, kws
|
|
|
+//}
|
|
|
+//
|
|
|
+//func FormatExportData(data *[]map[string]interface{}, webdomain string, dataType string, EntArr []string) *[]map[string]interface{} {
|
|
|
+// //格式化输出
|
|
|
+// for _, v := range *data {
|
|
|
+// //有中标企业 且 高级字段查询
|
|
|
+// if len(EntArr) > 0 && dataType == "2" {
|
|
|
+// //查询企业公示 法人 公司电话 公司邮箱地址
|
|
|
+// query := bson.M{"company_name": bson.M{"$in": EntArr}} //
|
|
|
+// if entData, ok := Mgo_Ent.Find(EntTable, query, nil, `{"company_name":1,"company_email":1,"legal_person":1,"company_phone":1}`, false, -1, -1); ok {
|
|
|
+// if entData != nil && *entData != nil && len(*entData) > 0 {
|
|
|
+// for _, ev := range *entData {
|
|
|
+// if v["s_winner"] == ev["company_name"] {
|
|
|
+// legal_person := ""
|
|
|
+// if ev["legal_person"] != nil && ev["legal_person"].(string) != "" {
|
|
|
+// legal_person = ev["legal_person"].(string)
|
|
|
+// var xx = "*"
|
|
|
+// switch len([]rune(legal_person)) {
|
|
|
+// case 3:
|
|
|
+// xx = "**"
|
|
|
+// case 4:
|
|
|
+// xx = "***"
|
|
|
+// }
|
|
|
+// legal_person = string([]rune(legal_person)[:1]) + xx
|
|
|
+// }
|
|
|
+// company_phone := ""
|
|
|
+// if ev["company_phone"] != nil && ev["company_phone"].(string) != "" {
|
|
|
+// company_phone = ev["company_phone"].(string)
|
|
|
+// if len([]rune(company_phone)) > 7 {
|
|
|
+// company_phone = company_phone[:7] + "****"
|
|
|
+// } else {
|
|
|
+// company_phone = "****"
|
|
|
+// }
|
|
|
+// }
|
|
|
+// company_email := ""
|
|
|
+// if ev["company_email"] != nil && ev["company_email"].(string) != "无" {
|
|
|
+// company_email = ev["company_email"].(string)
|
|
|
+// if len(strings.Split(company_email, "@")) > 1 {
|
|
|
+// company_email = "******" + "@" + strings.Split(company_email, "@")[1]
|
|
|
+// }
|
|
|
+// }
|
|
|
+// v["legal_person"] = legal_person
|
|
|
+// v["company_phone"] = company_phone
|
|
|
+// v["company_email"] = company_email
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// //====================字段补漏=========================
|
|
|
+// if v["toptype"] == "结果" && dataType == "2" && !(v["agency"] != nil && v["budget"] != nil && v["buyerperson"] != nil && v["buyertel"] != nil) {
|
|
|
+// r := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query":{"term":{"list.infoid":"%s"}},"_source": ["list"]}`, v["_id"]))
|
|
|
+// if len(*r) > 0 {
|
|
|
+// MsgList := (*r)[0]["list"]
|
|
|
+// if MsgList != nil {
|
|
|
+// list := util.ObjArrToMapArr(MsgList.([]interface{}))
|
|
|
+// for _, vv := range list {
|
|
|
+// if vv["subtype"] == "招标" {
|
|
|
+// if v["agency"] == nil && vv["agency"] != nil {
|
|
|
+// v["agency"] = vv["agency"]
|
|
|
+// }
|
|
|
+// if v["budget"] == nil && vv["budget"] != nil {
|
|
|
+// v["budget"] = vv["budget"]
|
|
|
+// }
|
|
|
+// if v["buyerperson"] == nil && vv["buyerperson"] != nil {
|
|
|
+// v["buyerperson"] = vv["buyerperson"]
|
|
|
+// }
|
|
|
+// if v["buyertel"] == nil && vv["buyertel"] != nil {
|
|
|
+// v["buyertel"] = vv["buyertel"]
|
|
|
+// }
|
|
|
+// break
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if v["area"] == "A" {
|
|
|
+// v["area"] = "全国"
|
|
|
+// }
|
|
|
+// if v["bidamount"] != nil {
|
|
|
+// v["bidamount"] = formatFloat(util.Float64All(v["bidamount"]))
|
|
|
+// }
|
|
|
+// if v["budget"] != nil {
|
|
|
+// v["budget"] = formatFloat(util.Float64All(v["budget"]))
|
|
|
+// }
|
|
|
+// if v["publishtime"] != nil {
|
|
|
+// date := v["publishtime"]
|
|
|
+// v["publishtime"] = util.FormatDateWithObj(&date, util.Date_Short_Layout)
|
|
|
+// }
|
|
|
+// if v["bidopentime"] != nil {
|
|
|
+// date := v["bidopentime"]
|
|
|
+// v["bidopentime"] = util.FormatDateWithObj(&date, util.Date_Short_Layout)
|
|
|
+// }
|
|
|
+// if v["_id"] != nil {
|
|
|
+// v["url"] = webdomain + "/article/content/" + util.CommonEncodeArticle("content", v["_id"].(string)) + ".html"
|
|
|
+// }
|
|
|
+// if v["currency"] == "" || v["currency"] == nil {
|
|
|
+// v["currency"] = "人民币"
|
|
|
+// }
|
|
|
+// if v["projectscope"] != "" && v["projectscope"] != nil {
|
|
|
+// str := ClearHtml.ReplaceAllString(v["projectscope"].(string), "")
|
|
|
+// str = ClearOther.ReplaceAllString(str, "")
|
|
|
+// str = strings.Replace(str, " ", "", -1)
|
|
|
+// if len([]rune(str)) > 100 {
|
|
|
+// str = util.SubString(str, 0, 100) + "..."
|
|
|
+// }
|
|
|
+// v["projectscope"] = str
|
|
|
+// }
|
|
|
+// if v["detail"] != "" && v["detail"] != nil {
|
|
|
+// str := ClearHtml.ReplaceAllString(v["detail"].(string), "")
|
|
|
+// str = ClearOther.ReplaceAllString(str, "")
|
|
|
+// str = strings.Replace(str, " ", "", -1)
|
|
|
+// if len([]rune(str)) > 100 {
|
|
|
+// str = util.SubString(str, 0, 100) + "..."
|
|
|
+// }
|
|
|
+// v["detail"] = str
|
|
|
+// }
|
|
|
+// if v["title"] != "" && v["title"] != nil {
|
|
|
+// str := ClearHtml.ReplaceAllString(v["title"].(string), "")
|
|
|
+// str = ClearOther.ReplaceAllString(str, "")
|
|
|
+// str = strings.Replace(str, " ", "", -1)
|
|
|
+// if len([]rune(str)) > 100 {
|
|
|
+// str = util.SubString(str, 0, 100) + "..."
|
|
|
+// }
|
|
|
+// v["title"] = str
|
|
|
+// }
|
|
|
+// if v["subtype"] == nil && v["toptype"] != nil {
|
|
|
+// v["subtype"] = v["toptype"]
|
|
|
+// }
|
|
|
+// }
|
|
|
+// return data
|
|
|
+//}
|
|
|
+//
|
|
|
+////保留到0.01分
|
|
|
+//func formatFloat(value float64) string {
|
|
|
+// str := strings.TrimRight(fmt.Sprintf("%.7f", value*10000/100000000), "0")
|
|
|
+// if str[len(str)-1:] == "." {
|
|
|
+// return str[:len(str)-1]
|
|
|
+// }
|
|
|
+// return str
|
|
|
+//}
|
|
|
+//
|
|
|
+//func FormatExportDatas(data *[]map[string]interface{}, webdomain string, dataType string, entId int) *[]map[string]interface{} {
|
|
|
+// //格式化输出
|
|
|
+// var (
|
|
|
+// entexportPool = make(chan bool, 20)
|
|
|
+// entexportWaitGroup = &sync.WaitGroup{}
|
|
|
+// // newsDatalen = make(chan bool, 20000)
|
|
|
+// // datalock = sync.RWMutex{}
|
|
|
+// )
|
|
|
+// log.Println("补充信息开始")
|
|
|
+// for _, v := range *data {
|
|
|
+// entexportWaitGroup.Add(1)
|
|
|
+// entexportPool <- true
|
|
|
+// go func(v map[string]interface{}) {
|
|
|
+// defer func() {
|
|
|
+// entexportWaitGroup.Done()
|
|
|
+// <-entexportPool
|
|
|
+// // datalock.Unlock()
|
|
|
+// }()
|
|
|
+// // datalock.Lock()
|
|
|
+// //二次验证
|
|
|
+// // id := util.ObjToString(v["_id"])
|
|
|
+// // isExist, err := redis.Exists("other", "entexportdata_"+id+"_"+fmt.Sprintln(entId))
|
|
|
+// // if err != nil {
|
|
|
+// // log.Println("企业搜索数据导出redis判重失败")
|
|
|
+// // } else if !isExist {
|
|
|
+// // newsDatalen <- true
|
|
|
+// // }
|
|
|
+// //
|
|
|
+// //有中标企业 且 高级字段查询
|
|
|
+// if dataType == "2" {
|
|
|
+// //查询企业公示 法人 公司电话 公司邮箱地址
|
|
|
+// // query := bson.M{"company_name": bson.M{"$in": EntArr}} //
|
|
|
+// s_winner := strings.Split(util.ObjToString(v["s_winner"]), ",")[0]
|
|
|
+// query := bson.M{"company_name": s_winner} //
|
|
|
+// if entData, ok := Mgo_Ent.Find(EntTable, query, nil, `{"company_name":1,"company_email":1,"legal_person":1,"company_phone":1}`, false, -1, -1); ok {
|
|
|
+// if entData != nil && *entData != nil && len(*entData) > 0 {
|
|
|
+// for _, ev := range *entData {
|
|
|
+// if v["s_winner"] == ev["company_name"] {
|
|
|
+// legal_person := ""
|
|
|
+// if ev["legal_person"] != nil {
|
|
|
+// legal_person = ev["legal_person"].(string)
|
|
|
+// }
|
|
|
+// company_phone := ""
|
|
|
+// if ev["company_phone"] != nil {
|
|
|
+// company_phone = ev["company_phone"].(string)
|
|
|
+// }
|
|
|
+// company_email := ""
|
|
|
+// if ev["company_email"] != nil && ev["company_email"] != "无" {
|
|
|
+// company_email = ev["company_email"].(string)
|
|
|
+// }
|
|
|
+// v["legal_person"] = legal_person
|
|
|
+// v["company_phone"] = company_phone
|
|
|
+// v["company_email"] = company_email
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// //====================字段补漏=========================
|
|
|
+// if v["toptype"] == "结果" && dataType == "2" && !(v["agency"] != nil && v["budget"] != nil && v["buyerperson"] != nil && v["buyertel"] != nil) {
|
|
|
+// r := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query":{"term":{"list.infoid":"%s"}},"_source": ["list"]}`, v["_id"]))
|
|
|
+// if len(*r) > 0 {
|
|
|
+// MsgList := (*r)[0]["list"]
|
|
|
+// if MsgList != nil {
|
|
|
+// list := util.ObjArrToMapArr(MsgList.([]interface{}))
|
|
|
+// for _, vv := range list {
|
|
|
+// if vv["subtype"] == "招标" {
|
|
|
+// if v["agency"] == nil && vv["agency"] != nil {
|
|
|
+// v["agency"] = vv["agency"]
|
|
|
+// }
|
|
|
+// if v["budget"] == nil && vv["budget"] != nil {
|
|
|
+// v["budget"] = vv["budget"]
|
|
|
+// }
|
|
|
+// if v["buyerperson"] == nil && vv["buyerperson"] != nil {
|
|
|
+// v["buyerperson"] = vv["buyerperson"]
|
|
|
+// }
|
|
|
+// if v["buyertel"] == nil && vv["buyertel"] != nil {
|
|
|
+// v["buyertel"] = vv["buyertel"]
|
|
|
+// }
|
|
|
+// break
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if v["area"] == "A" {
|
|
|
+// v["area"] = "全国"
|
|
|
+// }
|
|
|
+// // if v["bidamount"] != nil {
|
|
|
+// // v["bidamount"] = fmt.Sprintf("%g", util.Float64All(v["bidamount"]))
|
|
|
+// // }
|
|
|
+// // if v["budget"] != nil {
|
|
|
+// // v["budget"] = fmt.Sprintf("%g", util.Float64All(v["budget"]))
|
|
|
+// // }
|
|
|
+// if v["publishtime"] != nil {
|
|
|
+// date := v["publishtime"]
|
|
|
+// v["publishtime"] = util.FormatDateWithObj(&date, util.Date_Short_Layout)
|
|
|
+// }
|
|
|
+// if v["bidopentime"] != nil {
|
|
|
+// date := v["bidopentime"]
|
|
|
+// v["bidopentime"] = util.FormatDateWithObj(&date, util.Date_Short_Layout)
|
|
|
+// }
|
|
|
+// if v["currency"] == "" || v["currency"] == nil {
|
|
|
+// v["currency"] = "人民币"
|
|
|
+// }
|
|
|
+// if v["subtype"] == nil && v["toptype"] != nil {
|
|
|
+// v["subtype"] = v["toptype"]
|
|
|
+// }
|
|
|
+// if v["detail"] != "" && v["detail"] != nil {
|
|
|
+// str := ClearHtml.ReplaceAllString(v["detail"].(string), "")
|
|
|
+// str = ClearOther.ReplaceAllString(str, "")
|
|
|
+// str = strings.Replace(str, " ", "", -1)
|
|
|
+// v["detail"] = str
|
|
|
+// }
|
|
|
+// if v["_id"] != nil {
|
|
|
+// v["url"] = webdomain + "/article/content/" + util.CommonEncodeArticle("content", v["_id"].(string)) + ".html"
|
|
|
+// }
|
|
|
+// }(v)
|
|
|
+// }
|
|
|
+// entexportWaitGroup.Wait()
|
|
|
+// log.Println("补充信息结束")
|
|
|
+// return data
|
|
|
+//}
|
|
|
+//
|
|
|
+//func doSearch(sql string, start, count int, dataType string) *[]map[string]interface{} {
|
|
|
+// if sql != "" {
|
|
|
+// //筛选字段
|
|
|
+// if dataType != "" {
|
|
|
+// dataexport_field := `"_id","title","detail","area","city","publishtime","projectname","buyer","s_winner","bidamount","subtype","toptype"`
|
|
|
+// if dataType == "2" {
|
|
|
+// dataexport_field += `,"href","projectcode","buyerperson","buyertel","budget","bidopentime","agency","projectscope","winnerperson","winnertel"`
|
|
|
+// }
|
|
|
+// sql = sql[:len(sql)-1] + `,"_source":[` + dataexport_field + "]}"
|
|
|
+// }
|
|
|
+// //分页排序
|
|
|
+// sql = sql[:len(sql)-1] + `,"sort": {"publishtime":"desc"},"from":` + strconv.Itoa(start) + `,"size":` + strconv.Itoa(count) + "}"
|
|
|
+// }
|
|
|
+// return elastic.Get(INDEX, TYPE, sql)
|
|
|
+//}
|
|
|
+//
|
|
|
+//func getKeyWordArrFromDbResult(k interface{}) (arr []KeyWord) {
|
|
|
+// if k == nil {
|
|
|
+// return
|
|
|
+// }
|
|
|
+// kArr := k.([]interface{})
|
|
|
+// for _, v := range kArr {
|
|
|
+// kw := KeyWord{}
|
|
|
+// b, e := json.Marshal(v)
|
|
|
+// if e != nil {
|
|
|
+// log.Println(e.Error())
|
|
|
+// }
|
|
|
+// json.Unmarshal(b, &kw)
|
|
|
+// arr = append(arr, kw)
|
|
|
+// }
|
|
|
+// return
|
|
|
+//}
|
|
|
+//
|
|
|
+//func getStringArrFromDbResult(c interface{}) (arr []string) {
|
|
|
+// if c != nil {
|
|
|
+// cArr := c.([]interface{})
|
|
|
+// arr = util.ObjArrToStringArr(cArr)
|
|
|
+// }
|
|
|
+// return
|
|
|
+//}
|
|
|
+//
|
|
|
+////获取结果,空字段最少的数据
|
|
|
+//func ScreenData(arr *[]map[string]interface{}, dataType string, resultNum int, kws []KeyWord) (res []map[string]interface{}) {
|
|
|
+// AllMap := map[int][]map[string]interface{}{}
|
|
|
+//
|
|
|
+// NoKwsMap := map[int][]map[string]interface{}{}
|
|
|
+// lastNum := resultNum
|
|
|
+// for _, v := range *arr {
|
|
|
+// emptyNum := countOfTheEmpty(v, dataType)
|
|
|
+// if emptyNum == -1 {
|
|
|
+// continue
|
|
|
+// }
|
|
|
+// if len(kws) > 0 && kws[0].Keyword != "" {
|
|
|
+// var kwsFlag = true
|
|
|
+// for _, vk := range kws {
|
|
|
+// if strings.Contains(util.ObjToString(v["title"]), strings.Replace(vk.Keyword, "+", "", -1)) {
|
|
|
+// kwsFlag = false
|
|
|
+// continue
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if kwsFlag {
|
|
|
+// if NoKwsMap[emptyNum] == nil {
|
|
|
+// NoKwsMap[emptyNum] = []map[string]interface{}{v}
|
|
|
+// } else {
|
|
|
+// NoKwsMap[emptyNum] = append(NoKwsMap[emptyNum], v)
|
|
|
+// }
|
|
|
+// continue
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if AllMap[emptyNum] == nil {
|
|
|
+// AllMap[emptyNum] = []map[string]interface{}{v}
|
|
|
+// continue
|
|
|
+// }
|
|
|
+// AllMap[emptyNum] = append(AllMap[emptyNum], v)
|
|
|
+// }
|
|
|
+// //获取key
|
|
|
+// keys := []int{}
|
|
|
+// for k, _ := range AllMap {
|
|
|
+// keys = append(keys, k)
|
|
|
+// }
|
|
|
+// sort.Ints(keys)
|
|
|
+// log.Println("空字段数量", keys)
|
|
|
+// //选取结果
|
|
|
+// for _, v := range keys {
|
|
|
+// if len(AllMap[v]) >= resultNum {
|
|
|
+// return append(res, AllMap[v][:resultNum]...)
|
|
|
+// } else {
|
|
|
+// resultNum = resultNum - len(AllMap[v])
|
|
|
+// tmp := append(res, AllMap[v][:len(AllMap[v])]...)
|
|
|
+// res = tmp
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if len(res) < lastNum {
|
|
|
+// resultNum = lastNum - len(res)
|
|
|
+// //获取key
|
|
|
+// Nokeys := []int{}
|
|
|
+// for k, _ := range NoKwsMap {
|
|
|
+// Nokeys = append(Nokeys, k)
|
|
|
+// }
|
|
|
+// sort.Ints(Nokeys)
|
|
|
+// log.Println("没关键词的空字段数量", Nokeys)
|
|
|
+// //选取结果
|
|
|
+// for _, v := range Nokeys {
|
|
|
+// if len(NoKwsMap[v]) >= resultNum {
|
|
|
+// return append(res, NoKwsMap[v][:resultNum]...)
|
|
|
+// } else {
|
|
|
+// resultNum = resultNum - len(NoKwsMap[v])
|
|
|
+// tmp := append(res, NoKwsMap[v][:len(NoKwsMap[v])]...)
|
|
|
+// res = tmp
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// return res
|
|
|
+//}
|
|
|
+//func countOfTheEmpty(m map[string]interface{}, dataType string) int {
|
|
|
+// MsgType := m["subtype"]
|
|
|
+// // if MsgType == "拟建" {
|
|
|
+// // return -1
|
|
|
+// // }
|
|
|
+// //计算空字段数量
|
|
|
+// var count int = 0
|
|
|
+// //高级字段包
|
|
|
+// if dataType == "2" {
|
|
|
+// if m["href"] == "" || m["href"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["projectcode"] == "" || m["projectcode"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["buyerperson"] == "" || m["buyerperson"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["buyertel"] == "" || m["buyertel"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["budget"] == "" || m["budget"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["bidopentime"] == "" || m["bidopentime"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["agency"] == "" || m["agency"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["projectscope"] == "" || m["projectscope"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if m["city"] == "" || m["city"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["publishtime"] == "" || m["publishtime"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["projectname"] == "" || m["projectname"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["buyer"] == "" || m["buyer"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// if m["s_winner"] == "" || m["s_winner"] == nil {
|
|
|
+// if MsgType != "招标" {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if m["bidamount"] == "" || m["bidamount"] == nil {
|
|
|
+// if MsgType != "招标" {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if m["subtype"] == "" || m["subtype"] == nil {
|
|
|
+// count++
|
|
|
+// }
|
|
|
+// return count
|
|
|
+//}
|
|
|
+//
|
|
|
+//func GetEntDataExportCount(_id string, entId, entUserId, limitNum, current int, isFirst bool) (count, newCount int, data *[]map[string]interface{}) {
|
|
|
+// defer util.Catch()
|
|
|
+// var (
|
|
|
+// searchsWaitGroup = &sync.WaitGroup{}
|
|
|
+// searchsPool = make(chan bool, 20)
|
|
|
+// // res = &[]map[string]interface{}{}
|
|
|
+// newCountPool = make(chan bool, 20000)
|
|
|
+// )
|
|
|
+// count = GetDataExportSearchCountUseId(_id)
|
|
|
+// log.Println("count", count)
|
|
|
+// if count > ExConf.MsgMaxCount {
|
|
|
+// count = ExConf.MsgMaxCount
|
|
|
+// }
|
|
|
+// dataType := "2"
|
|
|
+// //数据导出数据查询
|
|
|
+//
|
|
|
+// res, err := GetDataExportSearchResult(_id, dataType, count)
|
|
|
+// if err != nil {
|
|
|
+// log.Println("企业数据导出错误 ", err)
|
|
|
+// return 0, 0, nil
|
|
|
+// }
|
|
|
+// // secondCount := 0
|
|
|
+//
|
|
|
+// // isOK := true
|
|
|
+// // if secondCount > current {
|
|
|
+// // isOK = false
|
|
|
+// // }
|
|
|
+// for _, v := range *res {
|
|
|
+// searchsWaitGroup.Add(1)
|
|
|
+// searchsPool <- true
|
|
|
+// id := util.ObjToString(v["_id"])
|
|
|
+// go func(id string) {
|
|
|
+// defer func() {
|
|
|
+// searchsWaitGroup.Done()
|
|
|
+// <-searchsPool
|
|
|
+// }()
|
|
|
+// isExist, err := redis.Exists("other", "entexportdata_"+id+"_"+fmt.Sprintln(entId))
|
|
|
+// if err != nil {
|
|
|
+// log.Println("企业搜索数据导出redis判重失败")
|
|
|
+// } else if isExist {
|
|
|
+// log.Println("数据重复，id ", id, "entid ", entId, "userid ", entUserId)
|
|
|
+// return
|
|
|
+// }
|
|
|
+// newCountPool <- true
|
|
|
+// if !isFirst {
|
|
|
+// redis.Put("other", "entexportdata_"+id+"_"+fmt.Sprintln(entId), 1, -1)
|
|
|
+// }
|
|
|
+// }(id)
|
|
|
+// if !isFirst {
|
|
|
+// delete(v, "_id")
|
|
|
+// v["entid"] = entId
|
|
|
+// v["userid"] = entUserId
|
|
|
+// v["infoid"] = id
|
|
|
+// v["createtime"] = time.Now().Unix()
|
|
|
+// }
|
|
|
+// }
|
|
|
+// searchsWaitGroup.Wait()
|
|
|
+// log.Println("企业数据导出--数据遍历完成")
|
|
|
+// newCount = len(newCountPool)
|
|
|
+// log.Println("new", newCount)
|
|
|
+// data = res
|
|
|
+// return
|
|
|
+//}
|
|
|
+//
|
|
|
+//func GetDataExportSearchResult(id string, dataType string, checkCount int) (*[]map[string]interface{}, error) {
|
|
|
+// defer util.Catch()
|
|
|
+// var (
|
|
|
+// onceSearchCount = 500
|
|
|
+// searchPool = make(chan bool, 20)
|
|
|
+// res []map[string]interface{}
|
|
|
+// )
|
|
|
+// //获取查询语句
|
|
|
+// scd := getSqlObjFromId(id)
|
|
|
+// if scd == nil {
|
|
|
+// return nil, errors.New("GetDataExportSearchResult-获取查询条件")
|
|
|
+// }
|
|
|
+// qstr := getDataExportSql(scd)
|
|
|
+// log.Printf("GetDataExportSearchResult-%s-sql:%s\n", scd.Id, qstr)
|
|
|
+// //数据导出数据查询
|
|
|
+// if checkCount > onceSearchCount { //分批次查询
|
|
|
+// batchNum := util.IntAll(math.Ceil(float64(checkCount) / float64(onceSearchCount)))
|
|
|
+// var searchWaitGroup = &sync.WaitGroup{}
|
|
|
+// var lock sync.Mutex
|
|
|
+// for n := 0; n < batchNum; n++ {
|
|
|
+// searchWaitGroup.Add(1)
|
|
|
+// searchPool <- true
|
|
|
+// go func(start int) {
|
|
|
+// defer func() {
|
|
|
+// searchWaitGroup.Done()
|
|
|
+// <-searchPool
|
|
|
+// }()
|
|
|
+// checkNum, checkOk := onceSearchCount, false
|
|
|
+// if start == (batchNum - 1) {
|
|
|
+// if checkCount%onceSearchCount != 0 {
|
|
|
+// checkNum = checkCount % onceSearchCount
|
|
|
+// }
|
|
|
+// }
|
|
|
+//
|
|
|
+// var tmp *[]map[string]interface{}
|
|
|
+// for i := 0; i < 3; i++ {
|
|
|
+// tmp = doSearch(qstr, start*onceSearchCount, onceSearchCount, dataType)
|
|
|
+// if tmp != nil && (len(*tmp) == checkNum) { //校验数据量是否够
|
|
|
+// checkOk = true
|
|
|
+// break
|
|
|
+// }
|
|
|
+// }
|
|
|
+// if tmp == nil {
|
|
|
+// log.Printf("GetDataExportSearchResult-%s-第%d页数据查询结果为空\n", scd.Id, start+1)
|
|
|
+// return
|
|
|
+// }
|
|
|
+// if checkOk {
|
|
|
+// log.Printf("GetDataExportSearchResult-%s-第%d页数据加载完成,共%d条\n", scd.Id, start+1, len(*tmp))
|
|
|
+// } else {
|
|
|
+// log.Printf("GetDataExportSearchResult-%s-第%d页数据加载异常,共%d条,预期%d条\n", scd.Id, start+1, len(*tmp), checkNum)
|
|
|
+// }
|
|
|
+// lock.Lock()
|
|
|
+// res = append(res, *tmp...)
|
|
|
+// lock.Unlock()
|
|
|
+// }(n)
|
|
|
+// }
|
|
|
+// searchWaitGroup.Wait()
|
|
|
+// log.Printf("GetDataExportSearchResult-%s-分批次加载数据总量为%d\n", scd.Id, len(res))
|
|
|
+// } else {
|
|
|
+// tmp := doSearch(qstr, 0, checkCount, dataType)
|
|
|
+// if tmp == nil || len(*tmp) == 0 {
|
|
|
+// log.Printf("GetDataExportSearchResult-%s-一次性加载数据异常\n", scd.Id)
|
|
|
+// } else {
|
|
|
+// res = *tmp
|
|
|
+// log.Printf("GetDataExportSearchResult-%s-一次性加载数据总量为%d\n", scd.Id, len(res))
|
|
|
+// }
|
|
|
+// }
|
|
|
+// //超级搜索一致的检索（防止数据导出和超级搜索数据量不一致）
|
|
|
+// if scd.Comeinfrom == "supersearchPage" && (len(scd.Keyword) != 0 || len(scd.Industry) != 0) {
|
|
|
+// if len(scd.Keyword) != 0 {
|
|
|
+// num := len(res)
|
|
|
+// searchTextSize := 0
|
|
|
+// if len(scd.Keyword) > 0 {
|
|
|
+// searchTextSize = len([]rune(scd.Keyword[0].Keyword))
|
|
|
+// }
|
|
|
+// if searchTextSize > 3 && num < 50 {
|
|
|
+// secondKWS := jy.HttpEs(scd.Keyword[0].Keyword, "ik_smart", DbConf.Elasticsearch.Main.Address)
|
|
|
+// scd.Keyword[0].Keyword = secondKWS
|
|
|
+// scd.SelectType = "title"
|
|
|
+// qstr = getDataExportSql(scd)
|
|
|
+// log.Printf("GetDataExportSearchResult-%s-分词查询-sql:%s\n", scd.Id, qstr)
|
|
|
+// res2 := doSearch(qstr, 0, 100, "")
|
|
|
+// res = *delRepeatMapArr(&res, res2)
|
|
|
+// if len(res) > 100 {
|
|
|
+// res = res[:100]
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+// //校验数量
|
|
|
+// if checkCount != len(res) {
|
|
|
+// return nil, fmt.Errorf("GetDataExportSearchResult-%s-数据总量校验异常，期望:%d,实际:%d", scd.Id, checkCount, len(res))
|
|
|
+// //发邮件
|
|
|
+// }
|
|
|
+// return &res, nil
|
|
|
+//}
|
wangkaiyue