|
@@ -647,11 +647,6 @@ func GetDataExportSelectReallyCount(bid mg.MongodbSim, biddingName string, ids [
|
|
|
func GetDataExportSelectResult(bidding mg.MongodbSim, biddingName string, scd *SieveCondition, dataType string, checkCount int) (*[]map[string]interface{}, error) {
|
|
|
sess := bidding.GetMgoConn()
|
|
|
defer bidding.DestoryMongoConn(sess)
|
|
|
- var queryIds []interface{}
|
|
|
- for _, idStr := range scd.SelectIds {
|
|
|
- queryIds = append(queryIds, mg.StringTOBsonId(idStr))
|
|
|
- }
|
|
|
-
|
|
|
selectMap := map[string]interface{}{
|
|
|
"_id": 1, "title": 1, "detail": 1, "area": 1, "city": 1, "publishtime": 1, "projectname": 1, "buyer": 1, "s_winner": 1, "bidamount": 1, "subtype": 1, "toptype": 1, "filetext": 1, "purchasing": 1,
|
|
|
}
|
|
@@ -660,38 +655,93 @@ func GetDataExportSelectResult(bidding mg.MongodbSim, biddingName string, scd *S
|
|
|
selectMap[key] = 1
|
|
|
}
|
|
|
}
|
|
|
- returnLsit := make([]map[string]interface{}, 0, len(queryIds))
|
|
|
- iter := sess.DB(biddingName).C("bidding").Select(selectMap).Find(map[string]interface{}{"_id": map[string]interface{}{
|
|
|
- "$in": queryIds,
|
|
|
- }}).Iter()
|
|
|
- for m := make(map[string]interface{}); iter.Next(&m); {
|
|
|
- m["_id"] = mg.BsonIdToSId(m["_id"])
|
|
|
- detail, _ := m["detail"].(string)
|
|
|
- if detail != "" {
|
|
|
- m["detail"] = contentfilterReg.ReplaceAllString(detail, "")
|
|
|
- }
|
|
|
- returnLsit = append(returnLsit, m)
|
|
|
- m = make(map[string]interface{})
|
|
|
+ if checkCount == -1 && len(scd.SelectIds) > 500 {
|
|
|
+ scd.SelectIds = scd.SelectIds[:500]
|
|
|
+ }
|
|
|
+ pool := make(chan bool, 10)
|
|
|
+ wait := &sync.WaitGroup{}
|
|
|
+ var lock sync.Mutex
|
|
|
+ returnLsit := make([]map[string]interface{}, 0, len(scd.SelectIds))
|
|
|
+ for _, v := range SplitArray(scd.SelectIds, 200) {
|
|
|
+ pool <- true
|
|
|
+ wait.Add(1)
|
|
|
+ go func(arr []string) error {
|
|
|
+ defer func() {
|
|
|
+ wait.Done()
|
|
|
+ <-pool
|
|
|
+ }()
|
|
|
+ var queryIds []interface{}
|
|
|
+ for _, idStr := range arr {
|
|
|
+ queryIds = append(queryIds, mg.StringTOBsonId(idStr))
|
|
|
+ }
|
|
|
+ iter := sess.DB(biddingName).C("bidding").Select(selectMap).Find(map[string]interface{}{"_id": map[string]interface{}{
|
|
|
+ "$in": queryIds,
|
|
|
+ }}).Iter()
|
|
|
+ for m := make(map[string]interface{}); iter.Next(&m); {
|
|
|
+ m["_id"] = mg.BsonIdToSId(m["_id"])
|
|
|
+ detail, _ := m["detail"].(string)
|
|
|
+ if detail != "" {
|
|
|
+ m["detail"] = contentfilterReg.ReplaceAllString(detail, "")
|
|
|
+ }
|
|
|
+ lock.Lock()
|
|
|
+ returnLsit = append(returnLsit, m)
|
|
|
+ lock.Unlock()
|
|
|
+ m = make(map[string]interface{})
|
|
|
+ }
|
|
|
+ iter_back := sess.DB(biddingName).C("bidding_back").Select(selectMap).Find(map[string]interface{}{"_id": map[string]interface{}{
|
|
|
+ "$in": queryIds,
|
|
|
+ }}).Iter()
|
|
|
+ for m := make(map[string]interface{}); iter_back.Next(&m); {
|
|
|
+ m["_id"] = mg.BsonIdToSId(m["_id"])
|
|
|
+ detail, _ := m["detail"].(string)
|
|
|
+ if detail != "" {
|
|
|
+ m["detail"] = contentfilterReg.ReplaceAllString(detail, "")
|
|
|
+ }
|
|
|
+ lock.Lock()
|
|
|
+ returnLsit = append(returnLsit, m)
|
|
|
+ lock.Unlock()
|
|
|
+ m = make(map[string]interface{})
|
|
|
+ }
|
|
|
+ return nil
|
|
|
+ }(v)
|
|
|
+
|
|
|
}
|
|
|
- if len(returnLsit) == checkCount {
|
|
|
+ wait.Wait()
|
|
|
+ if len(returnLsit) == checkCount || checkCount == -1 {
|
|
|
return &returnLsit, nil
|
|
|
+ } else {
|
|
|
+ return nil, fmt.Errorf("选择数据导出异常 数据量期望%d条,实际查询%d条", checkCount, len(returnLsit))
|
|
|
}
|
|
|
- iter_back := sess.DB(biddingName).C("bidding_back").Select(selectMap).Find(map[string]interface{}{"_id": map[string]interface{}{
|
|
|
- "$in": queryIds,
|
|
|
- }}).Iter()
|
|
|
- for m := make(map[string]interface{}); iter_back.Next(&m); {
|
|
|
- m["_id"] = mg.BsonIdToSId(m["_id"])
|
|
|
- detail, _ := m["detail"].(string)
|
|
|
- if detail != "" {
|
|
|
- m["detail"] = contentfilterReg.ReplaceAllString(detail, "")
|
|
|
+}
|
|
|
+
|
|
|
+// SplitArray 分割数组
|
|
|
+func SplitArray(arr []string, num int64) [][]string {
|
|
|
+ max := int64(len(arr))
|
|
|
+ //判断数组大小是否小于等于指定分割大小的值,是则把原数组放入二维数组返回
|
|
|
+ if max <= num {
|
|
|
+ return [][]string{arr}
|
|
|
+ }
|
|
|
+ //获取应该数组分割为多少份
|
|
|
+ var quantity int64
|
|
|
+ if max%num == 0 {
|
|
|
+ quantity = max / num
|
|
|
+ } else {
|
|
|
+ quantity = (max / num) + 1
|
|
|
+ }
|
|
|
+ //声明分割好的二维数组
|
|
|
+ var segments = make([][]string, 0)
|
|
|
+ //声明分割数组的截止下标
|
|
|
+ var start, end, i int64
|
|
|
+ for i = 1; i <= quantity; i++ {
|
|
|
+ end = i * num
|
|
|
+ if i != quantity {
|
|
|
+ segments = append(segments, arr[start:end])
|
|
|
+ } else {
|
|
|
+ segments = append(segments, arr[start:])
|
|
|
}
|
|
|
- returnLsit = append(returnLsit, m)
|
|
|
- m = make(map[string]interface{})
|
|
|
- }
|
|
|
- if len(returnLsit) == checkCount || checkCount == -1 {
|
|
|
- return &returnLsit, nil
|
|
|
+ start = i * num
|
|
|
}
|
|
|
- return nil, fmt.Errorf("选择数据导出异常 数据量期望%d条,实际查询%d条", checkCount, len(returnLsit))
|
|
|
+ return segments
|
|
|
}
|
|
|
|
|
|
func GetDataExportIds(elasticAddress string, scd *SieveCondition, checkCount int) ([]string, error) {
|
|
@@ -983,7 +1033,7 @@ func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdo
|
|
|
date := v["bidopentime"]
|
|
|
v["bidopentime"] = FormatDateWithObj(&date, Date_Short_Layout)
|
|
|
}
|
|
|
- if v["signendtime"] != nil {
|
|
|
+ if qutil.IntAll(v["signendtime"]) != 0 {
|
|
|
date := v["signendtime"]
|
|
|
v["signendtime"] = FormatDateWithObj(&date, Date_Short_Layout)
|
|
|
}
|
|
@@ -1053,7 +1103,7 @@ func doSearch(sql string, start, count int, dataType string) *[]map[string]inter
|
|
|
if dataType != "" {
|
|
|
dataexport_field := `"_id","title","detail","area","city","publishtime","projectname","buyer","s_winner","bidamount","subtype","toptype","filetext","purchasing"`
|
|
|
if dataType == "2" {
|
|
|
- dataexport_field += `,"href","projectcode","buyerperson","buyertel","budget","bidopentime","agency","projectscope","winnerperson","winnertel","bidendtime", "district", "signendtime", "buyeraddr"`
|
|
|
+ dataexport_field += `,"href","projectcode","buyerperson","buyertel","budget","bidopentime","agency","projectscope","winnerperson","winnertel","bidendtime","district","signendtime","buyeraddr"`
|
|
|
}
|
|
|
sql = sql[:len(sql)-1] + `,"_source":[` + dataexport_field + "]}"
|
|
|
}
|