Ver código fonte

fix:数据导出并发优化

duxin 1 ano atrás
pai
commit
86c8254d39
1 arquivos alterados com 12 adições e 9 exclusões
  1. 12 9
      common/src/qfw/util/dataexport/dataexport.go

+ 12 - 9
common/src/qfw/util/dataexport/dataexport.go

@@ -628,7 +628,8 @@ func GetDataExportSelectReallyCountFromEs(ids []string) int64 {
 	wait := &sync.WaitGroup{}
 	var total int64
 	var lock sync.Mutex
-	for _, v := range SplitArray(ids, 200) {
+	idArr := SplitArray(ids, 200)
+	for i := 0; i < len(idArr); i++ {
 		pool <- true
 		wait.Add(1)
 		go func(arr []string) {
@@ -644,7 +645,7 @@ func GetDataExportSelectReallyCountFromEs(ids []string) int64 {
 				lock.Unlock()
 			}
 			return
-		}(v)
+		}(idArr[i])
 	}
 	wait.Wait()
 	log.Printf("GetDataExportSelectReallyCount 选择数据共%d条记录,实际查询%d条\n", len(ids), total)
@@ -665,8 +666,8 @@ func GetDataExportSelectReallyCountFromMongo(bid mg.MongodbSim, biddingName stri
 	)
 	pool := make(chan bool, 10)
 	wait := &sync.WaitGroup{}
-
-	for _, i2 := range SplitArray(ids, 200) {
+	idArr := SplitArray(ids, 200)
+	for i := 0; i < len(idArr); i++ {
 		pool <- true
 		wait.Add(1)
 		go func(arr []string) {
@@ -702,7 +703,7 @@ func GetDataExportSelectReallyCountFromMongo(bid mg.MongodbSim, biddingName stri
 					lock.Unlock()
 				}
 			}
-		}(i2)
+		}(idArr[i])
 	}
 	wait.Wait()
 	return qutil.If(count > 0, count, -2).(int64)
@@ -720,7 +721,8 @@ func GetDataExportSelectResultFromEs(bidding mg.MongodbSim, biddingName string,
 	wait := &sync.WaitGroup{}
 	var lock sync.Mutex
 	returnLsit := make([]map[string]interface{}, 0, len(scd.SelectIds))
-	for _, v := range SplitArray(scd.SelectIds, 200) {
+	idArr := SplitArray(scd.SelectIds, 200)
+	for i := 0; i < len(idArr); i++ {
 		pool <- true
 		wait.Add(1)
 		go func(arr []string) error {
@@ -744,7 +746,7 @@ func GetDataExportSelectResultFromEs(bidding mg.MongodbSim, biddingName string,
 				}
 			}
 			return nil
-		}(v)
+		}(idArr[i])
 	}
 	wait.Wait()
 	if len(returnLsit) == checkCount || checkCount == -1 {
@@ -772,7 +774,8 @@ func GetDataExportSelectResultFromMongoDb(bidding mg.MongodbSim, biddingName str
 	wait := &sync.WaitGroup{}
 	var lock sync.Mutex
 	returnLsit := make([]map[string]interface{}, 0, len(scd.SelectIds))
-	for _, v := range SplitArray(scd.SelectIds, 200) {
+	idArr := SplitArray(scd.SelectIds, 200)
+	for i := 0; i < len(idArr); i++ {
 		pool <- true
 		wait.Add(1)
 		go func(arr []string) error {
@@ -813,7 +816,7 @@ func GetDataExportSelectResultFromMongoDb(bidding mg.MongodbSim, biddingName str
 				m = make(map[string]interface{})
 			}
 			return nil
-		}(v)
+		}(idArr[i])
 	}
 	wait.Wait()
 	if len(returnLsit) == checkCount || checkCount == -1 {