wip:只有数据量比支付条数少时返回异常，否则正常导出;数据导出选择导出使用mongodb查询，条件导出使用es查询具体数据

common/src/qfw/util/dataexport/dataexport.go

@@ -282,23 +282,9 @@ func getDataExportSql(scd *SieveCondition) string {
 				} else if strings.Contains(selectType, ",title") {
 					selectType = strings.Replace(selectType, ",title", "", -1)
 				}
+
 			}
-			selectMap := map[string]bool{}
-			for _, key := range strings.Split(selectType, ",") {
-				selectMap[key] = true
-			}
-			//todo: 领导说：如果前三个都有，拼语句的时候忽略第四个吧
-			if scd.Comeinfrom == "exportPage" && selectMap["detail"] && selectMap["filetext"] && selectMap["title"] {
-				delete(selectMap, "purchasing")
-				delete(selectMap, "projectname.pname")
-			}
-			if len(selectMap) > 0 {
-				var arr []string
-				for key, _ := range selectMap {
-					arr = append(arr, key)
-				}
-				queryItem = strings.Join(arr, "\",\"")
-			}
+			queryItem = strings.ReplaceAll(selectType, ",", "\",\"")
 		}
 		multi_match_new := fmt.Sprintf(multi_match, "%s", "\""+queryItem+"\"")
 
@@ -510,12 +496,12 @@ func GetSqlObjFromId(mongo mg.MongodbSim, _id string) *SieveCondition {
 }
 
 // 数据导出-查询结果数量
-func GetDataExportSearchCountByScdId(sim mg.MongodbSim, elasticAddress, id string) (count int) {
+func GetDataExportSearchCountByScdId(sim, bid mg.MongodbSim, biddingName, elasticAddress, id string) (count int) {
 	scd := GetSqlObjFromId(sim, id) //用户筛选条件
 	if scd.SelectIds != nil {
 		//部分数据可能已删除、不存在；此处需要统计返回实际数量
 		//return len(scd.SelectIds)
-		return int(GetDataExportSelectReallyCount(scd.SelectIds))
+		return int(GetDataExportSelectReallyCountFromMongo(bid, biddingName, scd.SelectIds))
 	}
 	return GetDataExportSearchCountBySieveCondition(scd, elasticAddress)
 }
@@ -600,9 +586,9 @@ func isNullSearch(scd *SieveCondition) (isNull bool) {
  * count 返回数量 (-1:预览数据查询)
  */
 
-func GetDataExportSearchResultByScdId(sim mg.MongodbSim, elasticAddress, id, dataType string, checkCount int) (*[]map[string]interface{}, error) {
+func GetDataExportSearchResultByScdId(sim, bid mg.MongodbSim, bidMgoDBName, elasticAddress, id, dataType string, checkCount int) (*[]map[string]interface{}, error) {
 	scd := GetSqlObjFromId(sim, id)
-	list, err := GetDataExportSearchResult(elasticAddress, scd, dataType, checkCount)
+	list, err := GetDataExportSearchResult(bid, bidMgoDBName, elasticAddress, scd, dataType, checkCount)
 	if list == nil || err != nil {
 		return nil, err
 	}
@@ -618,8 +604,8 @@ func GetDataExportIdArrByScdId(sim mg.MongodbSim, elasticAddress, id string, che
 // 收藏导出
 var contentfilterReg = regexp.MustCompile("<[^>]+>")
 
-// GetDataExportSelectReallyCount 查询实际可调导出数量
-func GetDataExportSelectReallyCount(ids []string) int64 {
+// GetDataExportSelectReallyCountFromEs 从elasticsearch查询实际可调导出数量
+func GetDataExportSelectReallyCountFromEs(ids []string) int64 {
 	pool := make(chan bool, 10)
 	wait := &sync.WaitGroup{}
 	var total int64
@@ -647,9 +633,42 @@ func GetDataExportSelectReallyCount(ids []string) int64 {
 	return total
 }
 
-func GetDataExportSelectResult(scd *SieveCondition, dataType string, checkCount int) (*[]map[string]interface{}, error) {
-	//sess := bidding.GetMgoConn()
-	//defer bidding.DestoryMongoConn(sess)
+// GetDataExportSelectReallyCountFromMongo 从mongodb查询实际可调导出数量
+func GetDataExportSelectReallyCountFromMongo(bid mg.MongodbSim, biddingName string, ids []string) int64 {
+	sess := bid.GetMgoConn()
+	defer bid.DestoryMongoConn(sess)
+	if ids == nil || len(ids) == 0 {
+		return 0
+	}
+	var queryIds []interface{}
+	for _, idStr := range ids {
+		queryIds = append(queryIds, mg.StringTOBsonId(idStr))
+	}
+	lenNum := int64(len(ids))
+	num1, err1 := sess.DB(biddingName).C("bidding").Find(map[string]interface{}{"_id": map[string]interface{}{
+		"$in": queryIds,
+	}}).Count()
+	if err1 == nil {
+		if num1 == lenNum {
+			return lenNum
+		}
+		num2, err2 := sess.DB(biddingName).C("bidding_back").Find(map[string]interface{}{"_id": map[string]interface{}{
+			"$in": queryIds,
+		}}).Count()
+		if err2 == nil {
+			if num2+num1 == lenNum {
+				return lenNum
+			} else if num1+num2 > lenNum {
+				return lenNum
+			} else if num1+num2 < lenNum {
+				return num1 + num2
+			}
+		}
+	}
+	return -2
+}
+
+func GetDataExportSelectResultFromEs(bidding mg.MongodbSim, biddingName string, scd *SieveCondition, dataType string, checkCount int) (*[]map[string]interface{}, error) {
 	bidField := `"_id", "title", "detail", "area", "city", "publishtime", "projectname", "buyer", "s_winner", "bidamount", "subtype", "toptype", "filetext", "purchasing"`
 	//selectMap := map[string]interface{}{
 	//	"_id": 1, "title": 1, "detail": 1, "area": 1, "city": 1, "publishtime": 1, "projectname": 1, "buyer": 1, "s_winner": 1, "bidamount": 1, "subtype": 1, "toptype": 1, "filetext": 1, "purchasing": 1,
@@ -675,12 +694,8 @@ func GetDataExportSelectResult(scd *SieveCondition, dataType string, checkCount
 				wait.Done()
 				<-pool
 			}()
-			//var queryIds []interface{}
-			//for _, idStr := range arr {
-			//	queryIds = append(queryIds, mg.StringTOBsonId(idStr))
-			//}
 			query := fmt.Sprintf(`{"query":{"bool":{"must":[{"terms":{"id":["%s"]}}]}},"_source": [%s],"size":%d}`, strings.Join(arr, "\",\""), bidField, len(arr))
-			log.Println("数据流量包 es count 信息查询：", query)
+			log.Println("GetDataExportSelectResultFromEs 数据流量包 es count 信息查询：", query)
 			data := *elastic.Get(INDEX, TYPE, query)
 			if data != nil && len(data) > 0 {
 				for _, bv := range data {
@@ -701,7 +716,76 @@ func GetDataExportSelectResult(scd *SieveCondition, dataType string, checkCount
 	if len(returnLsit) == checkCount || checkCount == -1 {
 		return &returnLsit, nil
 	} else {
-		return nil, fmt.Errorf("选择数据导出异常 数据量期望%d条，实际查询%d条", checkCount, len(returnLsit))
+		return nil, fmt.Errorf("GetDataExportSelectResultFromEs 选择数据导出异常 数据量期望%d条，实际查询%d条", checkCount, len(returnLsit))
+	}
+}
+
+func GetDataExportSelectResultFromMongoDb(bidding mg.MongodbSim, biddingName string, scd *SieveCondition, dataType string, checkCount int) (*[]map[string]interface{}, error) {
+	sess := bidding.GetMgoConn()
+	defer bidding.DestoryMongoConn(sess)
+	selectMap := map[string]interface{}{
+		"_id": 1, "title": 1, "detail": 1, "area": 1, "city": 1, "publishtime": 1, "projectname": 1, "buyer": 1, "s_winner": 1, "bidamount": 1, "subtype": 1, "toptype": 1, "filetext": 1, "purchasing": 1,
+	}
+	if dataType == "2" {
+		for _, key := range []string{"href", "projectcode", "buyerperson", "buyertel", "budget", "bidopentime", "agency", "projectscope", "winnerperson", "winnertel", "bidendtime", "district", "signendtime", "buyeraddr", "filetext", "buyerclass", "s_topscopeclass", "entidlist"} {
+			selectMap[key] = 1
+		}
+	}
+	if checkCount == -1 && len(scd.SelectIds) > 500 {
+		scd.SelectIds = scd.SelectIds[:500]
+	}
+	pool := make(chan bool, 10)
+	wait := &sync.WaitGroup{}
+	var lock sync.Mutex
+	returnLsit := make([]map[string]interface{}, 0, len(scd.SelectIds))
+	for _, v := range SplitArray(scd.SelectIds, 200) {
+		pool <- true
+		wait.Add(1)
+		go func(arr []string) error {
+			defer func() {
+				wait.Done()
+				<-pool
+			}()
+			var queryIds []interface{}
+			for _, idStr := range arr {
+				queryIds = append(queryIds, mg.StringTOBsonId(idStr))
+			}
+			iter := sess.DB(biddingName).C("bidding").Select(selectMap).Find(map[string]interface{}{"_id": map[string]interface{}{
+				"$in": queryIds,
+			}}).Iter()
+			for m := make(map[string]interface{}); iter.Next(&m); {
+				m["_id"] = mg.BsonIdToSId(m["_id"])
+				detail, _ := m["detail"].(string)
+				if detail != "" {
+					m["detail"] = contentfilterReg.ReplaceAllString(detail, "")
+				}
+				lock.Lock()
+				returnLsit = append(returnLsit, m)
+				lock.Unlock()
+				m = make(map[string]interface{})
+			}
+			iter_back := sess.DB(biddingName).C("bidding_back").Select(selectMap).Find(map[string]interface{}{"_id": map[string]interface{}{
+				"$in": queryIds,
+			}}).Iter()
+			for m := make(map[string]interface{}); iter_back.Next(&m); {
+				m["_id"] = mg.BsonIdToSId(m["_id"])
+				detail, _ := m["detail"].(string)
+				if detail != "" {
+					m["detail"] = contentfilterReg.ReplaceAllString(detail, "")
+				}
+				lock.Lock()
+				returnLsit = append(returnLsit, m)
+				lock.Unlock()
+				m = make(map[string]interface{})
+			}
+			return nil
+		}(v)
+	}
+	wait.Wait()
+	if len(returnLsit) == checkCount || checkCount == -1 {
+		return &returnLsit, nil
+	} else {
+		return nil, fmt.Errorf("GetDataExportSelectResultFromMongoDb 选择数据导出异常 数据量期望%d条，实际查询%d条", checkCount, len(returnLsit))
 	}
 }
 
@@ -797,13 +881,13 @@ func GetDataExportIds(elasticAddress string, scd *SieveCondition, checkCount int
 // GetDataExportSearchResult 获取数据导出内容
 // entmg 高级字段包查询企业电话邮箱等字段
 // checkCount -1 预览500条
-func GetDataExportSearchResult(elasticAddress string, scd *SieveCondition, dataType string, checkCount int) (*[]map[string]interface{}, error) {
+func GetDataExportSearchResult(bid mg.MongodbSim, bidMgoDBName, elasticAddress string, scd *SieveCondition, dataType string, checkCount int) (*[]map[string]interface{}, error) {
 	defer qutil.Catch()
 	if scd == nil {
 		return nil, errors.New("GetDataExportSearchResult-未获取到查询信息")
 	}
 	if scd.SelectIds != nil {
-		idSelectDates, idSelectErr := GetDataExportSelectResult(scd, dataType, checkCount)
+		idSelectDates, idSelectErr := GetDataExportSelectResultFromMongoDb(bid, bidMgoDBName, scd, dataType, checkCount)
 		if idSelectErr != nil {
 			return nil, idSelectErr
 		}
@@ -852,7 +936,7 @@ func GetDataExportSearchResult(elasticAddress string, scd *SieveCondition, dataT
 		res = doSearchByBatch(qstr, dataType, checkCount, fmt.Sprintf("%s-%s", "GetDataExportSearchResult", scd.Id))
 	}
 	//校验数量
-	if checkCount > len(res) && checkCount != -1 {
+	if checkCount != len(res) && checkCount != -1 {
 		return nil, fmt.Errorf("GetDataExportSearchResult-%s-数据总量校验异常，期望:%d,实际:%d", scd.Id, checkCount, len(res))
 	}
 	if selectType != scd.SelectType {
@@ -921,7 +1005,7 @@ func doSearchByBatch(query, dataType string, searchCount int, flag string) (res
 	return
 }
 
-func FormatExportData(data *[]map[string]interface{}, webdomain string, dataType string, encry ...bool) *[]map[string]interface{} {
+func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdomain string, dataType string, encry ...bool) *[]map[string]interface{} {
 	//格式化输出
 	isEncry := false
 	if len(encry) > 0 {