Merge branch 'feature/v1.0.10' into master

common/src/qfw/util/dataexport/dataexport.go

@@ -86,6 +86,11 @@ var onceSearchCount = 500
 var ExportTable string = "export_search"
 var searchPool = make(chan bool, 8)
 
+const (
+	bidBaseField   = `"_id", "title", "detail", "area", "city", "publishtime", "projectname", "buyer", "s_winner", "bidamount", "subtype", "toptype", "filetext", "purchasing"`
+	bidseniorField = bidBaseField + `,"href", "projectcode", "buyerperson", "buyertel", "budget", "bidopentime", "agency", "projectscope", "winnerperson", "winnertel", "bidendtime", "district", "signendtime", "buyeraddr","buyerclass","s_topscopeclass","entidlist","signaturedate"`
+)
+
 var topType = map[string]string{
 	"招标预告":   "预告",
 	"招标公告":   "招标",
@@ -653,44 +658,60 @@ func GetDataExportSelectReallyCountFromMongo(bid mg.MongodbSim, biddingName stri
 	if ids == nil || len(ids) == 0 {
 		return 0
 	}
-	var queryIds []interface{}
-	for _, idStr := range ids {
-		queryIds = append(queryIds, mg.StringTOBsonId(idStr))
-	}
-	lenNum := int64(len(ids))
-	num1, err1 := sess.DB(biddingName).C("bidding").Find(map[string]interface{}{"_id": map[string]interface{}{
-		"$in": queryIds,
-	}}).Count()
-	if err1 == nil {
-		if num1 == lenNum {
-			return lenNum
-		}
-		num2, err2 := sess.DB(biddingName).C("bidding_back").Find(map[string]interface{}{"_id": map[string]interface{}{
-			"$in": queryIds,
-		}}).Count()
-		if err2 == nil {
-			if num2+num1 == lenNum {
-				return lenNum
-			} else if num1+num2 > lenNum {
-				return lenNum
-			} else if num1+num2 < lenNum {
-				return num1 + num2
+
+	var (
+		count int64
+		lock  sync.Mutex
+	)
+	pool := make(chan bool, 10)
+	wait := &sync.WaitGroup{}
+
+	for _, i2 := range SplitArray(ids, 200) {
+		pool <- true
+		wait.Add(1)
+		go func(arr []string) {
+			defer func() {
+				wait.Done()
+				<-pool
+			}()
+			lenNum := int64(len(arr))
+			var (
+				queryIds   []interface{}
+				num1, num2 int64
+				err        error
+			)
+			for _, idStr := range arr {
+				queryIds = append(queryIds, mg.StringTOBsonId(idStr))
 			}
-		}
+			num1, err = sess.DB(biddingName).C("bidding").Find(map[string]interface{}{"_id": map[string]interface{}{
+				"$in": queryIds,
+			}}).Count()
+			if err == nil {
+				if num1 == lenNum {
+					lock.Lock()
+					count += num1
+					lock.Unlock()
+					return
+				}
+				num2, err = sess.DB(biddingName).C("bidding_back").Find(map[string]interface{}{"_id": map[string]interface{}{
+					"$in": queryIds,
+				}}).Count()
+				if err == nil {
+					lock.Lock()
+					count += qutil.If(num2+num1 >= lenNum, lenNum, num2+num1).(int64)
+					lock.Unlock()
+				}
+			}
+		}(i2)
 	}
-	return -2
+	wait.Wait()
+	return qutil.If(count > 0, count, -2).(int64)
 }
 
 func GetDataExportSelectResultFromEs(bidding mg.MongodbSim, biddingName string, scd *SieveCondition, dataType string, checkCount int) (*[]map[string]interface{}, error) {
-	bidField := `"_id", "title", "detail", "area", "city", "publishtime", "projectname", "buyer", "s_winner", "bidamount", "subtype", "toptype", "filetext", "purchasing"`
-	//selectMap := map[string]interface{}{
-	//	"_id": 1, "title": 1, "detail": 1, "area": 1, "city": 1, "publishtime": 1, "projectname": 1, "buyer": 1, "s_winner": 1, "bidamount": 1, "subtype": 1, "toptype": 1, "filetext": 1, "purchasing": 1,
-	//}
+	bidField := bidBaseField
 	if dataType == "2" {
-		bidField = bidField + `,"href", "projectcode", "buyerperson", "buyertel", "budget", "bidopentime", "agency", "projectscope", "winnerperson", "winnertel", "bidendtime", "district", "signendtime", "buyeraddr","buyerclass","s_topscopeclass","entidlist"`
-		//for _, key := range []string{"href", "projectcode", "buyerperson", "buyertel", "budget", "bidopentime", "agency", "projectscope", "winnerperson", "winnertel", "bidendtime", "district", "signendtime", "buyeraddr", "filetext"} {
-		//	selectMap[key] = 1
-		//}
+		bidField = bidseniorField
 	}
 	if checkCount == -1 && len(scd.SelectIds) > 500 {
 		scd.SelectIds = scd.SelectIds[:500]
@@ -740,7 +761,7 @@ func GetDataExportSelectResultFromMongoDb(bidding mg.MongodbSim, biddingName str
 		"_id": 1, "title": 1, "detail": 1, "area": 1, "city": 1, "publishtime": 1, "projectname": 1, "buyer": 1, "s_winner": 1, "bidamount": 1, "subtype": 1, "toptype": 1, "filetext": 1, "purchasing": 1,
 	}
 	if dataType == "2" {
-		for _, key := range []string{"href", "projectcode", "buyerperson", "buyertel", "budget", "bidopentime", "agency", "projectscope", "winnerperson", "winnertel", "bidendtime", "district", "signendtime", "buyeraddr", "filetext", "buyerclass", "s_topscopeclass", "entidlist"} {
+		for _, key := range []string{"href", "projectcode", "buyerperson", "buyertel", "budget", "bidopentime", "agency", "projectscope", "winnerperson", "winnertel", "bidendtime", "district", "signendtime", "buyeraddr", "filetext", "buyerclass", "s_topscopeclass", "entidlist", "signaturedate"} {
 			selectMap[key] = 1
 		}
 	}
@@ -1146,6 +1167,10 @@ func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdo
 			date := v["bidendtime"]
 			v["bidendtime"] = FormatDateWithObj(&date, Date_Short_Layout)
 		}
+		if v["signaturedate"] != nil {
+			date := v["signaturedate"]
+			v["signaturedate"] = FormatDateWithObj(&date, Date_Short_Layout)
+		}
 		if v["_id"] != nil {
 			encodeId := CommonEncodeArticle("content", v["_id"].(string))
 			v["url"] = webdomain + "/article/content/" + encodeId + ".html"
@@ -1206,9 +1231,9 @@ func doSearch(sql string, start, count int, dataType string) *[]map[string]inter
 	if sql != "" {
 		//筛选字段
 		if dataType != "" {
-			dataexport_field := `"_id","title","detail","area","city","publishtime","projectname","buyer","s_winner","bidamount","subtype","toptype","filetext","purchasing"`
+			dataexport_field := bidBaseField
 			if dataType == "2" {
-				dataexport_field += `,"href","projectcode","buyerperson","buyertel","budget","bidopentime","agency","projectscope","winnerperson","winnertel","bidendtime","district","signendtime","buyeraddr","buyerclass","s_topscopeclass","entidlist"`
+				dataexport_field = bidseniorField
 			}
 			sql = sql[:len(sql)-1] + `,"_source":[` + dataexport_field + "]}"
 		}