feat:加并发

common/src/qfw/util/dataexport/dataexport.go

@@ -2,8 +2,6 @@ package dataexport
 
 import (
 	//"config"
-	. "app.yhyue.com/moapp/jybase/date"
-	. "app.yhyue.com/moapp/jybase/encrypt"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -16,6 +14,9 @@ import (
 	"sync"
 	"time"
 
+	. "app.yhyue.com/moapp/jybase/date"
+	. "app.yhyue.com/moapp/jybase/encrypt"
+
 	"github.com/gogf/gf/v2/util/gconv"
 
 	qutil "app.yhyue.com/moapp/jybase/common"
@@ -1075,175 +1076,190 @@ func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdo
 		return time1 > time2
 	})
 	var entCacheMap = map[string]map[string]interface{}{}
+	pool := make(chan bool, 5)
+	wait := &sync.WaitGroup{}
+	lock := &sync.Mutex{}
 	for index := 0; index < len(*data); index++ {
-		v := (*data)[index]
-		//有中标企业 且 高级字段查询
-		if dataType == "2" {
-			//查询企业公示 法人 公司电话 公司邮箱地址
-			entidlist, ok := v["entidlist"].([]interface{})
-			if ok && len(entidlist) > 0 {
-				var winnerMaps []map[string]interface{}
-				for _, entIdObj := range entidlist {
-					entId := qutil.ObjToString(entIdObj)
-					if entId == "" {
-						continue
-					}
-					if entCacheMap[entId] != nil {
-						winnerMaps = append(winnerMaps, entCacheMap[entId])
-					} else if entDetail := elastic.Get("qyxy", "qyxy", fmt.Sprintf(`{"query":{"bool":{"must":[{"term":{"id":"%s"}}]}},"size":1,"_source":["company_name","company_email","company_phone","legal_person"]}`, entId)); entDetail != nil && len(*entDetail) > 0 {
-						thisEntMap := map[string]interface{}{}
-						legal_person := ""
-						if (*entDetail)[0]["legal_person"] != nil {
-							legal_person = (*entDetail)[0]["legal_person"].(string)
-							if isEncry {
-								var xx = "*"
-								switch len([]rune(legal_person)) {
-								case 3:
-									xx = "**"
-								case 4:
-									xx = "***"
+		go func(v map[string]interface{}) {
+			defer qutil.Catch()
+			defer func() {
+				<-pool
+				wait.Done()
+			}()
+			//有中标企业 且 高级字段查询
+			if dataType == "2" {
+				//查询企业公示 法人 公司电话 公司邮箱地址
+				entidlist, ok := v["entidlist"].([]interface{})
+				if ok && len(entidlist) > 0 {
+					var winnerMaps []map[string]interface{}
+					for _, entIdObj := range entidlist {
+						entId := qutil.ObjToString(entIdObj)
+						if entId == "" {
+							continue
+						}
+						lock.Lock()
+						ecm := entCacheMap[entId]
+						lock.Unlock()
+						if ecm != nil {
+							winnerMaps = append(winnerMaps, ecm)
+						} else if entDetail := elastic.Get("qyxy", "qyxy", fmt.Sprintf(`{"query":{"bool":{"must":[{"term":{"id":"%s"}}]}},"size":1,"_source":["company_name","company_email","company_phone","legal_person"]}`, entId)); entDetail != nil && len(*entDetail) > 0 {
+							thisEntMap := map[string]interface{}{}
+							legal_person := ""
+							if (*entDetail)[0]["legal_person"] != nil {
+								legal_person = (*entDetail)[0]["legal_person"].(string)
+								if isEncry {
+									var xx = "*"
+									switch len([]rune(legal_person)) {
+									case 3:
+										xx = "**"
+									case 4:
+										xx = "***"
+									}
+									legal_person = string([]rune(legal_person)[:1]) + xx
 								}
-								legal_person = string([]rune(legal_person)[:1]) + xx
 							}
-						}
-						company_phone := ""
-						if (*entDetail)[0]["company_phone"] != nil {
-							company_phone = (*entDetail)[0]["company_phone"].(string)
-							if isEncry {
-								if len([]rune(company_phone)) > 7 {
-									company_phone = company_phone[:7] + "****"
-								} else {
-									company_phone = "****"
+							company_phone := ""
+							if (*entDetail)[0]["company_phone"] != nil {
+								company_phone = (*entDetail)[0]["company_phone"].(string)
+								if isEncry {
+									if len([]rune(company_phone)) > 7 {
+										company_phone = company_phone[:7] + "****"
+									} else {
+										company_phone = "****"
+									}
 								}
 							}
-						}
-						company_email := ""
-						if (*entDetail)[0]["company_email"] != nil && (*entDetail)[0]["company_email"] != "无" {
-							company_email = (*entDetail)[0]["company_email"].(string)
-							if isEncry {
-								if len(strings.Split(company_email, "@")) > 1 {
-									company_email = "******" + "@" + strings.Split(company_email, "@")[1]
+							company_email := ""
+							if (*entDetail)[0]["company_email"] != nil && (*entDetail)[0]["company_email"] != "无" {
+								company_email = (*entDetail)[0]["company_email"].(string)
+								if isEncry {
+									if len(strings.Split(company_email, "@")) > 1 {
+										company_email = "******" + "@" + strings.Split(company_email, "@")[1]
+									}
 								}
 							}
+							company_name := ""
+							if (*entDetail)[0]["company_name"] != nil {
+								company_name = (*entDetail)[0]["company_name"].(string)
+							}
+							thisEntMap["legal_person"] = legal_person
+							thisEntMap["company_phone"] = company_phone
+							thisEntMap["company_email"] = company_email
+							thisEntMap["company_name"] = company_name
+							lock.Lock()
+							entCacheMap[entId] = thisEntMap
+							lock.Unlock()
+							winnerMaps = append(winnerMaps, thisEntMap)
 						}
-						company_name := ""
-						if (*entDetail)[0]["company_name"] != nil {
-							company_name = (*entDetail)[0]["company_name"].(string)
-						}
-						thisEntMap["legal_person"] = legal_person
-						thisEntMap["company_phone"] = company_phone
-						thisEntMap["company_email"] = company_email
-						thisEntMap["company_name"] = company_name
-						entCacheMap[entId] = thisEntMap
-						winnerMaps = append(winnerMaps, thisEntMap)
+					}
+					if len(winnerMaps) > 0 {
+						v["winnerMaps"] = winnerMaps
 					}
 				}
-				if len(winnerMaps) > 0 {
-					v["winnerMaps"] = winnerMaps
-				}
+				delete(v, "entidlist")
 			}
-			delete(v, "entidlist")
-		}
-		//====================字段补漏=========================
-		if v["toptype"] == "结果" && dataType == "2" && !(v["agency"] != nil && v["budget"] != nil && v["buyerperson"] != nil && v["buyertel"] != nil) {
-			r := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query":{"term":{"list.infoid":"%s"}},"_source": ["list"]}`, v["_id"]))
-			if r != nil && len(*r) > 0 {
-				MsgList := (*r)[0]["list"]
-				if MsgList != nil {
-					list := qutil.ObjArrToMapArr(MsgList.([]interface{}))
-					for _, vv := range list {
-						if vv["subtype"] == "招标" {
-							if v["agency"] == nil && vv["agency"] != nil {
-								v["agency"] = vv["agency"]
-							}
-							if v["budget"] == nil && vv["budget"] != nil {
-								v["budget"] = vv["budget"]
-							}
-							if v["buyerperson"] == nil && vv["buyerperson"] != nil {
-								v["buyerperson"] = vv["buyerperson"]
-							}
-							if v["buyertel"] == nil && vv["buyertel"] != nil {
-								v["buyertel"] = vv["buyertel"]
+			//====================字段补漏=========================
+			if v["toptype"] == "结果" && dataType == "2" && !(v["agency"] != nil && v["budget"] != nil && v["buyerperson"] != nil && v["buyertel"] != nil) {
+				r := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query":{"term":{"list.infoid":"%s"}},"_source": ["list"]}`, v["_id"]))
+				if r != nil && len(*r) > 0 {
+					MsgList := (*r)[0]["list"]
+					if MsgList != nil {
+						list := qutil.ObjArrToMapArr(MsgList.([]interface{}))
+						for _, vv := range list {
+							if vv["subtype"] == "招标" {
+								if v["agency"] == nil && vv["agency"] != nil {
+									v["agency"] = vv["agency"]
+								}
+								if v["budget"] == nil && vv["budget"] != nil {
+									v["budget"] = vv["budget"]
+								}
+								if v["buyerperson"] == nil && vv["buyerperson"] != nil {
+									v["buyerperson"] = vv["buyerperson"]
+								}
+								if v["buyertel"] == nil && vv["buyertel"] != nil {
+									v["buyertel"] = vv["buyertel"]
+								}
+								break
 							}
-							break
 						}
 					}
 				}
 			}
-		}
-		if v["area"] == "A" {
-			v["area"] = "全国"
-		}
-		if v["bidamount"] != nil {
-			v["bidamount"] = formatFloat(qutil.Float64All(v["bidamount"]))
-		}
-		if v["budget"] != nil {
-			v["budget"] = formatFloat(qutil.Float64All(v["budget"]))
-		}
-		if v["publishtime"] != nil {
-			date := v["publishtime"]
-			v["publishtime"] = FormatDateWithObj(&date, Date_Short_Layout)
-		}
-		if v["bidopentime"] != nil {
-			date := v["bidopentime"]
-			v["bidopentime"] = FormatDateWithObj(&date, Date_Short_Layout)
-		}
-		if qutil.IntAll(v["signendtime"]) != 0 {
-			date := v["signendtime"]
-			v["signendtime"] = FormatDateWithObj(&date, Date_Short_Layout)
-		}
-		if v["bidendtime"] != nil {
-			date := v["bidendtime"]
-			v["bidendtime"] = FormatDateWithObj(&date, Date_Short_Layout)
-		}
-		if v["signaturedate"] != nil {
-			date := v["signaturedate"]
-			v["signaturedate"] = FormatDateWithObj(&date, Date_Short_Layout)
-		}
-		if v["_id"] != nil {
-			encodeId := CommonEncodeArticle("content", v["_id"].(string))
-			v["url"] = webdomain + "/article/content/" + encodeId + ".html"
-			v["url_jump"] = webdomain + "/front/reloadTo/article/content/" + encodeId + ".html"
-		}
-		if v["currency"] == "" || v["currency"] == nil {
-			v["currency"] = "人民币"
-		}
+			if v["area"] == "A" {
+				v["area"] = "全国"
+			}
+			if v["bidamount"] != nil {
+				v["bidamount"] = formatFloat(qutil.Float64All(v["bidamount"]))
+			}
+			if v["budget"] != nil {
+				v["budget"] = formatFloat(qutil.Float64All(v["budget"]))
+			}
+			if v["publishtime"] != nil {
+				date := v["publishtime"]
+				v["publishtime"] = FormatDateWithObj(&date, Date_Short_Layout)
+			}
+			if v["bidopentime"] != nil {
+				date := v["bidopentime"]
+				v["bidopentime"] = FormatDateWithObj(&date, Date_Short_Layout)
+			}
+			if qutil.IntAll(v["signendtime"]) != 0 {
+				date := v["signendtime"]
+				v["signendtime"] = FormatDateWithObj(&date, Date_Short_Layout)
+			}
+			if v["bidendtime"] != nil {
+				date := v["bidendtime"]
+				v["bidendtime"] = FormatDateWithObj(&date, Date_Short_Layout)
+			}
+			if v["signaturedate"] != nil {
+				date := v["signaturedate"]
+				v["signaturedate"] = FormatDateWithObj(&date, Date_Short_Layout)
+			}
+			if v["_id"] != nil {
+				encodeId := CommonEncodeArticle("content", v["_id"].(string))
+				v["url"] = webdomain + "/article/content/" + encodeId + ".html"
+				v["url_jump"] = webdomain + "/front/reloadTo/article/content/" + encodeId + ".html"
+			}
+			if v["currency"] == "" || v["currency"] == nil {
+				v["currency"] = "人民币"
+			}
 
-		if isEncry {
-			if v["projectscope"] != "" && v["projectscope"] != nil {
-				str := ClearHtml.ReplaceAllString(v["projectscope"].(string), "")
-				str = ClearOther.ReplaceAllString(str, "")
-				str = strings.Replace(str, " ", "", -1)
-				if len([]rune(str)) > 100 {
-					str = qutil.SubString(str, 0, 100) + "..."
+			if isEncry {
+				if v["projectscope"] != "" && v["projectscope"] != nil {
+					str := ClearHtml.ReplaceAllString(v["projectscope"].(string), "")
+					str = ClearOther.ReplaceAllString(str, "")
+					str = strings.Replace(str, " ", "", -1)
+					if len([]rune(str)) > 100 {
+						str = qutil.SubString(str, 0, 100) + "..."
+					}
+					v["projectscope"] = str
 				}
-				v["projectscope"] = str
-			}
-			if v["detail"] != "" && v["detail"] != nil {
-				str := ClearHtml.ReplaceAllString(v["detail"].(string), "")
-				str = ClearOther.ReplaceAllString(str, "")
-				str = strings.Replace(str, " ", "", -1)
-				if len([]rune(str)) > 100 {
-					str = qutil.SubString(str, 0, 100) + "..."
+				if v["detail"] != "" && v["detail"] != nil {
+					str := ClearHtml.ReplaceAllString(v["detail"].(string), "")
+					str = ClearOther.ReplaceAllString(str, "")
+					str = strings.Replace(str, " ", "", -1)
+					if len([]rune(str)) > 100 {
+						str = qutil.SubString(str, 0, 100) + "..."
+					}
+					v["detail"] = str
 				}
-				v["detail"] = str
-			}
-			if v["title"] != "" && v["title"] != nil {
-				str := ClearHtml.ReplaceAllString(v["title"].(string), "")
-				str = ClearOther.ReplaceAllString(str, "")
-				str = strings.Replace(str, " ", "", -1)
-				if len([]rune(str)) > 100 {
-					str = qutil.SubString(str, 0, 100) + "..."
+				if v["title"] != "" && v["title"] != nil {
+					str := ClearHtml.ReplaceAllString(v["title"].(string), "")
+					str = ClearOther.ReplaceAllString(str, "")
+					str = strings.Replace(str, " ", "", -1)
+					if len([]rune(str)) > 100 {
+						str = qutil.SubString(str, 0, 100) + "..."
+					}
+					v["title"] = str
 				}
-				v["title"] = str
-			}
 
-		}
+			}
 
-		if v["subtype"] == nil && v["toptype"] != nil {
-			v["subtype"] = v["toptype"]
-		}
+			if v["subtype"] == nil && v["toptype"] != nil {
+				v["subtype"] = v["toptype"]
+			}
+		}((*data)[index])
 	}
+	wait.Wait()
 	return data
 }