fix:数据导出预览最优数据筛选优化

common/src/qfw/util/dataexport/dataexport.go

@@ -94,6 +94,7 @@ const (
 var ClearOther = regexp.MustCompile("[\n\r\\s\u3000\u2003\u00a0]")
 var ClearHtml = regexp.MustCompile("<[^>]*>")
 var onceSearchCount = 500
+var PreviewSearchCount = 100
 var ExportTable string = "export_search"
 var searchPool = make(chan bool, 8)
 
@@ -250,13 +251,6 @@ func getDataExportSql(scd *SieveCondition) string {
 			scd.Subtype = TopTypesPreviewsProject
 		case SearchGroupResultsProject:
 			scd.Subtype = TopTypesResultsProject
-
-		}
-		if scd.SearchGroup == SearchGroupBidding { // 搜索分组处理  招标采购公告 超前项目
-
-		} else if scd.SearchGroup == SearchGroupLeadingProject {
-
-		} else {
 		}
 	}
 	if scd.Subtype != "" {
@@ -1054,7 +1048,8 @@ func doSearchByBatch(query, dataType string, searchCount int, flag string) (res
 		searchWaitGroup.Wait()
 		log.Printf("%s-分批次加载数据总量为%d\n", flag, len(res))
 	} else {
-		queryCount := qutil.If(searchCount == -1, onceSearchCount, searchCount).(int)
+		//queryCount := qutil.If(searchCount == -1, onceSearchCount, searchCount).(int)
+		queryCount := qutil.If(searchCount == -1, PreviewSearchCount, searchCount).(int)
 		searchPool <- true
 		tmp := doSearch(query, 0, queryCount, dataType)
 		<-searchPool
@@ -1074,11 +1069,83 @@ func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdo
 	if len(encry) > 0 {
 		isEncry = true
 	}
+	if data == nil {
+		return data
+	}
 	sort.Slice(*data, func(i, j int) bool {
 		time1 := qutil.Int64All((*data)[i]["publishtime"])
 		time2 := qutil.Int64All((*data)[j]["publishtime"])
 		return time1 > time2
 	})
+	qyxyMap := make(map[string]bool)
+	qyxyEsMap := make(map[string]*map[string]interface{})
+	projectsetMap := make(map[string]bool)
+	projectsetEsMap := make(map[string]*map[string]interface{})
+	for _, m := range *data {
+		if dataType == "2" {
+			entidlist, ok := m["entidlist"].([]interface{})
+			if ok && len(entidlist) > 0 {
+				//var winnerMaps []map[string]interface{}
+				for _, entIdObj := range entidlist {
+					entId := qutil.ObjToString(entIdObj)
+					if entId == "" {
+						continue
+					}
+					qyxyMap[entId] = true
+				}
+			}
+		}
+		if m["toptype"] == "结果" && dataType == "2" && !(m["agency"] != nil && m["budget"] != nil && m["buyerperson"] != nil && m["buyertel"] != nil) {
+			projectsetMap[qutil.InterfaceToStr(m["_id"])] = true
+		}
+	}
+
+	if len(qyxyMap) > 0 { //查询企业
+		var (
+			qyxyArr []string
+			qCount  int
+		)
+		for s := range qyxyMap {
+			qCount++
+			qyxyArr = append(qyxyArr, fmt.Sprintf(`%s`, s))
+			if len(qyxyArr) == 100 || (qCount == len(qyxyMap) && len(qyxyArr) > 0) {
+				qyxyEsData := elastic.Get("qyxy", "qyxy", fmt.Sprintf(`{"query":{"bool":{"should":[{"terms":{"id":["%s"]}}]}},"size":%d,"_source":["company_name","company_email","company_phone","legal_person","id"]}`, strings.Join(qyxyArr, `","`), len(qyxyArr)))
+				if qyxyEsData != nil && len(*qyxyEsData) > 0 {
+					for _, m := range *qyxyEsData {
+						qyxyEsMap[qutil.InterfaceToStr(m["id"])] = &m
+					}
+				}
+				qyxyArr = []string{}
+			}
+		}
+	}
+	if len(projectsetMap) > 0 { //查询项目
+		var (
+			projectsetArr []string
+			pCount        int
+		)
+		for s := range projectsetMap {
+			pCount++
+			projectsetArr = append(projectsetArr, fmt.Sprintf(`%s`, s))
+			if len(projectsetArr) == 100 || (pCount == len(projectsetMap) && len(projectsetArr) > 0) {
+				projectsetEsData := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query": {"bool": {"should": [{"terms": {"list.infoid": ["%s"]}}]}},"_source": ["list"],"size": %d}`, strings.Join(projectsetArr, `","`), len(projectsetArr)*3)) //查询双倍数量 避免缺失数据
+				if projectsetEsData != nil && len(*projectsetEsData) > 0 {
+					for _, m := range *projectsetEsData {
+						MsgList := m["list"]
+						if MsgList != nil {
+							list := qutil.ObjArrToMapArr(MsgList.([]interface{}))
+							for _, m2 := range list {
+								if projectsetMap[qutil.InterfaceToStr(m2["infoid"])] {
+									projectsetEsMap[qutil.InterfaceToStr(m2["infoid"])] = &m
+								}
+							}
+						}
+					}
+				}
+				projectsetArr = []string{}
+			}
+		}
+	}
 	var entCacheMap = map[string]map[string]interface{}{}
 	pool := make(chan bool, 5)
 	wait := &sync.WaitGroup{}
@@ -1108,11 +1175,12 @@ func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdo
 						lock.Unlock()
 						if ecm != nil {
 							winnerMaps = append(winnerMaps, ecm)
-						} else if entDetail := elastic.Get("qyxy", "qyxy", fmt.Sprintf(`{"query":{"bool":{"must":[{"term":{"id":"%s"}}]}},"size":1,"_source":["company_name","company_email","company_phone","legal_person"]}`, entId)); entDetail != nil && len(*entDetail) > 0 {
+							//} else if entDetail := elastic.Get("qyxy", "qyxy", fmt.Sprintf(`{"query":{"bool":{"must":[{"term":{"id":"%s"}}]}},"size":1,"_source":["company_name","company_email","company_phone","legal_person"]}`, entId)); entDetail != nil && len(*entDetail) > 0 {
+						} else if entDetail := qyxyEsMap[entId]; entDetail != nil && len(*entDetail) > 0 {
 							thisEntMap := map[string]interface{}{}
 							legal_person := ""
-							if (*entDetail)[0]["legal_person"] != nil {
-								legal_person = (*entDetail)[0]["legal_person"].(string)
+							if (*entDetail)["legal_person"] != nil {
+								legal_person = (*entDetail)["legal_person"].(string)
 								if isEncry {
 									var xx = "*"
 									switch len([]rune(legal_person)) {
@@ -1125,8 +1193,8 @@ func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdo
 								}
 							}
 							company_phone := ""
-							if (*entDetail)[0]["company_phone"] != nil {
-								company_phone = (*entDetail)[0]["company_phone"].(string)
+							if (*entDetail)["company_phone"] != nil {
+								company_phone = (*entDetail)["company_phone"].(string)
 								if isEncry {
 									if len([]rune(company_phone)) > 7 {
 										company_phone = company_phone[:7] + "****"
@@ -1136,8 +1204,8 @@ func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdo
 								}
 							}
 							company_email := ""
-							if (*entDetail)[0]["company_email"] != nil && (*entDetail)[0]["company_email"] != "无" {
-								company_email = (*entDetail)[0]["company_email"].(string)
+							if (*entDetail)["company_email"] != nil && (*entDetail)["company_email"] != "无" {
+								company_email = (*entDetail)["company_email"].(string)
 								if isEncry {
 									if len(strings.Split(company_email, "@")) > 1 {
 										company_email = "******" + "@" + strings.Split(company_email, "@")[1]
@@ -1145,8 +1213,8 @@ func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdo
 								}
 							}
 							company_name := ""
-							if (*entDetail)[0]["company_name"] != nil {
-								company_name = (*entDetail)[0]["company_name"].(string)
+							if (*entDetail)["company_name"] != nil {
+								company_name = (*entDetail)["company_name"].(string)
 							}
 							thisEntMap["legal_person"] = legal_person
 							thisEntMap["company_phone"] = company_phone
@@ -1165,31 +1233,32 @@ func FormatExportData(entmg mg.MongodbSim, data *[]map[string]interface{}, webdo
 				delete(v, "entidlist")
 			}
 			//====================字段补漏=========================
-			if v["toptype"] == "结果" && dataType == "2" && !(v["agency"] != nil && v["budget"] != nil && v["buyerperson"] != nil && v["buyertel"] != nil) {
-				r := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query":{"term":{"list.infoid":"%s"}},"_source": ["list"]}`, v["_id"]))
-				if r != nil && len(*r) > 0 {
-					MsgList := (*r)[0]["list"]
-					if MsgList != nil {
-						list := qutil.ObjArrToMapArr(MsgList.([]interface{}))
-						for _, vv := range list {
-							if vv["subtype"] == "招标" {
-								if v["agency"] == nil && vv["agency"] != nil {
-									v["agency"] = vv["agency"]
-								}
-								if v["budget"] == nil && vv["budget"] != nil {
-									v["budget"] = vv["budget"]
-								}
-								if v["buyerperson"] == nil && vv["buyerperson"] != nil {
-									v["buyerperson"] = vv["buyerperson"]
-								}
-								if v["buyertel"] == nil && vv["buyertel"] != nil {
-									v["buyertel"] = vv["buyertel"]
-								}
-								break
+			//if v["toptype"] == "结果" && dataType == "2" && !(v["agency"] != nil && v["budget"] != nil && v["buyerperson"] != nil && v["buyertel"] != nil) {
+			if r := projectsetEsMap[qutil.InterfaceToStr(v["_id"])]; r != nil && len(*r) > 0 {
+				//r := elastic.Get("projectset", "projectset", fmt.Sprintf(`{"query":{"term":{"list.infoid":"%s"}},"_source": ["list"]}`, v["_id"]))
+				//if r != nil && len(*r) > 0 {
+				MsgList := (*r)["list"]
+				if MsgList != nil {
+					list := qutil.ObjArrToMapArr(MsgList.([]interface{}))
+					for _, vv := range list {
+						if vv["subtype"] == "招标" {
+							if v["agency"] == nil && vv["agency"] != nil {
+								v["agency"] = vv["agency"]
+							}
+							if v["budget"] == nil && vv["budget"] != nil {
+								v["budget"] = vv["budget"]
+							}
+							if v["buyerperson"] == nil && vv["buyerperson"] != nil {
+								v["buyerperson"] = vv["buyerperson"]
+							}
+							if v["buyertel"] == nil && vv["buyertel"] != nil {
+								v["buyertel"] = vv["buyertel"]
 							}
+							break
 						}
 					}
 				}
+				//}
 			}
 			if v["area"] == "A" {
 				v["area"] = "全国"