project es xg

createEsIndex/bidding_es.go

@@ -34,7 +34,7 @@ func biddingTask(mapInfo map[string]interface{}) {
 	if stype == "bidding" {
 		uq := bson.M{"gtid": bson.M{"$gte": util.ObjToString(mapInfo["gtid"])},
 			"lteid": bson.M{"$lte": util.ObjToString(mapInfo["lteid"])}}
-		MgoB.Update("bidding_processing_ids", uq, bson.M{"$set": bson.M{"dataprocess": 8, "updatetime": time.Now().Unix()}}, false, false)
+		MgoB.Update("bidding_processing_ids", uq, bson.M{"$set": bson.M{"dataprocess": 8, "updatetime": time.Now().Unix()}}, false, true)
 	}
 	q, _ := mapInfo["query"].(map[string]interface{})
 	if q == nil {
@@ -76,6 +76,9 @@ func biddingTask(mapInfo map[string]interface{}) {
 			if util.IntAll(tmp["dataprocess"]) != 8 {
 				return
 			}
+			//if util.IntAll(tmp["extracttype"]) != 1 {
+			//	return
+			//}
 			if stype == "bidding_history" && tmp["history_updatetime"] == nil {
 				return
 			}

createEsIndex/init.go

@@ -1,14 +1,21 @@
 package main
 
 import (
+	util "app.yhyue.com/data_processing/common_utils"
 	"app.yhyue.com/data_processing/common_utils/elastic"
 	"app.yhyue.com/data_processing/common_utils/log"
 	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"esindex/config"
 	"fmt"
+	"go.uber.org/zap"
 	"os"
 )
 
+var (
+	ProjectField = make(map[string]string, 500)
+	ProjectListF = make(map[string]string, 200)
+)
+
 // InitLog @Description
 // @Author J 2022/7/26 15:30
 func InitLog() {
@@ -69,3 +76,17 @@ func InitEs() {
 	}
 	Es1.InitElasticSize()
 }
+
+func InitField() {
+	info, _ := MgoB.Find("bidding_processing_field", `{"stype": "project"}`, nil, nil, false, -1, -1)
+	if len(*info) > 0 {
+		for _, m := range *info {
+			if util.IntAll(m["level"]) == 1 {
+				ProjectField[util.ObjToString(m["field"])] = util.ObjToString(m["ftype"])
+			} else if util.IntAll(m["level"]) == 2 {
+				ProjectListF[util.ObjToString(m["field"])] = util.ObjToString(m["ftype"])
+			}
+		}
+	}
+	log.Info("InitField", zap.Int("ProjectField", len(ProjectField)), zap.Int("ProjectListF", len(ProjectListF)))
+}

createEsIndex/main.go

@@ -54,6 +54,7 @@ func init() {
 	InitLog()
 	InitMgo()
 	InitEs()
+	InitField()
 	oss.InitOss()
 
 	JyUdpAddr = &net.UDPAddr{

createEsIndex/project_es.go

@@ -7,6 +7,7 @@ import (
 	"esindex/config"
 	"go.uber.org/zap"
 	"math"
+	"reflect"
 	"regexp"
 	"strconv"
 )
@@ -50,73 +51,127 @@ func projectTask(data []byte, mapInfo map[string]interface{}) {
 		if n%2000 == 0 {
 			log.Info("current", zap.Int("count", n))
 		}
-		pp := map[string]map[string]interface{}{}
-		if packages, ok := tmp["package"].(map[string]interface{}); ok {
-			for _, pks := range packages {
-				if pk, ok := pks.([]interface{}); ok {
-					for _, v := range pk {
-						if p, ok := v.(map[string]interface{}); ok {
-							winner := util.ObjToString(p["winner"])
-							bidamount := util.Float64All((p["bidamount"]))
-							if len(winner) > 4 && bidamount > 0 {
-								p := map[string]interface{}{
-									"winner":    winner,
-									"bidamount": bidamount,
+		newTmp := make(map[string]interface{})
+		for f, ftype := range ProjectField {
+			if tmp[f] != nil {
+				if f == "package" {
+					pp := map[string]map[string]interface{}{}
+					if packages, ok := tmp["package"].(map[string]interface{}); ok {
+						for _, pks := range packages {
+							if pk, ok := pks.([]interface{}); ok {
+								for _, v := range pk {
+									if p, ok := v.(map[string]interface{}); ok {
+										winner := util.ObjToString(p["winner"])
+										bidamount := util.Float64All((p["bidamount"]))
+										if len(winner) > 4 && bidamount > 0 {
+											p := map[string]interface{}{
+												"winner":    winner,
+												"bidamount": bidamount,
+											}
+											pp[winner] = p
+										}
+									}
 								}
-								pp[winner] = p
 							}
 						}
+					} else {
+						winner := util.ObjToString(tmp["winner"])
+						bidamount := util.Float64All(tmp["bidamount"])
+						if len(winner) > 4 && bidamount > 0 {
+							p := map[string]interface{}{
+								"winner":    winner,
+								"bidamount": bidamount,
+							}
+							pp[winner] = p
+						}
+					}
+					pk1 := []map[string]interface{}{}
+					for _, v := range pp {
+						pk1 = append(pk1, v)
+					}
+					if len(pk1) > 0 {
+						newTmp["package1"] = pk1
+					}
+				} else if f == "topscopeclass" {
+					if topscopeclass, ok := tmp["topscopeclass"].([]interface{}); ok {
+						tc := []string{}
+						m2 := map[string]bool{}
+						for _, v := range topscopeclass {
+							str := util.ObjToString(v)
+							str = regLetter.ReplaceAllString(str, "") // 去除字母
+							if !m2[str] {
+								m2[str] = true
+								tc = append(tc, str)
+							}
+						}
+						newTmp["topscopeclass"] = tc
+					}
+				} else if f == "list" {
+					if list, ok := tmp[f].([]interface{}); ok {
+						var newList []map[string]interface{}
+						for _, item := range list {
+							item1 := item.(map[string]interface{})
+							listm := make(map[string]interface{})
+							for f1, ftype1 := range ProjectListF {
+								if item1[f1] != nil {
+									if f == "topscopeclass" || f == "subscopeclass" {
+										listm[f] = item1[f1]
+									} else {
+										if fieldval := item1[f1]; reflect.TypeOf(fieldval).String() != ftype1 {
+											continue
+										} else {
+											if fieldval != "" {
+												listm[f1] = fieldval
+											}
+										}
+									}
+								}
+							}
+							newList = append(newList, listm)
+						}
+						newTmp[f] = newList
+					}
+				} else if f == "budget" || f == "bidamount" || f == "sortprice" {
+					if tmp[f] != nil && util.Float64All(f) <= 1000000000 {
+						newTmp[f] = tmp[f]
+					}
+				} else if f == "projectscope" {
+					projectscopeRune := []rune(util.ObjToString(tmp[f]))
+					if len(projectscopeRune) > 1000 {
+						newTmp[f] = util.ObjToString(tmp[f])[:1000]
+					} else {
+						newTmp[f] = tmp[f]
+					}
+				} else if f == "s_projectname" {
+					newTmp[f] = tmp["projectname"]
+				} else if f == "ids" || f == "_id" || f == "mpc" || f == "mpn" || f == "review_experts" || f == "winnerorder" ||
+					f == "entidlist" || f == "first_cooperation" {
+					newTmp[f] = tmp[f]
+				} else {
+					if fieldval := tmp[f]; reflect.TypeOf(fieldval).String() != ftype {
+						continue
+					} else {
+						if fieldval != "" {
+							newTmp[f] = fieldval
+						}
 					}
 				}
 			}
-		} else {
-			winner := util.ObjToString(tmp["winner"])
-			bidamount := util.Float64All(tmp["bidamount"])
-			if len(winner) > 4 && bidamount > 0 {
-				p := map[string]interface{}{
-					"winner":    winner,
-					"bidamount": bidamount,
-				}
-				pp[winner] = p
-			}
 		}
 
-		pk1 := []map[string]interface{}{}
-		for _, v := range pp {
-			pk1 = append(pk1, v)
-		}
-		if len(pk1) > 0 {
-			tmp["package1"] = pk1
-		}
-		budget := util.Float64All(tmp["budget"])
-		bidamount := util.Float64All(tmp["bidamount"])
+		budget := util.Float64All(newTmp["budget"])
+		bidamount := util.Float64All(newTmp["bidamount"])
 		if float64(budget) > 0 && float64(bidamount) > 0 {
 			rate := float64(1) - float64(bidamount)/float64(budget)
 			f, _ := strconv.ParseFloat(strconv.FormatFloat(rate, 'f', 4, 64), 64)
 			//不在0~0.6之间，不生成费率；只生成预算，中标金额舍弃，索引增加折扣率异常标识
 			if f < 0 || f > 0.6 {
-				delete(tmp, "bidamount")
-				tmp["prate_flag"] = 1
+				delete(newTmp, "bidamount")
+				newTmp["prate_flag"] = 1
 			} else {
-				tmp["project_rate"] = f
+				newTmp["project_rate"] = f
 			}
 		}
-		if topscopeclass, ok := tmp["topscopeclass"].([]interface{}); ok {
-			tc := []string{}
-			m2 := map[string]bool{}
-			for _, v := range topscopeclass {
-				str := util.ObjToString(v)
-				str = regLetter.ReplaceAllString(str, "") // 去除字母
-				if !m2[str] {
-					m2[str] = true
-					tc = append(tc, str)
-				}
-			}
-			tmp["topscopeclass"] = tc
-		}
-		//不生索引字段
-		delete(tmp, "package")
-		delete(tmp, "infofield")
 
 		bidopentime := util.Int64All(tmp["bidopentime"]) //开标日期
 		fzb_publishtime := int64(0)                      //记录第一个招标信息的publishtime
@@ -124,18 +179,10 @@ func projectTask(data []byte, mapInfo map[string]interface{}) {
 		list := tmp["list"].([]interface{})
 		for _, m := range list {
 			tmpM := m.(map[string]interface{})
-			//删除purchasing,review_experts
-			delete(tmpM, "purchasing")
-			delete(tmpM, "review_experts")
 			if bidamount, ok := tmpM["bidamount"].(string); ok && len(bidamount) > 0 { //bidamount为string类型，转成float
 				tmpB := util.Float64All(tmpM["bidamount"])
 				tmpM["bidamount"] = tmpB
 			}
-			//projectscope截断
-			listProjectscopeRune := []rune(util.ObjToString(tmpM["projectscope"]))
-			if len(listProjectscopeRune) > 1000 {
-				tmpM["projectscope"] = string(listProjectscopeRune[:1000])
-			}
 			//计算bidcycle标书表编制周期字段
 			if !bidcycle_flag && bidopentime > 0 { //bidopentime>0证明list中有bidopentime，无则不用计算bidcycle
 				if toptype := util.ObjToString(tmpM["toptype"]); toptype == "招标" {
@@ -163,30 +210,10 @@ func projectTask(data []byte, mapInfo map[string]interface{}) {
 			if tmpTime := bidopentime - fzb_publishtime; tmpTime > 0 {
 				f_day := float64(tmpTime) / float64(86400)
 				day := math.Ceil(f_day)
-				tmp["bidcycle"] = int(day)
+				newTmp["bidcycle"] = int(day)
 			}
 		}
-		//projectscope截断
-		projectscopeRune := []rune(util.ObjToString(tmp["projectscope"]))
-		if len(projectscopeRune) > 1000 {
-			tmp["projectscope"] = string(projectscopeRune[:1000])
-		}
-		//		if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
-		//			tmp["budget"] = nil
-		//		}
-		//		if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
-		//			tmp["bidamount"] = nil
-		//		}
-		//go IS.Add("project")
-
-		if tmp["bidamount"] != nil && util.Float64All(tmp["bidamount"]) > 1000000000 {
-			delete(tmp, "bidamount")
-		}
-		if tmp["budget"] != nil && util.Float64All(tmp["budget"]) > 1000000000 {
-			delete(tmp, "budget")
-		}
-		tmp["s_projectname"] = tmp["projectname"]
-		saveProjectEsPool <- tmp
+		saveProjectEsPool <- newTmp
 		tmp = make(map[string]interface{})
 	}
 	log.Info("create project index...over", zap.Any("mapInfo", mapInfo), zap.Int("count", n))