调整方法...

data_mgo_to_tidb/bidding.go

@@ -3,14 +3,12 @@ package main
 import (
 	"data_tidb/config"
 	"fmt"
-	"github.com/shopspring/decimal"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"reflect"
 	"regexp"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -45,12 +43,12 @@ func TaskBidding() {
 				wg.Done()
 			}()
 			if util.IntAll(tmp["extracttype"]) != -1 {
-				taskBase(tmp)    //基础标讯数据
-				taskExpand(tmp)  //扩展数据
-				taskDetail(tmp)  //正文信息
-				taskAtts(tmp)    //附件信息
-				taskIntent(tmp)  //采购意向
-				taskPackage(tmp) //分包
+				taskBase(tmp)   //基础标讯数据
+				taskExpand(tmp) //扩展数据
+				//taskDetail(tmp)  //正文信息
+				//taskAtts(tmp)    //附件信息
+				//taskIntent(tmp)  //采购意向
+				//taskPackage(tmp) //分包
 			}
 		}(tmp)
 		tmp = make(map[string]interface{})
@@ -59,78 +57,54 @@ func TaskBidding() {
 	log.Info(fmt.Sprintf("is over --- %d", count))
 }
 
-// @Description 基本信息
 func taskBase(tmp map[string]interface{}) {
-	saveM := make(map[string]interface{})
-	for _, nf := range BaseField {
-		f := nf[2:]
-		if f == "info_id" {
-			saveM[nf] = mongodb.BsonIdToSId(tmp["_id"])
-		} else if f == "area_code" {
-			if tmp["area"] != nil {
-				saveM[nf] = AreaCode[util.ObjToString(tmp["area"])]
-			}
-		} else if f == "city_code" {
-			if tmp["area"] != nil && tmp["city"] != nil {
-				c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"])
-				saveM[nf] = AreaCode[c]
-			}
-		} else if f == "district_code" {
-			if tmp["area"] != nil && tmp["city"] != nil && tmp["district"] != nil {
-				c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"]) + "," + util.ObjToString(tmp["district"])
-				saveM[nf] = AreaCode[c]
-			}
-		} else if f == "toptype_code" {
-			if obj := util.ObjToString(tmp["toptype"]); obj != "" {
-				saveM[nf] = TopTypeCode[obj]
-			}
-		} else if f == "subtype_code" {
-			if obj := util.ObjToString(tmp["subtype"]); obj != "" {
-				saveM[nf] = SubTypeCode[obj]
-			}
-		} else if f == "buyerclass_code" {
-			if obj := util.ObjToString(tmp["buyerclass"]); obj != "" {
-				saveM[nf] = BuyerCode[obj]
-			}
-		} else if f == "createtime" || f == "updatetime" {
-			saveM[nf] = time.Now().Format(util.Date_Full_Layout)
-		} else if f == "comeintime" || f == "publishtime" || f == "bidopentime" || f == "bidendtime" {
-			if tmp[f] != nil && util.IntAll(tmp[f]) > 0 {
-				t := util.Int64All(tmp[f])
-				saveM[nf] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
-			}
-		} else if f == "multipackage" || f == "isValidFile" {
-			if tmp[f] == nil {
-				saveM[nf] = 0
-			} else {
-				saveM[nf] = tmp[f]
-			}
-		} else if f == "buyer_id" {
-			if b := util.ObjToString(tmp["buyer"]); b != "" {
-				if code := getNameId(b); code != "" {
-					saveM[nf] = code
-				}
-			}
-		} else if f == "agency_id" {
-			if b := util.ObjToString(tmp["agency"]); b != "" {
-				if code := getNameId(b); code != "" {
-					saveM[nf] = code
-				}
-			}
-		} else {
-			if tmp[f] != nil {
-				if BaseVMap[f] != nil {
-					saveM[nf], _ = verifyF(f, tmp[f], BaseVMap[f])
-				} else {
-					saveM[nf] = util.ObjToString(tmp[f])
-				}
+	tmpid := mongodb.BsonIdToSId(tmp["_id"])
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	area, city, district := util.ObjToString(tmp["area"]), util.ObjToString(tmp["city"]), util.ObjToString(tmp["district"])
+	if area != "" {
+		info["area_code"] = AreaCode[area]
+		if city != "" {
+			info["city_code"] = AreaCode[area+","+city]
+			if district != "" {
+				info["district_code"] = AreaCode[area+","+city+","+district]
 			}
 		}
 	}
-	InsertGlobalMysqlData("dwd_f_bid_baseinfo", saveM, mongodb.BsonIdToSId(tmp["_id"]))
+	if toptype := util.ObjToString(tmp["toptype"]); toptype != "" {
+		info["s_toptype_code"] = TopTypeCode["toptype"]
+	}
+	if subtype := util.ObjToString(tmp["subtype"]); subtype != "" {
+		info["s_subtype_code"] = SubTypeCode[subtype]
+	}
+	if buyerclass := util.ObjToString(tmp["buyerclass"]); buyerclass != "" {
+		info["s_buyerclass_code"] = BuyerCode[buyerclass]
+	}
+	//特别结构
+	info["i_isValidFile"] = util.IntAll(tmp["isValidFile"])
+	info["i_multipackage"] = util.IntAll(tmp["multipackage"])
+	//文本相关
+	transferTextInfo(tmp, &info, []string{"title", "projectname", "projectcode", "purchasing", "site", "href"}, []int{})
+	//金额相关
+	transferMoneyRateInfo(tmp, &info, []string{"budget", "bidamount", "biddiscount"}, []float64{1000000000.0, 1000000000.0, 100.0})
+	//时间相关
+	transferDateTimeInfo(tmp, &info, []string{"comeintime", "publishtime", "bidopentime", "bidendtime"})
+	//主体相关
+	if code := getNameId(util.ObjToString(tmp["buyer"])); code != "" {
+		info["s_buyer_id"] = code
+	}
+	if code := getNameId(util.ObjToString(tmp["agency"])); code != "" {
+		info["s_agency_id"] = code
+	}
+	info["d_updatetime"] = time.Now().Format(util.Date_Full_Layout)
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	InsertGlobalMysqlData("dwd_f_bid_baseinfo", info, mongodb.BsonIdToSId(tmp["_id"]))
 }
 
 func getNameId(name string) string {
+	if name == "" {
+		return ""
+	}
 	info := MysqlTool.FindOne("dws_f_ent_baseinfo", map[string]interface{}{"name": name}, "name_id", "")
 	if info != nil && (*info)["name_id"] != nil {
 		return util.ObjToString((*info)["name_id"])
@@ -139,75 +113,46 @@ func getNameId(name string) string {
 	}
 }
 
-// @Description 扩展信息
 func taskExpand(tmp map[string]interface{}) {
-	saveM := make(map[string]interface{})
-	for _, nf := range ExpandField {
-		f := nf[2:]
-		if f == "info_id" {
-			saveM[nf] = mongodb.BsonIdToSId(tmp["_id"])
-		} else if f == "project_startdate" || f == "project_completedate" || f == "signstarttime" || f == "bidendtime" || f == "bidstarttime" || f == "docstarttime" ||
-			f == "docendtime" || f == "signaturedate" || f == "signendtime" {
-			if tmp[f] != nil && util.IntAll(tmp[f]) > 0 {
-				t := util.Int64All(tmp[f])
-				saveM[nf] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
-			}
-		} else if f == "createtime" || f == "updatetime" {
-			saveM[nf] = time.Now().Format(util.Date_Full_Layout)
-		} else if f == "bidway" {
-			if util.ObjToString(tmp[f]) == "电子投标" {
-				saveM[nf] = 1
-			} else if util.ObjToString(tmp[f]) == "纸质投标" {
-				saveM[nf] = 0
-			}
-		} else if f == "review_experts" { //评标专家
-			if tmp[f] != nil {
-				if reflect.TypeOf(tmp[f]).String() == "string" {
-					saveM[nf] = tmp[f]
-				} else if reflect.TypeOf(tmp[f]).String() == "[]interface {}" {
-					if arr, ok := tmp[f].([]interface{}); ok {
-						saveM[nf] = strings.Join(util.ObjArrToStringArr(arr), ",")
-					}
-				}
-			}
-		} else if f == "bid_guarantee" || f == "contract_guarantee" {
-			if tmp[f] != nil {
-				if tmp[f].(bool) {
-					saveM[nf] = 1
-				} else {
-					saveM[nf] = 0
-				}
-			}
-		} else if f == "agencyfee" {
-			if tmp[f] != nil {
-				if reflect.TypeOf(tmp[f]).String() == "string" {
-					v2, err := strconv.ParseFloat(strings.ReplaceAll(util.ObjToString(tmp[f]), "%", ""), 64)
-					if err != nil {
-						v, _ := decimal.NewFromFloat(v2).Div(decimal.NewFromFloat(float64(100))).Float64()
-						saveM[nf] = v
-					}
-				} else {
-					saveM[nf], _ = util.FormatFloat(util.Float64All(tmp[f]), 4)
-				}
-			}
-		} else if f == "project_duration" {
-			if tmp[f] != nil {
-				saveM[nf] = util.IntAll(tmp[f])
-			}
-		} else {
-			if tmp[f] != nil {
-				if ExpandVMap[f] != nil {
-					saveM[nf], _ = verifyF(f, tmp[f], ExpandVMap[f])
-				} else {
-					saveM[nf] = util.ObjToString(tmp[f])
-				}
+	tmpid := mongodb.BsonIdToSId(tmp["_id"])
+	info := map[string]interface{}{}
+	info["s_info_id"] = tmpid
+	//文本相关
+	transferTextInfo(tmp, &info, []string{"projectperiod", "project_scale", "project_timeunit", "bidmethod", "getdocmethod", "currency", "funds", "payway", "bid_bond", "contract_bond", "contractcode", "buyerzipcode", "bidopenaddress", "buyeraddr", "agencyaddr", "projectaddr", "enterprise_qualification", "personnel_qualification", "performance_qualification", "enterprise_credit"}, []int{1000, 5000, 50, 255, 500, 20, 5000, 500, 1000, 500, 100, 100, 1000, 1000, 1000, 1000, 20000, 20000, 20000, 20000})
+	//时间相关
+	transferDateTimeInfo(tmp, &info, []string{"project_startdate", "project_completedate", "signstarttime", "bidendtime", "bidstarttime", "docstarttime", "docendtime", "signaturedate", "signendtime"})
+	//布尔相关
+	transferBoolInfo(tmp, &info, []string{"bid_guarantee", "contract_guarantee"})
+	//金额相关
+	transferMoneyRateInfo(tmp, &info, []string{"docamount", "agencyfee", "agencyrate"}, []float64{10000000.0, 10000000.0, 10000000.0})
+	//投标方式
+	bidway := util.ObjToString(tmp["bidway"])
+	if bidway == "电子投标" {
+		info["i_bidway"] = 1
+	} else if bidway == "纸质投标" {
+		info["i_bidway"] = 0
+	} else {
+
+	}
+	//评审专家
+	if tmp["review_experts"] != nil {
+		if reflect.TypeOf(tmp["review_experts"]).String() == "string" {
+			info["s_review_experts"] = tmp["review_experts"]
+		} else if reflect.TypeOf(tmp["review_experts"]).String() == "[]interface {}" {
+			if arr, ok := tmp["review_experts"].([]interface{}); ok {
+				info["s_review_experts"] = strings.Join(util.ObjArrToStringArr(arr), ",")
 			}
 		}
 	}
-	InsertGlobalMysqlData("dwd_f_bid_expand_baseinfo", saveM, mongodb.BsonIdToSId(tmp["_id"]))
+	//工期时长
+	if tmp["project_duration"] != nil {
+		info["i_project_duration"] = util.IntAll(tmp["project_duration"])
+	}
+	info["d_updatetime"] = time.Now().Format(util.Date_Full_Layout)
+	info["d_createtime"] = time.Now().Format(util.Date_Full_Layout)
+	InsertGlobalMysqlData("dwd_f_bid_expand_baseinfo", info, tmpid)
 }
 
-// @Description 正文信息
 func taskDetail(tmp map[string]interface{}) {
 	tmpid := mongodb.BsonIdToSId(tmp["_id"])
 	s_detail := util.ObjToString(tmp["detail"])
@@ -222,7 +167,6 @@ func taskDetail(tmp map[string]interface{}) {
 	InsertGlobalMysqlData("dwd_f_bid_detail", info, tmpid)
 }
 
-// @Description 附件
 func taskAtts(tmp map[string]interface{}) {
 	tmpid := mongodb.BsonIdToSId(tmp["_id"])
 	f_baseInfo := map[string]interface{}{}
@@ -280,7 +224,6 @@ func taskAttsAttach(att_info map[string]interface{}, tmpid string, f_id int64) {
 	}
 }
 
-// @Description 采购意向
 func taskIntent(tmp map[string]interface{}) {
 	procurementlist := IsMarkInterfaceMap(tmp["procurementlist"])
 	tmpid := mongodb.BsonIdToSId(tmp["_id"])
@@ -315,7 +258,6 @@ func taskIntent(tmp map[string]interface{}) {
 	}
 }
 
-// @Description 分包基本信息
 func taskPackage(tmp map[string]interface{}) {
 	tmpid := mongodb.BsonIdToSId(tmp["_id"])
 	//筛选分包

data_mgo_to_tidb/field-criteria.json

@@ -1,268 +0,0 @@
-{
-  "dws_f_bid_baseinfo": {
-    "field_array": [
-      "s_info_id",
-      "s_area_code",
-      "s_city_code",
-      "s_district_code",
-      "f_budget",
-      "f_bidamount",
-      "f_biddiscount",
-      "s_title",
-      "s_toptype_code",
-      "s_subtype_code",
-      "s_projectname",
-      "s_projectcode",
-      "s_buyerclass_code",
-      "d_publishtime",
-      "d_comeintime",
-      "d_bidopentime",
-      "d_bidendtime",
-      "i_isValidFile",
-      "i_multipackage",
-      "s_purchasing",
-      "s_site",
-      "s_href",
-      "d_updatetime",
-      "d_createtime",
-      "s_buyer_id",
-      "s_agency_id"
-    ],
-    "field_criteria": {
-      "budget": {
-        "stype": "float",
-        "min": 0,
-        "max": 1000000000,
-        "decimal": 4
-      },
-      "bidamount": {
-        "stype": "float",
-        "min": 0,
-        "max": 1000000000,
-        "decimal": 4
-      },
-      "biddiscount": {
-        "stype": "float",
-        "min": 0,
-        "max": 100,
-        "decimal": 4
-      },
-      "title": {
-        "stype": "string",
-        "length": 500,
-        "intercept": true
-      },
-      "projectname": {
-        "stype": "string",
-        "length": 500,
-        "intercept": true
-      },
-      "projectcode": {
-        "stype": "string",
-        "length": 100,
-        "intercept": true
-      },
-      "site": {
-        "stype": "string",
-        "length": 100,
-        "intercept": false
-      }
-    }
-  },
-  "dws_f_bid_expand_baseinfo": {
-    "field_array": [
-      "s_info_id",
-      "s_projectperiod",
-      "s_project_scale",
-      "i_project_duration",
-      "s_project_timeunit",
-      "d_project_startdate",
-      "d_project_completedate",
-      "d_signstarttime",
-      "d_signendtime",
-      "d_docstarttime",
-      "d_docendtime",
-      "d_bidendtime",
-      "d_bidstarttime",
-      "d_signaturedate",
-      "s_review_experts",
-      "s_bidmethod",
-      "s_getdocmethod",
-      "i_bidway",
-      "f_docamount",
-      "f_agencyrate",
-      "f_agencyfee",
-      "s_currency",
-      "s_funds",
-      "s_payway",
-      "s_bid_bond",
-      "i_bid_guarantee",
-      "s_contract_bond",
-      "i_contract_guarantee",
-      "s_contractcode",
-      "s_buyerzipcode"   ,
-      "s_bidopenaddress",
-      "s_buyeraddr",
-      "s_agencyaddr",
-      "s_projectaddr",
-      "s_enterprise_qualification",
-      "s_personnel_qualification",
-      "s_performance_qualification",
-      "s_enterprise_credit",
-      "d_updatetime",
-      "d_createtime"
-    ],
-    "field_criteria": {
-      "projectperiod": {
-        "stype": "string",
-        "length": 1000,
-        "intercept": true
-      },
-      "project_timeunit": {
-        "stype": "string",
-        "length": 50,
-        "intercept": true
-      },
-      "funds": {
-        "stype": "string",
-        "length": 5000,
-        "intercept": true
-      },
-      "currency": {
-        "stype": "string",
-        "length": 20,
-        "intercept": true
-      },
-      "payway": {
-        "stype": "string",
-        "length": 500,
-        "intercept": true
-      },
-      "bidmethod": {
-        "stype": "string",
-        "length": 255,
-        "intercept": true
-      },
-      "bid_bond": {
-        "stype": "string",
-        "length": 1000,
-        "intercept": true
-      },
-      "contract_bond": {
-        "stype": "string",
-        "length": 500,
-        "intercept": true
-      },
-      "bidopenaddress": {
-        "stype": "string",
-        "length": 1000,
-        "intercept": true
-      },
-      "contractcode": {
-        "stype": "string",
-        "length": 100,
-        "intercept": false
-      },
-      "getdocmethod": {
-        "stype": "string",
-        "length": 500,
-        "intercept": true
-      },
-      "projectaddr": {
-        "stype": "string",
-        "length": 1000,
-        "intercept": true
-      },
-      "project_scale": {
-        "stype": "string",
-        "length": 5000,
-        "intercept": true
-      },
-      "enterprise_qualification": {
-        "stype": "string",
-        "length": 20000,
-        "intercept": true
-      },
-      "personnel_qualification": {
-        "stype": "string",
-        "length": 20000,
-        "intercept": true
-      },
-      "performance_qualification": {
-        "stype": "string",
-        "length": 20000,
-        "intercept": true
-      },
-      "enterprise_credit": {
-        "stype": "string",
-        "length": 20000,
-        "intercept": true
-      },
-      "docamount": {
-        "stype": "float",
-        "min": 0,
-        "max": 99999999,
-        "decimal": 2
-      },
-      "agencyrate": {
-        "stype": "float",
-        "min": 0,
-        "max": 999999,
-        "decimal": 4
-      },
-      "agencyfee": {
-        "stype": "float",
-        "min": 0,
-        "max": 99999999,
-        "decimal": 2
-      },
-      "supervisorrate": {
-        "stype": "float",
-        "min": 0,
-        "max": 999999,
-        "decimal": 4
-      }
-    }
-  },
-  "dws_f_project_baseinfo": {
-    "field_array": ["projectid", "projectcode", "projectname", "area_code", "city_code", "district_code", "budget", "bidamount", "bidstatus", "bidtype",
-      "bidopentime", "createtime", "firsttime", "zbtime", "jgtime", "lasttime", "buyer_id", "agency_id", "updatetime"],
-    "field_criteria": {
-      "budget": {
-        "stype": "float",
-        "min": 0,
-        "max": 100000000000,
-        "decimal": 4
-      },
-      "bidamount": {
-        "stype": "float",
-        "min": 0,
-        "max": 100000000000,
-        "decimal": 4
-      },
-      "projectname": {
-        "stype": "string",
-        "length": 500,
-        "intercept": true
-      }
-    }
-  },
-  "dws_f_project_business": {
-    "field_array": ["projectid", "buyer_id", "agency_id", "winner_id", "area_code", "city_code", "district_code", "budget", "bidamount", "bidstatus", "bidtype",
-      "buyerclass_code", "firsttime", "zbtime", "jgtime", "lasttime", "bidopentime"],
-    "field_criteria": {
-      "budget": {
-        "stype": "float",
-        "min": 0,
-        "max": 100000000000,
-        "decimal": 4
-      },
-      "bidamount": {
-        "stype": "float",
-        "min": 0,
-        "max": 100000000000,
-        "decimal": 4
-      }
-    }
-  }
-}

data_mgo_to_tidb/init.go

@@ -21,19 +21,7 @@ var ( //通用
 	BuyerCode      = make(map[string]string, 100)
 	TopScopeCode   = make(map[string]interface{}, 20)
 	SubScopeCode   = make(map[string]interface{}, 70)
-)
-var ( //标讯
-	BaseField   []string
-	BaseVMap    map[string]interface{}
-	ExpandField []string
-	ExpandVMap  map[string]interface{}
-)
-var ( //项目
-	ProField    []string
-	ProVMap     map[string]interface{}
-	ProBusField []string
-	ProBusVMap  map[string]interface{}
-	FileTypeArr = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"}
+	FileTypeArr    = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"}
 )
 
 func InitInfo() {
@@ -136,42 +124,6 @@ func InitField() {
 		}
 	}
 	log.Info("InitField", zap.Int("TopScopeCode", len(TopScopeCode)), zap.Int("SubScopeCode", len(SubScopeCode)))
-
-	util.ReadConfig("./field-criteria.json", &FCriteria)
-	if len(FCriteria) > 0 {
-		if m, o := FCriteria["dws_f_bid_baseinfo"].(map[string]interface{}); o {
-			BaseField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
-			BaseVMap = m["field_criteria"].(map[string]interface{})
-		} else {
-			log.Error("InitField", zap.String("field_array", "dws_f_bid_baseinfo"))
-			panic("dws_f_bid_baseinfo")
-		}
-		if m, o := FCriteria["dws_f_bid_expand_baseinfo"].(map[string]interface{}); o {
-			ExpandField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
-			ExpandVMap = m["field_criteria"].(map[string]interface{})
-		} else {
-			log.Error("InitField", zap.String("field_array", "dws_f_bid_expand_baseinfo"))
-			panic("dws_f_bid_expand_baseinfo")
-		}
-		//项目相关-字段限制暂时注释
-		//if m, o := FCriteria["dws_f_project_baseinfo"].(map[string]interface{}); o {
-		//	ProField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
-		//	ProVMap = m["field_criteria"].(map[string]interface{})
-		//} else {
-		//	log.Error("InitField", zap.String("field_array", "dws_f_project_baseinfo"))
-		//	panic("dws_f_project_baseinfo")
-		//}
-		//if m, o := FCriteria["dws_f_project_business"].(map[string]interface{}); o {
-		//	ProBusField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
-		//	ProBusVMap = m["field_criteria"].(map[string]interface{})
-		//} else {
-		//	log.Error("InitField", zap.String("field_array", "dws_f_project_business"))
-		//	panic("dws_f_project_business")
-		//}
-	} else {
-		log.Error("InitField, 未找到field-criteria.json文件")
-		panic("InitField, 未找到field-criteria.json文件")
-	}
 }
 
 // 插入并打印信息

data_mgo_to_tidb/pkg.go

@@ -4,7 +4,6 @@ import (
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"strings"
 	"time"
-	"unicode/utf8"
 )
 
 // 标讯分包基本信息-标讯版
@@ -68,9 +67,9 @@ func CPBiddingPackageGoodsBaseInfo(tmp map[string]interface{}, tmpid string, pid
 		info := map[string]interface{}{}
 		info["s_info_id"] = tmpid
 		info["i_package_id"] = pid
-		info["s_itemname"] = cleanLengthName(util.ObjToString(v["itemname"]), 255)
-		info["s_brand"] = cleanLengthName(util.ObjToString(v["brandname"]), 255)
-		info["s_model"] = cleanLengthName(util.ObjToString(v["model"]), 255)
+		info["s_itemname"] = cleanLength(util.ObjToString(v["itemname"]), 255)
+		info["s_brand"] = cleanLength(util.ObjToString(v["brandname"]), 255)
+		info["s_model"] = cleanLength(util.ObjToString(v["model"]), 255)
 		if number := util.IntAll(v["number"]); number > 0 && number < 1000000000 {
 			info["i_num"] = number
 		}
@@ -213,12 +212,3 @@ func isExistsWins(winArr []string, win string) bool {
 func cleaname(name string) string {
 	return name
 }
-
-// 通用长度清洗
-func cleanLengthName(name string, l int) string {
-	new_name := name
-	if utf8.RuneCountInString(name) > l {
-		new_name = string(([]rune(new_name))[:l])
-	}
-	return new_name
-}

data_mgo_to_tidb/util.go

@@ -3,57 +3,75 @@ package main
 import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
-	"reflect"
-	"strings"
 	"unicode/utf8"
 )
 
-// @Description 字段验证、处理
-// @Author J 2022/10/8 17:16
-func verifyF(f string, v interface{}, tmp1 interface{}) (interface{}, bool) {
-	tmp := tmp1.(map[string]interface{})
-	stype := util.ObjToString(tmp["stype"])
-	vt := reflect.TypeOf(v).String()
-	if strings.Contains(vt, stype) {
-		switch stype {
-		case "string":
-			v1 := util.ObjToString(v)
-			lens := util.IntAll(tmp["length"])
-			if tmp["intercept"].(bool) {
-				if utf8.RuneCountInString(v1) > lens {
-					return string(([]rune(v1))[:lens]), true
-				} else {
-					return v1, false
-				}
+// 转换时间
+func transferDateTimeInfo(tmp map[string]interface{}, info *map[string]interface{}, keys []string) {
+	for _, v := range keys {
+		if t := util.Int64All(tmp[v]); t > 0 {
+			key := "d_" + v
+			(*info)[key] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
+		}
+	}
+}
+
+// 转换文本
+func transferTextInfo(tmp map[string]interface{}, info *map[string]interface{}, keys []string, maxs []int) {
+	for k, v := range keys {
+		if value := util.ObjToString(tmp[v]); value != "" {
+			key := "s_" + v
+			if len(maxs) > k {
+				(*info)[key] = cleanLength(value, maxs[k])
 			} else {
-				if utf8.RuneCountInString(v1) > lens {
-					return "", true
-				} else {
-					return v1, false
-				}
-			}
-		case "float":
-			v1 := util.Float64All(v)
-			min := util.Float64All(tmp["mix"])
-			max := util.Float64All(tmp["max"])
-			if min != 0 && v1 < min {
-				return nil, true
-			}
-			if max != 0 && v1 >= max {
-				return nil, true
+				(*info)[key] = value
 			}
-			dm := util.IntAll(tmp["decimal"]) //小数点位数
-			v2, err := util.FormatFloat(v1, dm)
-			if err != nil {
-				return nil, true
+		}
+	}
+}
+
+// 转换布尔
+func transferBoolInfo(tmp map[string]interface{}, info *map[string]interface{}, keys []string) {
+	for _, v := range keys {
+		if tmp[v] != nil {
+			key := "i_" + v
+			if tmp[v].(bool) {
+				(*info)[key] = 1
+			} else {
+				(*info)[key] = 0
 			}
-			return v2, false
+		}
+	}
+}
 
-		default:
-			break
+// 转换金额费用
+func transferMoneyRateInfo(tmp map[string]interface{}, info *map[string]interface{}, keys []string, maxs []float64) {
+	for k, v := range keys {
+		key := "f_" + v
+		if value, b := cleanFloat64(util.Float64All(tmp[v]), maxs[k]); tmp[v] != nil && b {
+			(*info)[key] = value
 		}
 	}
-	return nil, true
+}
+
+// 通用长度清洗
+func cleanLength(name string, l int) string {
+	if l == -1 {
+		return name
+	}
+	new_name := name
+	if utf8.RuneCountInString(name) > l {
+		new_name = string(([]rune(new_name))[:l])
+	}
+	return new_name
+}
+
+// 通用浮点清洗
+func cleanFloat64(value float64, max float64) (float64, bool) {
+	if value <= max {
+		return value, true
+	}
+	return float64(0), false
 }
 
 func IsMarkInterfaceArr(t interface{}) []string {