package main import ( "data_tidb/config" "fmt" "go.uber.org/zap" util "jygit.jydev.jianyu360.cn/data_processing/common_utils" "jygit.jydev.jianyu360.cn/data_processing/common_utils/log" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb" "os" ) var ( MongoB, MongoP *mongodb.MongodbSim MysqlTool *mysqldb.Mysql FCriteria map[string]interface{} saveSize = 200 saveBasePool = make(chan map[string]interface{}, 5000) saveBaseSp = make(chan bool, 1) saveExpandPool = make(chan map[string]interface{}, 5000) saveExpandSp = make(chan bool, 1) saveTagPool = make(chan map[string]interface{}, 5000) saveTagSp = make(chan bool, 1) saveAttrPool = make(chan map[string]interface{}, 5000) saveAttrSp = make(chan bool, 1) saveIfmPool = make(chan map[string]interface{}, 5000) saveIfmSp = make(chan bool, 1) saveIntentPool = make(chan map[string]interface{}, 5000) saveIntentSp = make(chan bool, 1) saveWinnerPool = make(chan map[string]interface{}, 5000) saveWinnerSp = make(chan bool, 1) savePkgPool = make(chan map[string]interface{}, 5000) savePkgSp = make(chan bool, 1) savePurPool = make(chan map[string]interface{}, 5000) savePurSp = make(chan bool, 1) saveProPool = make(chan map[string]interface{}, 5000) saveProSp = make(chan bool, 1) saveProbPool = make(chan map[string]interface{}, 5000) saveProbSp = make(chan bool, 1) saveProTagPool = make(chan map[string]interface{}, 5000) saveProTagSp = make(chan bool, 1) saveRelationPool = make(chan map[string]interface{}, 5000) saveRelationSp = make(chan bool, 1) saveErrPool = make(chan map[string]interface{}, 5000) saveErrSp = make(chan bool, 1) AreaCode = make(map[string]string, 5000) TopTypeCode = make(map[string]string, 10) SubTypeCode = make(map[string]string, 40) BuyerCode = make(map[string]string, 100) TopScopeCode = make(map[string]interface{}, 20) SubScopeCode = make(map[string]interface{}, 70) BaseField []string BaseVMap map[string]interface{} ExpandField []string ExpandVMap map[string]interface{} ProField []string ProVMap map[string]interface{} ProBusField []string ProBusVMap map[string]interface{} TagsField = []string{"infoid", "labelcode", "labelvalues", "labelweight", "createtime"} AttrField = []string{"infoid", "filename", "fid", "ftype", "org_url", "size", "file_type", "createtime"} FileTypeArr = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"} IfmField = []string{"infoid", "approvecode", "approvedept", "approvestatus", "approvetime", "approvenumber", "approvecontent", "projecttype", "approvecity", "updatetime", "createtime"} PurField = []string{"infoid", "item", "itemname", "brandname", "specs", "model", "unitname", "number", "unitprice", "totalprice"} IntentField = []string{"infoid", "projectname", "projectscope", "item", "buyer_id", "totalprice", "expurasingtime", "reserved_amount", "updatetime", "createtime"} WinnerField = []string{"infoid", "winner_id", "winnersort", "package_id", "createtime", "updatetime"} PackageField = []string{"infoid", "packagename", "packagename", "packageorgin", "packagedetail", "budget", "bidamount", "winner_id", "updatetime", "createtime"} ProTagsField = []string{"projectid", "labelcode", "labelvalues", "labelweight", "createtime"} RelationField = []string{"projectid", "infoid", "name_id", "contact_id", "identity_type", "createtime"} ) // InitLog @Description // @Author J 2022/7/26 15:30 func InitLog() { logcfg := config.Conf.Log err := log.InitLog( log.Path(logcfg.LogPath), log.Level(logcfg.LogLevel), log.Compress(logcfg.Compress), log.MaxSize(logcfg.MaxSize), log.MaxBackups(logcfg.MaxBackups), log.MaxAge(logcfg.MaxAge), log.Format(logcfg.Format), ) if err != nil { fmt.Printf("InitLog failed: %v\n", err) os.Exit(1) } } func InitMgo() { MongoB = &mongodb.MongodbSim{ MongodbAddr: config.Conf.DB.MongoB.Addr, DbName: config.Conf.DB.MongoB.Dbname, Size: config.Conf.DB.MongoB.Size, UserName: config.Conf.DB.MongoB.User, Password: config.Conf.DB.MongoB.Password, } MongoB.InitPool() MongoP = &mongodb.MongodbSim{ MongodbAddr: config.Conf.DB.MongoP.Addr, DbName: config.Conf.DB.MongoP.Dbname, Size: config.Conf.DB.MongoP.Size, UserName: config.Conf.DB.MongoP.User, Password: config.Conf.DB.MongoP.Password, } MongoP.InitPool() } func InitMysql() { dbcfg := config.Conf.DB.Mysql MysqlTool = &mysqldb.Mysql{ Address: dbcfg.Addr, DBName: dbcfg.Dbname, UserName: dbcfg.User, PassWord: dbcfg.Password, } MysqlTool.Init() } func InitField() { info := MysqlTool.Find("code_area", nil, "", "", -1, -1) for _, m := range *info { var key string for i, v := range []string{"area", "city", "district"} { if i == 0 && util.ObjToString(m[v]) != "" { key = util.ObjToString(m[v]) } else if util.ObjToString(m[v]) != "" { key += "," + util.ObjToString(m[v]) } } AreaCode[key] = util.ObjToString(m["code"]) } log.Info("InitField", zap.Int("AreaCode", len(AreaCode))) info1 := MysqlTool.Find("code_bidtopsubtype", nil, "", "", -1, -1) for _, m := range *info1 { if util.IntAll(m["level"]) == 1 { TopTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"]) } if util.IntAll(m["level"]) == 2 { SubTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"]) } } log.Info("InitField", zap.Int("TopTypeCode", len(TopTypeCode)), zap.Int("SubTypeCode", len(SubTypeCode))) info2 := MysqlTool.Find("code_buyerclass", nil, "", "", -1, -1) for _, m := range *info2 { BuyerCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"]) } log.Info("InitField", zap.Int("BuyerCode", len(BuyerCode))) info3 := MysqlTool.Find("code_bidscope", nil, "", "", -1, -1) for _, m := range *info3 { if util.IntAll(m["level"]) == 1 { TopScopeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"]) } if util.IntAll(m["level"]) == 2 { SubScopeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"]) } } log.Info("InitField", zap.Int("TopScopeCode", len(TopScopeCode)), zap.Int("SubScopeCode", len(SubScopeCode))) util.ReadConfig("./field-criteria.json", &FCriteria) if len(FCriteria) > 0 { if m, o := FCriteria["dws_f_bid_baseinfo"].(map[string]interface{}); o { BaseField = util.ObjArrToStringArr(m["field_array"].([]interface{})) BaseVMap = m["field_criteria"].(map[string]interface{}) } else { log.Error("InitField", zap.String("field_array", "dws_f_bid_baseinfo")) panic("dws_f_bid_baseinfo") } if m, o := FCriteria["dws_f_bid_expand_baseinfo"].(map[string]interface{}); o { ExpandField = util.ObjArrToStringArr(m["field_array"].([]interface{})) ExpandVMap = m["field_criteria"].(map[string]interface{}) } else { log.Error("InitField", zap.String("field_array", "dws_f_bid_expand_baseinfo")) panic("dws_f_bid_expand_baseinfo") } if m, o := FCriteria["dws_f_project_baseinfo"].(map[string]interface{}); o { ProField = util.ObjArrToStringArr(m["field_array"].([]interface{})) ProVMap = m["field_criteria"].(map[string]interface{}) } else { log.Error("InitField", zap.String("field_array", "dws_f_project_baseinfo")) panic("dws_f_project_baseinfo") } if m, o := FCriteria["dws_f_project_business"].(map[string]interface{}); o { ProBusField = util.ObjArrToStringArr(m["field_array"].([]interface{})) ProBusVMap = m["field_criteria"].(map[string]interface{}) } else { log.Error("InitField", zap.String("field_array", "dws_f_project_business")) panic("dws_f_project_business") } } else { log.Error("InitField, 未找到field-criteria.json文件") panic("InitField, 未找到field-criteria.json文件") } }