package main import ( "data_mgo_to_tidb_project/config" "fmt" "os" "regexp" "github.com/gogf/gf/v2/util/gconv" "go.uber.org/zap" util "jygit.jydev.jianyu360.cn/data_processing/common_utils" "jygit.jydev.jianyu360.cn/data_processing/common_utils/log" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb" "jygit.jydev.jianyu360.cn/data_processing/common_utils/redis" ) var ( regLetter = regexp.MustCompile("[a-z]*") ) var ( //通用 MongoP *mongodb.MongodbSim MysqlTool *mysqldb.Mysql FCriteria map[string]interface{} AreaCode = make(map[string]string, 5000) TopTypeCode = make(map[string]string, 10) SubTypeCode = make(map[string]string, 40) BuyerCode = make(map[string]string, 100) TopScopeCode = make(map[string]interface{}, 20) SubScopeCode = make(map[string]interface{}, 70) ) var ( //标讯 saveSize = 200 saveBasePool = make(chan map[string]interface{}, 5000) saveBaseSp = make(chan bool, 1) saveExpandPool = make(chan map[string]interface{}, 5000) saveExpandSp = make(chan bool, 1) saveDetailPool = make(chan map[string]interface{}, 5000) saveDetailSp = make(chan bool, 1) saveAttrPool = make(chan map[string]interface{}, 5000) saveAttrSp = make(chan bool, 1) saveIntentPool = make(chan map[string]interface{}, 5000) saveIntentSp = make(chan bool, 1) saveBidderPool = make(chan map[string]interface{}, 5000) saveBidderSp = make(chan bool, 1) saveGoodsPool = make(chan map[string]interface{}, 5000) saveGoodsSp = make(chan bool, 1) BaseField []string BaseVMap map[string]interface{} ExpandField []string ExpandVMap map[string]interface{} DetailField = []string{"s_info_id", "s_detail", "s_contenthtml", "d_updatetime", "d_createtime"} AttrField = []string{"s_info_id", "s_file_id", "s_file_text", "i_is_bidding_doc", "d_updatetime", "d_createtime"} IntentField = []string{"s_info_id", "s_intention_name", "s_intention_demand", "s_item", "s_buyer_id", "f_totalprice", "s_expurasingtime", "s_reserved_amount", "d_updatetime", "d_createtime"} BidderField = []string{"s_info_id", "i_package_id", "s_bidder_id", "s_bidder_name", "s_other_bidder_id", "s_other_bidder_name", "i_is_bidders", "i_is_winner", "d_updatetime", "d_createtime"} GoodsField = []string{"s_info_id", "i_package_id", "s_purchasing_tag", "s_itemname", "s_brand", "s_model", "s_specs", "i_num", "f_unit", "f_totalprice", "d_updatetime", "d_createtime"} ) var ( //项目 saveProPool = make(chan map[string]interface{}, 5000) saveProSp = make(chan bool, 1) saveProbPool = make(chan map[string]interface{}, 5000) saveProbSp = make(chan bool, 1) saveProTagPool = make(chan map[string]interface{}, 5000) saveProTagSp = make(chan bool, 1) saveRelationPool = make(chan map[string]interface{}, 5000) saveRelationSp = make(chan bool, 1) saveErrPool = make(chan map[string]interface{}, 5000) saveErrSp = make(chan bool, 1) ProField []string ProVMap map[string]interface{} ProBusField []string ProBusVMap map[string]interface{} ProTagsField = []string{"projectid", "labelcode", "labelvalues", "labelweight", "createtime"} RelationField = []string{"projectid", "infoid", "name_id", "contact_id", "identity_type", "createtime"} FileTypeArr = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"} ) // InitLog @Description // @Author J 2022/7/26 15:30 func InitLog() { logcfg := config.Conf.Log err := log.InitLog( log.Path(logcfg.LogPath), log.Level(logcfg.LogLevel), log.Compress(logcfg.Compress), log.MaxSize(logcfg.MaxSize), log.MaxBackups(logcfg.MaxBackups), log.MaxAge(logcfg.MaxAge), log.Format(logcfg.Format), ) if err != nil { fmt.Printf("InitLog failed: %v\n", err) os.Exit(1) } } func InitMgo() { MongoP = &mongodb.MongodbSim{ MongodbAddr: config.Conf.DB.MongoP.Addr, DbName: config.Conf.DB.MongoP.Dbname, Size: config.Conf.DB.MongoP.Size, UserName: config.Conf.DB.MongoP.User, Password: config.Conf.DB.MongoP.Password, } MongoP.InitPool() } func InitMysql() { dbcfg := config.Conf.DB.Mysql MysqlTool = &mysqldb.Mysql{ Address: dbcfg.Addr, DBName: dbcfg.Dbname, UserName: dbcfg.User, PassWord: dbcfg.Password, } MysqlTool.Init() } func InitRedis() { //172.17.4.238:1479 redis.InitRedisByDb(config.Conf.DB.Redis.Address, 10, 10, 1, 300) } func InitField() { util.ReadConfig("./field-criteria.json", &FCriteria) if len(FCriteria) > 0 { //项目相关-字段限制暂时注释 if m, o := FCriteria["dws_f_project_baseinfo"].(map[string]interface{}); o { ProField = util.ObjArrToStringArr(m["field_array"].([]interface{})) ProVMap = m["field_criteria"].(map[string]interface{}) } else { log.Error("InitField", zap.String("field_array", "dws_f_project_baseinfo")) panic("dws_f_project_baseinfo") } if m, o := FCriteria["dws_f_project_business"].(map[string]interface{}); o { ProBusField = util.ObjArrToStringArr(m["field_array"].([]interface{})) ProBusVMap = m["field_criteria"].(map[string]interface{}) } else { log.Error("InitField", zap.String("field_array", "dws_f_project_business")) panic("dws_f_project_business") } } else { log.Error("InitField, 未找到field-criteria.json文件") panic("InitField, 未找到field-criteria.json文件") } } func InitCode() { //加载地区 data := MysqlTool.SelectBySql(`select * from global_common_data.code_area;`) if data == nil || len(*data) == 0 { log.Info("code_area代码表加载失败") panic("code_area代码表加载失败") } for _, v := range *data { code := gconv.String(v["code"]) area := gconv.String(v["area"]) city := gconv.String(v["city"]) district := gconv.String(v["district"]) if city == "" && district == "" { //加载省份 AreaCode[area] = code } if city != "" && district == "" { //加载城市 AreaCode[fmt.Sprintf("%s,%s", area, city)] = code } if city != "" && district != "" { //加载区县 AreaCode[fmt.Sprintf("%s,%s,%s", area, city, district)] = code } } //加载采购单位 buyerclassData := MysqlTool.SelectBySql(`select * from global_common_data.code_buyerclass;`) if buyerclassData == nil || len(*buyerclassData) == 0 { log.Info("code_buyerclass代码表加载失败") panic("code_buyerclass代码表加载失败") } for _, v := range *buyerclassData { code := gconv.String(v["code"]) name := gconv.String(v["name"]) BuyerCode[name] = code } //加载 bidtopsubtypeData := MysqlTool.Find("code_bidtopsubtype", nil, "", "", -1, -1) if bidtopsubtypeData == nil || len(*bidtopsubtypeData) == 0 { log.Info("code_bidtopsubtype代码表加载失败") panic("code_bidtopsubtype代码表加载失败") } for _, m := range *bidtopsubtypeData { if util.IntAll(m["level"]) == 1 { TopTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"]) } if util.IntAll(m["level"]) == 2 { SubTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"]) } } } func InitNsq() { if config.Conf.Alarm.Address != "" { AlertRemind.Addr = config.Conf.Alarm.Address AlertRemind.Toppic = config.Conf.Alarm.Toppic AlertRemind.Id = config.Conf.Alarm.Id } }