data_field_dispose/data_mgo_to_tidb/init.go

package main

import (
	"data_tidb/config"
	"fmt"
	"go.uber.org/zap"
	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
	"os"
)

var ( //通用
	MongoB, MongoP *mongodb.MongodbSim
	MysqlTool      *mysqldb.Mysql
	FCriteria      map[string]interface{}
	AreaCode       = make(map[string]string, 5000)
	TopTypeCode    = make(map[string]string, 10)
	SubTypeCode    = make(map[string]string, 40)
	BuyerCode      = make(map[string]string, 100)
	TopScopeCode   = make(map[string]interface{}, 20)
	SubScopeCode   = make(map[string]interface{}, 70)
)
var ( //标讯
	saveSize       = 200
	saveBasePool   = make(chan map[string]interface{}, 5000)
	saveBaseSp     = make(chan bool, 1)
	saveExpandPool = make(chan map[string]interface{}, 5000)
	saveExpandSp   = make(chan bool, 1)
	saveDetailPool = make(chan map[string]interface{}, 5000)
	saveDetailSp   = make(chan bool, 1)
	saveAttrPool   = make(chan map[string]interface{}, 5000)
	saveAttrSp     = make(chan bool, 1)
	saveIntentPool = make(chan map[string]interface{}, 5000)
	saveIntentSp   = make(chan bool, 1)
	saveBidderPool = make(chan map[string]interface{}, 5000)
	saveBidderSp   = make(chan bool, 1)
	saveGoodsPool  = make(chan map[string]interface{}, 5000)
	saveGoodsSp    = make(chan bool, 1)

	BaseField   []string
	BaseVMap    map[string]interface{}
	ExpandField []string
	ExpandVMap  map[string]interface{}
	DetailField = []string{"s_info_id", "s_detail", "s_contenthtml", "d_updatetime", "d_createtime"}
	AttrField   = []string{"s_info_id", "s_file_id", "s_file_text", "i_is_bidding_doc", "d_updatetime", "d_createtime"}
	IntentField = []string{"s_info_id", "s_intention_name", "s_intention_demand", "s_item", "s_buyer_id", "f_totalprice", "s_expurasingtime", "s_reserved_amount", "d_updatetime", "d_createtime"}
	BidderField = []string{"s_info_id", "i_package_id", "s_bidder_id", "s_bidder_name", "s_other_bidder_id", "s_other_bidder_name", "i_is_bidders", "i_is_winner", "d_updatetime", "d_createtime"}
	GoodsField  = []string{"s_info_id", "i_package_id", "s_purchasing_tag", "s_itemname", "s_brand", "s_model", "s_specs", "i_num", "f_unit", "f_totalprice", "d_updatetime", "d_createtime"}
)
var ( //项目
	saveProPool      = make(chan map[string]interface{}, 5000)
	saveProSp        = make(chan bool, 1)
	saveProbPool     = make(chan map[string]interface{}, 5000)
	saveProbSp       = make(chan bool, 1)
	saveProTagPool   = make(chan map[string]interface{}, 5000)
	saveProTagSp     = make(chan bool, 1)
	saveRelationPool = make(chan map[string]interface{}, 5000)
	saveRelationSp   = make(chan bool, 1)
	saveErrPool      = make(chan map[string]interface{}, 5000)
	saveErrSp        = make(chan bool, 1)
	ProField         []string
	ProVMap          map[string]interface{}
	ProBusField      []string
	ProBusVMap       map[string]interface{}
	ProTagsField     = []string{"projectid", "labelcode", "labelvalues", "labelweight", "createtime"}
	RelationField    = []string{"projectid", "infoid", "name_id", "contact_id", "identity_type", "createtime"}
	FileTypeArr      = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"}
)

func InitInfo() {
	config.Init("./common.toml")
	InitLog()
	InitMgo()
	InitMysql()
	InitField()
	log.Info("init success")
}

// InitLog @Description
// @Author J 2022/7/26 15:30
func InitLog() {
	logcfg := config.Conf.Log
	err := log.InitLog(
		log.Path(logcfg.LogPath),
		log.Level(logcfg.LogLevel),
		log.Compress(logcfg.Compress),
		log.MaxSize(logcfg.MaxSize),
		log.MaxBackups(logcfg.MaxBackups),
		log.MaxAge(logcfg.MaxAge),
		log.Format(logcfg.Format),
	)
	if err != nil {
		fmt.Printf("InitLog failed: %v\n", err)
		os.Exit(1)
	}
}

func InitMgo() {
	MongoB = &mongodb.MongodbSim{
		MongodbAddr: config.Conf.DB.MongoB.Addr,
		DbName:      config.Conf.DB.MongoB.Dbname,
		Size:        config.Conf.DB.MongoB.Size,
		UserName:    config.Conf.DB.MongoB.User,
		Password:    config.Conf.DB.MongoB.Password,
	}
	MongoB.InitPool()
	MongoP = &mongodb.MongodbSim{
		MongodbAddr: config.Conf.DB.MongoP.Addr,
		DbName:      config.Conf.DB.MongoP.Dbname,
		Size:        config.Conf.DB.MongoP.Size,
		UserName:    config.Conf.DB.MongoP.User,
		Password:    config.Conf.DB.MongoP.Password,
	}
	MongoP.InitPool()
}

func InitMysql() {
	dbcfg := config.Conf.DB.Mysql
	MysqlTool = &mysqldb.Mysql{
		Address:  dbcfg.Addr,
		DBName:   dbcfg.Dbname,
		UserName: dbcfg.User,
		PassWord: dbcfg.Password,
	}
	MysqlTool.Init()
}

func InitField() {
	info := MysqlTool.Find("code_area", nil, "", "", -1, -1)
	for _, m := range *info {
		var key string
		for i, v := range []string{"area", "city", "district"} {
			if i == 0 && util.ObjToString(m[v]) != "" {
				key = util.ObjToString(m[v])
			} else if util.ObjToString(m[v]) != "" {
				key += "," + util.ObjToString(m[v])
			}
		}
		AreaCode[key] = util.ObjToString(m["code"])
	}
	log.Info("InitField", zap.Int("AreaCode", len(AreaCode)))

	info1 := MysqlTool.Find("code_bidtopsubtype", nil, "", "", -1, -1)
	for _, m := range *info1 {
		if util.IntAll(m["level"]) == 1 {
			TopTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
		}
		if util.IntAll(m["level"]) == 2 {
			SubTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
		}
	}
	log.Info("InitField", zap.Int("TopTypeCode", len(TopTypeCode)), zap.Int("SubTypeCode", len(SubTypeCode)))

	info2 := MysqlTool.Find("code_buyerclass", nil, "", "", -1, -1)
	for _, m := range *info2 {
		BuyerCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
	}
	log.Info("InitField", zap.Int("BuyerCode", len(BuyerCode)))

	info3 := MysqlTool.Find("code_bidscope", nil, "", "", -1, -1)
	for _, m := range *info3 {
		if util.IntAll(m["level"]) == 1 {
			TopScopeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
		}
		if util.IntAll(m["level"]) == 2 {
			SubScopeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
		}
	}
	log.Info("InitField", zap.Int("TopScopeCode", len(TopScopeCode)), zap.Int("SubScopeCode", len(SubScopeCode)))

	util.ReadConfig("./field-criteria.json", &FCriteria)
	if len(FCriteria) > 0 {
		if m, o := FCriteria["dws_f_bid_baseinfo"].(map[string]interface{}); o {
			BaseField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
			BaseVMap = m["field_criteria"].(map[string]interface{})
		} else {
			log.Error("InitField", zap.String("field_array", "dws_f_bid_baseinfo"))
			panic("dws_f_bid_baseinfo")
		}
		if m, o := FCriteria["dws_f_bid_expand_baseinfo"].(map[string]interface{}); o {
			ExpandField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
			ExpandVMap = m["field_criteria"].(map[string]interface{})
		} else {
			log.Error("InitField", zap.String("field_array", "dws_f_bid_expand_baseinfo"))
			panic("dws_f_bid_expand_baseinfo")
		}
		//项目相关-字段限制暂时注释
		//if m, o := FCriteria["dws_f_project_baseinfo"].(map[string]interface{}); o {
		//	ProField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
		//	ProVMap = m["field_criteria"].(map[string]interface{})
		//} else {
		//	log.Error("InitField", zap.String("field_array", "dws_f_project_baseinfo"))
		//	panic("dws_f_project_baseinfo")
		//}
		//if m, o := FCriteria["dws_f_project_business"].(map[string]interface{}); o {
		//	ProBusField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
		//	ProBusVMap = m["field_criteria"].(map[string]interface{})
		//} else {
		//	log.Error("InitField", zap.String("field_array", "dws_f_project_business"))
		//	panic("dws_f_project_business")
		//}
	} else {
		log.Error("InitField, 未找到field-criteria.json文件")
		panic("InitField, 未找到field-criteria.json文件")
	}
}

// 插入并打印信息
func InsertGlobalMysqlData(name string, data map[string]interface{}, mark string) int64 {
	inb := MysqlTool.Insert(name, data)
	if inb == -1 {
		log.Info("插入数据异常", zap.String(name, mark))
	}
	return inb
}