data_field_dispose/data_mgo_to_tidb_project/init.go

package main

import (
	"data_mgo_to_tidb_project/config"
	"fmt"
	"os"
	"regexp"

	"github.com/gogf/gf/v2/util/gconv"
	"go.uber.org/zap"
	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
)

var (
	regLetter = regexp.MustCompile("[a-z]*")
)

var ( //通用
	MongoP       *mongodb.MongodbSim
	MysqlTool    *mysqldb.Mysql
	FCriteria    map[string]interface{}
	AreaCode     = make(map[string]string, 5000)
	TopTypeCode  = make(map[string]string, 10)
	SubTypeCode  = make(map[string]string, 40)
	BuyerCode    = make(map[string]string, 100)
	TopScopeCode = make(map[string]interface{}, 20)
	SubScopeCode = make(map[string]interface{}, 70)
)
var ( //标讯
	saveSize       = 200
	saveBasePool   = make(chan map[string]interface{}, 5000)
	saveBaseSp     = make(chan bool, 1)
	saveExpandPool = make(chan map[string]interface{}, 5000)
	saveExpandSp   = make(chan bool, 1)
	saveDetailPool = make(chan map[string]interface{}, 5000)
	saveDetailSp   = make(chan bool, 1)
	saveAttrPool   = make(chan map[string]interface{}, 5000)
	saveAttrSp     = make(chan bool, 1)
	saveIntentPool = make(chan map[string]interface{}, 5000)
	saveIntentSp   = make(chan bool, 1)
	saveBidderPool = make(chan map[string]interface{}, 5000)
	saveBidderSp   = make(chan bool, 1)
	saveGoodsPool  = make(chan map[string]interface{}, 5000)
	saveGoodsSp    = make(chan bool, 1)

	BaseField   []string
	BaseVMap    map[string]interface{}
	ExpandField []string
	ExpandVMap  map[string]interface{}
	DetailField = []string{"s_info_id", "s_detail", "s_contenthtml", "d_updatetime", "d_createtime"}
	AttrField   = []string{"s_info_id", "s_file_id", "s_file_text", "i_is_bidding_doc", "d_updatetime", "d_createtime"}
	IntentField = []string{"s_info_id", "s_intention_name", "s_intention_demand", "s_item", "s_buyer_id", "f_totalprice", "s_expurasingtime", "s_reserved_amount", "d_updatetime", "d_createtime"}
	BidderField = []string{"s_info_id", "i_package_id", "s_bidder_id", "s_bidder_name", "s_other_bidder_id", "s_other_bidder_name", "i_is_bidders", "i_is_winner", "d_updatetime", "d_createtime"}
	GoodsField  = []string{"s_info_id", "i_package_id", "s_purchasing_tag", "s_itemname", "s_brand", "s_model", "s_specs", "i_num", "f_unit", "f_totalprice", "d_updatetime", "d_createtime"}
)
var ( //项目
	saveProPool      = make(chan map[string]interface{}, 5000)
	saveProSp        = make(chan bool, 1)
	saveProbPool     = make(chan map[string]interface{}, 5000)
	saveProbSp       = make(chan bool, 1)
	saveProTagPool   = make(chan map[string]interface{}, 5000)
	saveProTagSp     = make(chan bool, 1)
	saveRelationPool = make(chan map[string]interface{}, 5000)
	saveRelationSp   = make(chan bool, 1)
	saveErrPool      = make(chan map[string]interface{}, 5000)
	saveErrSp        = make(chan bool, 1)
	ProField         []string
	ProVMap          map[string]interface{}
	ProBusField      []string
	ProBusVMap       map[string]interface{}
	ProTagsField     = []string{"projectid", "labelcode", "labelvalues", "labelweight", "createtime"}
	RelationField    = []string{"projectid", "infoid", "name_id", "contact_id", "identity_type", "createtime"}
	FileTypeArr      = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"}
)

// InitLog @Description
// @Author J 2022/7/26 15:30
func InitLog() {
	logcfg := config.Conf.Log
	err := log.InitLog(
		log.Path(logcfg.LogPath),
		log.Level(logcfg.LogLevel),
		log.Compress(logcfg.Compress),
		log.MaxSize(logcfg.MaxSize),
		log.MaxBackups(logcfg.MaxBackups),
		log.MaxAge(logcfg.MaxAge),
		log.Format(logcfg.Format),
	)
	if err != nil {
		fmt.Printf("InitLog failed: %v\n", err)
		os.Exit(1)
	}
}

func InitMgo() {

	MongoP = &mongodb.MongodbSim{
		MongodbAddr: config.Conf.DB.MongoP.Addr,
		DbName:      config.Conf.DB.MongoP.Dbname,
		Size:        config.Conf.DB.MongoP.Size,
		UserName:    config.Conf.DB.MongoP.User,
		Password:    config.Conf.DB.MongoP.Password,
	}
	MongoP.InitPool()
}

func InitMysql() {
	dbcfg := config.Conf.DB.Mysql
	MysqlTool = &mysqldb.Mysql{
		Address:  dbcfg.Addr,
		DBName:   dbcfg.Dbname,
		UserName: dbcfg.User,
		PassWord: dbcfg.Password,
	}
	MysqlTool.Init()
}

func InitRedis() {
	//172.17.4.238:1479
	redis.InitRedisByDb(config.Conf.DB.Redis.Address, 10, 10, 1, 300)
}

func InitField() {
	util.ReadConfig("./field-criteria.json", &FCriteria)
	if len(FCriteria) > 0 {
		//项目相关-字段限制暂时注释
		if m, o := FCriteria["dws_f_project_baseinfo"].(map[string]interface{}); o {
			ProField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
			ProVMap = m["field_criteria"].(map[string]interface{})
		} else {
			log.Error("InitField", zap.String("field_array", "dws_f_project_baseinfo"))
			panic("dws_f_project_baseinfo")
		}
		if m, o := FCriteria["dws_f_project_business"].(map[string]interface{}); o {
			ProBusField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
			ProBusVMap = m["field_criteria"].(map[string]interface{})
		} else {
			log.Error("InitField", zap.String("field_array", "dws_f_project_business"))
			panic("dws_f_project_business")
		}
	} else {
		log.Error("InitField, 未找到field-criteria.json文件")
		panic("InitField, 未找到field-criteria.json文件")
	}
}

func InitCode() {
	//加载地区
	data := MysqlTool.SelectBySql(`select * from global_common_data.code_area;`)
	if data == nil || len(*data) == 0 {
		log.Info("code_area代码表加载失败")
		panic("code_area代码表加载失败")
	}
	for _, v := range *data {
		code := gconv.String(v["code"])
		area := gconv.String(v["area"])
		city := gconv.String(v["city"])
		district := gconv.String(v["district"])
		if city == "" && district == "" {
			//加载省份
			AreaCode[area] = code
		}
		if city != "" && district == "" {
			//加载城市
			AreaCode[fmt.Sprintf("%s,%s", area, city)] = code
		}
		if city != "" && district != "" {
			//加载区县
			AreaCode[fmt.Sprintf("%s,%s,%s", area, city, district)] = code
		}
	}

	//加载采购单位
	buyerclassData := MysqlTool.SelectBySql(`select * from global_common_data.code_buyerclass;`)
	if buyerclassData == nil || len(*buyerclassData) == 0 {
		log.Info("code_buyerclass代码表加载失败")
		panic("code_buyerclass代码表加载失败")
	}
	for _, v := range *buyerclassData {
		code := gconv.String(v["code"])
		name := gconv.String(v["name"])
		BuyerCode[name] = code
	}

	//加载
	bidtopsubtypeData := MysqlTool.Find("code_bidtopsubtype", nil, "", "", -1, -1)
	if bidtopsubtypeData == nil || len(*bidtopsubtypeData) == 0 {
		log.Info("code_bidtopsubtype代码表加载失败")
		panic("code_bidtopsubtype代码表加载失败")
	}
	for _, m := range *bidtopsubtypeData {
		if util.IntAll(m["level"]) == 1 {
			TopTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
		}
		if util.IntAll(m["level"]) == 2 {
			SubTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
		}
	}
}

func InitNsq() {
	if config.Conf.Alarm.Address != "" {
		AlertRemind.Addr = config.Conf.Alarm.Address
		AlertRemind.Toppic = config.Conf.Alarm.Toppic
		AlertRemind.Id = config.Conf.Alarm.Id
	}
}