data_field_dispose/data_tidb/init.go

package main

import (
	util "app.yhyue.com/data_processing/common_utils"
	"app.yhyue.com/data_processing/common_utils/log"
	"app.yhyue.com/data_processing/common_utils/mongodb"
	"app.yhyue.com/data_processing/common_utils/mysqldb"
	"data_tidb/config"
	"fmt"
	"go.uber.org/zap"
	"os"
)

var (
	MongoB, MongoP *mongodb.MongodbSim
	MysqlTool      *mysqldb.Mysql

	FCriteria map[string]interface{}

	saveSize       = 200
	saveBasePool   = make(chan map[string]interface{}, 5000)
	saveBaseSp     = make(chan bool, 1)
	saveExpandPool = make(chan map[string]interface{}, 5000)
	saveExpandSp   = make(chan bool, 1)
	saveTagPool    = make(chan map[string]interface{}, 5000)
	saveTagSp      = make(chan bool, 1)
	saveAttrPool   = make(chan map[string]interface{}, 5000)
	saveAttrSp     = make(chan bool, 1)
	saveIfmPool    = make(chan map[string]interface{}, 5000)
	saveIfmSp      = make(chan bool, 1)
	saveIntentPool = make(chan map[string]interface{}, 5000)
	saveIntentSp   = make(chan bool, 1)
	saveWinnerPool = make(chan map[string]interface{}, 5000)
	saveWinnerSp   = make(chan bool, 1)
	savePkgPool    = make(chan map[string]interface{}, 5000)
	savePkgSp      = make(chan bool, 1)
	savePurPool    = make(chan map[string]interface{}, 5000)
	savePurSp      = make(chan bool, 1)

	saveProPool    = make(chan map[string]interface{}, 5000)
	saveProSp      = make(chan bool, 1)
	saveProbPool   = make(chan map[string]interface{}, 5000)
	saveProbSp     = make(chan bool, 1)
	saveProTagPool = make(chan map[string]interface{}, 5000)
	saveProTagSp   = make(chan bool, 1)

	saveRelationPool = make(chan map[string]interface{}, 5000)
	saveRelationSp   = make(chan bool, 1)

	saveErrPool = make(chan map[string]interface{}, 5000)
	saveErrSp   = make(chan bool, 1)

	AreaCode     = make(map[string]string, 5000)
	TopTypeCode  = make(map[string]string, 10)
	SubTypeCode  = make(map[string]string, 40)
	BuyerCode    = make(map[string]string, 100)
	TopScopeCode = make(map[string]interface{}, 20)
	SubScopeCode = make(map[string]interface{}, 70)

	BaseField   []string
	BaseVMap    map[string]interface{}
	ProField    []string
	ProVMap     map[string]interface{}
	ProBusField []string
	ProBusVMap  map[string]interface{}
	ExpandField = []string{"infoid", "projectperiod", "project_duration", "project_timeunit", "project_startdate", "project_completedate", "signstarttime",
		"signendtime", "docstarttime", "docendtime", "bidendtime", "bidstarttime", "signaturedate", "bidway", "docamount", "agencyrate", "agencyfee",
		"currency", "funds", "payway", "bidamounttype", "review_experts", "bidmethod", "bid_bond", "bid_guarantee", "contract_bond", "contract_guarantee",
		"bidopenaddress", "contractcode", "supervisorrate", "getdocmethod", "projectaddr", "project_scale", "purchasing_tag", "updatetime", "createtime"}
	TagsField    = []string{"infoid", "labelcode", "labelvalues", "labelweight", "createtime"}
	AttrField    = []string{"infoid", "filename", "fid", "ftype", "org_url", "size", "file_type", "createtime"}
	FileTypeArr  = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"}
	IfmField     = []string{"infoid", "approvecode", "approvedept", "approvestatus", "approvetime", "approvenumber", "approvecontent", "projecttype", "approvecity", "updatetime", "createtime"}
	PurField     = []string{"infoid", "item", "itemname", "brandname", "specs", "model", "unitname", "number", "unitprice", "totalprice"}
	IntentField  = []string{"infoid", "projectname", "projectscope", "item", "buyer_id", "totalprice", "expurasingtime", "reserved_amount", "updatetime", "createtime"}
	WinnerField  = []string{"infoid", "winner_id", "winnersort", "package_id", "createtime", "updatetime"}
	PackageField = []string{"infoid", "packagename", "packagename", "packageorgin", "packagedetail", "budget", "bidamount", "winner_id", "updatetime", "createtime"}

	ProTagsField = []string{"projectid", "labelcode", "labelvalues", "labelweight", "createtime"}

	RelationField = []string{"projectid", "infoid", "name_id", "contact_id", "identity_type", "createtime"}
)

// InitLog @Description
// @Author J 2022/7/26 15:30
func InitLog() {
	logcfg := config.Conf.Log

	err := log.InitLog(
		log.Path(logcfg.LogPath),
		log.Level(logcfg.LogLevel),
		log.Compress(logcfg.Compress),
		log.MaxSize(logcfg.MaxSize),
		log.MaxBackups(logcfg.MaxBackups),
		log.MaxAge(logcfg.MaxAge),
		log.Format(logcfg.Format),
	)
	if err != nil {
		fmt.Printf("InitLog failed: %v\n", err)
		os.Exit(1)
	}
}

func InitMgo() {
	MongoB = &mongodb.MongodbSim{
		MongodbAddr: config.Conf.DB.MongoB.Addr,
		DbName:      config.Conf.DB.MongoB.Dbname,
		Size:        config.Conf.DB.MongoB.Size,
		UserName:    config.Conf.DB.MongoB.User,
		Password:    config.Conf.DB.MongoB.Password,
	}
	MongoB.InitPool()
	MongoP = &mongodb.MongodbSim{
		MongodbAddr: config.Conf.DB.MongoP.Addr,
		DbName:      config.Conf.DB.MongoP.Dbname,
		Size:        config.Conf.DB.MongoP.Size,
		UserName:    config.Conf.DB.MongoP.User,
		Password:    config.Conf.DB.MongoP.Password,
	}
	MongoP.InitPool()
}

func InitMysql() {
	dbcfg := config.Conf.DB.Mysql
	MysqlTool = &mysqldb.Mysql{
		Address:  dbcfg.Addr,
		DBName:   dbcfg.Dbname,
		UserName: dbcfg.User,
		PassWord: dbcfg.Password,
	}
	MysqlTool.Init()
}

func InitField() {
	info := MysqlTool.Find("code_area", nil, "", "", -1, -1)
	for _, m := range *info {
		var key string
		for i, v := range []string{"area", "city", "district"} {
			if i == 0 && util.ObjToString(m[v]) != "" {
				key = util.ObjToString(m[v])
			} else if util.ObjToString(m[v]) != "" {
				key += "," + util.ObjToString(m[v])
			}
		}
		AreaCode[key] = util.ObjToString(m["code"])
	}
	log.Info("InitField", zap.Int("AreaCode", len(AreaCode)))

	info1 := MysqlTool.Find("code_bidtopsubtype", nil, "", "", -1, -1)
	for _, m := range *info1 {
		if util.IntAll(m["level"]) == 1 {
			TopTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
		}
		if util.IntAll(m["level"]) == 2 {
			SubTypeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
		}
	}
	log.Info("InitField", zap.Int("TopTypeCode", len(TopTypeCode)), zap.Int("SubTypeCode", len(SubTypeCode)))

	info2 := MysqlTool.Find("code_buyerclass", nil, "", "", -1, -1)
	for _, m := range *info2 {
		BuyerCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
	}
	log.Info("InitField", zap.Int("BuyerCode", len(BuyerCode)))

	info3 := MysqlTool.Find("code_bidscope", nil, "", "", -1, -1)
	for _, m := range *info3 {
		if util.IntAll(m["level"]) == 1 {
			TopScopeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
		}
		if util.IntAll(m["level"]) == 2 {
			SubScopeCode[util.ObjToString(m["name"])] = util.ObjToString(m["code"])
		}
	}
	log.Info("InitField", zap.Int("TopScopeCode", len(TopScopeCode)), zap.Int("SubScopeCode", len(SubScopeCode)))

	util.ReadConfig("./field-criteria.json", &FCriteria)
	if len(FCriteria) > 0 {
		if m, o := FCriteria["dws_f_bid_baseinfo"].(map[string]interface{}); o {
			BaseField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
			BaseVMap = m["field_criteria"].(map[string]interface{})
		} else {
			log.Error("InitField", zap.String("field_array", "dws_f_bid_baseinfo"))
			os.Exit(-1)
		}
		if m, o := FCriteria["dws_f_project_baseinfo"].(map[string]interface{}); o {
			ProField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
			ProVMap = m["field_criteria"].(map[string]interface{})
		} else {
			log.Error("InitField", zap.String("field_array", "dws_f_project_baseinfo"))
			os.Exit(-1)
		}
		if m, o := FCriteria["dws_f_project_business"].(map[string]interface{}); o {
			ProBusField = util.ObjArrToStringArr(m["field_array"].([]interface{}))
			ProBusVMap = m["field_criteria"].(map[string]interface{})
		} else {
			log.Error("InitField", zap.String("field_array", "dws_f_project_business"))
			os.Exit(-1)
		}
	} else {
		log.Error("InitField, 未找到field-criteria.json文件")
		os.Exit(-1)
	}
}