datatag/CMPlatform/util/config.go

package util

import (
	"app.yhyue.com/moapp/jybase/common"
	elastic "app.yhyue.com/moapp/jybase/es"
	"app.yhyue.com/moapp/jybase/log"
	"app.yhyue.com/moapp/jybase/mongodb"
	"app.yhyue.com/moapp/jybase/mysql"
	"app.yhyue.com/moapp/jybase/redis"
	"bytes"
	"fmt"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/transform"
	"io"
	"os"
	"sort"
)

var (
	Sysconfig       map[string]interface{} //配置文件
	Mgo             *mongodb.MongodbSim
	MgoCus          *mongodb.MongodbSim //企业服务库
	MgoEn           *mongodb.MongodbSim
	EsIndex, EsType string
	EsAllIndex      string
	InterimIndex    string
	TotalIndex      string
	MgoEnC          string
	Subday          float64
	PreviewHref     string //数据预览地址
	CustomerApi     string
	LatestId        = ""                //起始id
	MgoTag          *mongodb.MongodbSim //标签库连接
	MgoSave         *mongodb.MongodbSim //数据保存库连接
	MgoBuyer        *mongodb.MongodbSim //医院等级信息
	MgoEnps         *mongodb.MongodbSim //企业信息
	MgoExtract      *mongodb.MongodbSim //抽取
	MgoBidding      *mongodb.MongodbSim //bidding
	BiddingColl     string
	SaveColl        string
	HospColl        string
	BuyerEntColl    string
	EnpsColl        string
	Es, EsCus       elastic.Es //es
	Index           string
	Itype           string
	IndexCus        string
	TaskTime        int64 //定时任务时间
	//历史数据
	SId             string
	EId             string
	CustomerName    string
	SaveCollProject string
	SaveCollbid     string
	SaveUserMail    string
	ExtractColl     string
	JyMysql         *mysql.Mysql
	JyPushMysql     *mysql.Mysql
	JyMysqls        *mysql.Mysql
	EntMysql        *mysql.Mysql
	MgoSite         *mongodb.MongodbSim
	SiteColl        string
	IsNewSql        int

	RpcJyServer, RpcProServer string
)

var (
	TopTypeArr    []string                  //公告一级分类
	SubTypeArr    []string                  //公告二级分类
	Province      []string                  //省
	ProvinceCitys map[string][]string       //省:[市1,市2]
	CityDistricts map[string][]string       //市:[区1,区2]
	BuyerClass    []string                  //采购单位行业
	BuyerClassMap map[string][]string       //采购单位行业带拼音字母
	MatchTypeMap  *[]map[string]interface{} //匹配方式
	MatchTypeMap2 map[string]interface{}    //匹配方式
	MatchTypeMap3 *[]map[string]interface{} //匹配方式
	ExistFiled    *[]map[string]interface{} //存在的字段
	ScopeClassMap *[]map[string]interface{} //公告行业
	TagFieldMap   *[]map[string]interface{} //标签属性值
	TaskTypeMap   *[]map[string]interface{} //任务类型
	BuryClassType map[string]interface{}
	ProvinceMap   map[string]interface{}
	SiteArr       []string //网站名称
)

func InitMgoPool() {
	Mgo = &mongodb.MongodbSim{
		MongodbAddr: common.ObjToString(Sysconfig["mgodb"]),
		Size:        common.IntAll(Sysconfig["dbsize"]),
		DbName:      common.ObjToString(Sysconfig["dbname"]),
	}
	Mgo.InitPool()

	customer := Sysconfig["customer"].(map[string]interface{})
	CustomerApi = common.ObjToString(customer["api"])
	MgoCus = &mongodb.MongodbSim{
		MongodbAddr: common.ObjToString(customer["mgodb"]),
		Size:        common.IntAll(customer["dbsize"]),
		DbName:      common.ObjToString(customer["dbname"]),
	}
	MgoCus.InitPool()
}

func initMgoEn() {
	MgoEn = &mongodb.MongodbSim{
		MongodbAddr: common.ObjToString(Sysconfig["mgodben"]),
		Size:        common.IntAll(Sysconfig["dbsize"]),
		DbName:      common.ObjToString(Sysconfig["dbnameen"]),
	}
	MgoEnC = common.ObjToString(Sysconfig["mongoenc"])
	MgoEn.InitPool()
}

func InitOther() {
	initCitys()
	initInfoType()
	initBuyerClass()
	initMatchType()
	initExitField()
	initScopeClass()
	initTaskType()
	initEs()
	InitLog()

	// initMgoEn()
	Subday = common.Float64All(Sysconfig["subday"])
	PreviewHref = common.ObjToString(Sysconfig["preview_href"])
	initTagField()
	initdb()
	initBuryClassType()
	initProvince()
	initSite()
	IsNewSql = common.IntAll(Sysconfig["isNewSql"])

	RpcJyServer = common.ObjToString(Sysconfig["rpcServer_jy"])
	RpcProServer = common.ObjToString(Sysconfig["rpcServer_project"])
}

func initCitys() {
	//map初始化
	if ProvinceCitys == nil {
		ProvinceCitys = make(map[string][]string)
	}
	if CityDistricts == nil {
		CityDistricts = make(map[string][]string)
	}
	datas, _ := Mgo.Find("code_city", `{}`, `{"_id":1}`, `{}`, false, -1, -1)
	for k, data := range *datas {
		brief := data["brief"].(string)
		if k != 0 {
			Province = append(Province, brief)
		}
		if data["citys"] != nil {
			citys := data["citys"].([]interface{})
			citytmp := []string{}
			for _, city := range citys {
				cityMap := city.(map[string]interface{})
				for cname, districts := range cityMap {
					citytmp = append(citytmp, cname)
					tmp := districts.([]interface{})
					districtarr := common.ObjArrToStringArr(tmp)
					CityDistricts[cname] = districtarr
				}
			}
			ProvinceCitys[brief] = citytmp
		}
	}
	Province = append(Province, common.ObjToString((*datas)[0]["brief"]))
}

func initSite() {
	if SiteArr == nil {
		SiteArr = []string{}
	}
	datas, _ := MgoSite.Find("site", `{}`, `{"_id":1}`, `{}`, false, -1, -1)
	for _, data := range *datas {
		name := data["site"].(string)
		SiteArr = append(SiteArr, name)
	}
	//log.Println("网站名称", SiteArr)
}

func initInfoType() {
	datas, _ := Mgo.Find("code_infotype", `{}`, `{"_id":1}`, `{}`, false, -1, -1)
	for _, data := range *datas {
		name := data["name"].(string)
		TopTypeArr = append(TopTypeArr, name)
		child := common.ObjArrToStringArr(data["child"].([]interface{}))
		SubTypeArr = append(SubTypeArr, child...)
	}
	//common.Debug(InfoTypeArr, InfoTypeMap)
}

func initBuyerClass() {
	if BuyerClassMap == nil {
		BuyerClassMap = make(map[string][]string)
	}
	datas, _ := Mgo.Find("code_buyerclass", `{}`, `{"name":1}`, `{}`, false, -1, -1)
	for _, data := range *datas {
		name := data["name"].(string)
		BuyerClass = append(BuyerClass, name)
	}
	sort.Sort(ByPinyin(BuyerClass))
	for _, name := range BuyerClass {
		str, _ := New(name).Mode(InitialsInCapitals).Convert()
		//fmt.Println(str, err)
		zimu := str[0:1]
		BuyerClassMap[zimu] = append(BuyerClassMap[zimu], name)
	}

}

func initMatchType() {
	//map初始化
	if MatchTypeMap2 == nil {
		MatchTypeMap2 = make(map[string]interface{})
	}
	MatchTypeMap, _ = Mgo.Find("code_matchtype", `{}`, `{"_id":1}`, `{}`, false, -1, -1)
	MatchTypeMap3, _ = Mgo.Find("code_matchtype", `{"type": 1}`, nil, nil, false, -1, -1)
	for _, d := range *MatchTypeMap {
		code := common.ObjToString(d["code"])
		name := common.ObjToString(d["name"])
		MatchTypeMap2[code] = name
	}
}

func initExitField() {
	ExistFiled, _ = Mgo.Find("code_field", `{}`, `{"_id":1}`, `{}`, false, -1, -1)
	for _, data := range *ExistFiled {
		str, _ := New(data["name"].(string)).Mode(InitialsInCapitals).Convert()
		//fmt.Println(str, err)
		//zimu := str[0:1]
		data["zimu"] = str
	}

	for i := 0; i < len(*ExistFiled); i++ {
		for j := 0; j < len(*ExistFiled)-i-1; j++ {
			if (*ExistFiled)[j]["zimu"].(string) > (*ExistFiled)[j+1]["zimu"].(string) {
				(*ExistFiled)[j], (*ExistFiled)[j+1] = (*ExistFiled)[j+1], (*ExistFiled)[j]
			}
		}
	}
	//for _, data := range *datas {
	//	name := data["name"].(string)
	//	ExistFiled = append(ExistFiled, name)
	//}
}

func initScopeClass() {
	ScopeClassMap, _ = Mgo.Find("code_scopeclass", `{}`, `{"_id": 1}`, `{}`, false, -1, -1)
}

func initTaskType() {
	TaskTypeMap, _ = Mgo.Find("code_task", `{}`, `{"_id": 1}`, `{}`, false, -1, -1)
}

func initEs() {
	EsIndex, EsType = common.ObjToString(Sysconfig["elasticsearch_index"]), common.ObjToString(Sysconfig["elasticsearch_type"])
	EsAllIndex = common.ObjToString(Sysconfig["elasticsearch_all_index"])
	TotalIndex = common.ObjToString(Sysconfig["elasticsearch_total_index"])
	InterimIndex = common.ObjToString(Sysconfig["elasticsearch_interim_index"])
	// elastic.InitElasticSize(common.ObjToString(Sysconfig["elasticsearch"]), common.IntAllDef(common.Int64All(Sysconfig["elasticPoolSize"]), 30))
	es := Sysconfig["es"].(map[string]interface{})
	Es = elastic.NewEs(common.ObjToString(es["version"]), common.ObjToString(es["addr"]), common.IntAllDef(es["pool"], 15), common.ObjToString(es["userName"]), common.ObjToString(es["password"]))
	Index = common.ObjToString(es["index"])
	Itype = common.ObjToString(es["itype"])
	esCm := Sysconfig["es_customer"].(map[string]interface{})
	EsCus = elastic.NewEs(common.ObjToString(esCm["version"]), common.ObjToString(esCm["addr"]), common.IntAllDef(esCm["pool"], 15), common.ObjToString(esCm["userName"]), common.ObjToString(esCm["password"]))
	IndexCus = common.ObjToString(esCm["index"])

}

func initTagField() {
	TagFieldMap, _ = Mgo.Find("code_tagfield", `{}`, `{"_id": 1}`, `{}`, false, -1, -1)
}

func initBuryClassType() {
	infoType := JyMysql.SelectBySql("SELECT id,name FROM infotype")
	BuryClassType = make(map[string]interface{})
	for _, val := range *infoType {
		k := common.ObjToString(val["name"])
		BuryClassType[k] = val["id"]
	}
}

func initProvince() {
	proArr := JyMysql.SelectBySql("SELECT id,name FROM province")
	ProvinceMap = make(map[string]interface{})
	for _, val := range *proArr {
		k := common.ObjToString(val["name"])
		ProvinceMap[k] = val["id"]
	}
}

func initdb() {
	//mgo save
	save := Sysconfig["save"].(map[string]interface{})
	SaveColl = common.ObjToString(save["coll"])
	SaveCollProject = common.ObjToString(save["coll_project"])
	SaveCollbid = common.ObjToString(save["coll_bid"])
	SaveUserMail = common.ObjToString(save["coll_usermail"])
	MgoSave = &mongodb.MongodbSim{
		MongodbAddr: common.ObjToString(save["addr"]),
		DbName:      common.ObjToString(save["db"]),
		Size:        common.IntAllDef(save["size"], 15),
	}
	MgoSave.InitPool()
	//mgo buyerinfo
	buyerinfo := Sysconfig["buyerinfo"].(map[string]interface{})
	HospColl = common.ObjToString(buyerinfo["hospcoll"])
	BuyerEntColl = common.ObjToString(buyerinfo["buyerentcoll"])
	MgoBuyer = &mongodb.MongodbSim{
		MongodbAddr: common.ObjToString(buyerinfo["addr"]),
		DbName:      common.ObjToString(buyerinfo["db"]),
		Size:        common.IntAllDef(buyerinfo["size"], 15),
		UserName:    common.ObjToString(buyerinfo["username"]),
		Password:    common.ObjToString(buyerinfo["password"]),
		ReplSet:     common.ObjToString(buyerinfo["replSet"]),
	}
	MgoBuyer.InitPool()
	//mgo enterprise
	enterprise := Sysconfig["enterprise"].(map[string]interface{})
	EnpsColl = common.ObjToString(enterprise["coll"])
	MgoEnps = &mongodb.MongodbSim{
		MongodbAddr: common.ObjToString(enterprise["addr"]),
		DbName:      common.ObjToString(enterprise["db"]),
		Size:        common.IntAllDef(enterprise["size"], 15),
		UserName:    common.ObjToString(buyerinfo["username"]),
		Password:    common.ObjToString(buyerinfo["password"]),
		ReplSet:     common.ObjToString(buyerinfo["replSet"]),
	}
	MgoEnps.InitPool()
	extract := Sysconfig["extract"].(map[string]interface{})
	ExtractColl = common.ObjToString(extract["coll"])
	MgoExtract = &mongodb.MongodbSim{
		MongodbAddr: common.ObjToString(extract["addr"]),
		DbName:      common.ObjToString(extract["db"]),
		Size:        common.IntAllDef(extract["size"], 15),
	}
	MgoExtract.InitPool()

	//bidding库
	bidding := Sysconfig["bidding_mgo"].(map[string]interface{})
	BiddingColl = common.ObjToString(bidding["coll"])
	MgoBidding = &mongodb.MongodbSim{
		MongodbAddr: common.ObjToString(bidding["addr"]),
		DbName:      common.ObjToString(bidding["db"]),
		Size:        common.IntAll(bidding["size"]),
		UserName:    common.ObjToString(bidding["username"]),
		Password:    common.ObjToString(bidding["password"]),
		ReplSet:     common.ObjToString(bidding["replSet"]),
	}
	MgoBidding.InitPool()

	site := Sysconfig["site_mgo"].(map[string]interface{})
	SiteColl = common.ObjToString(site["coll"])
	MgoSite = &mongodb.MongodbSim{
		MongodbAddr: common.ObjToString(site["addr"]),
		Size:        common.IntAll(site["size"]),
		DbName:      common.ObjToString(site["db"]),
	}
	MgoSite.InitPool()

	//es
	// Es = &elastic.Elastic{
	// 	S_esurl: common.ObjToString(es["addr"]),
	// 	I_size:  common.IntAllDef(es["pool"], 15),
	// }
	redis.InitRedis(common.ObjToString(Sysconfig["redis_addrs"]))
	jyMysql := common.ObjToMap(Sysconfig["jyPushMysql"])
	JyMysql = &mysql.Mysql{
		Address:  (*jyMysql)["address"].(string),
		UserName: (*jyMysql)["username"].(string),
		PassWord: (*jyMysql)["password"].(string),
		DBName:   (*jyMysql)["dbName"].(string),
	}
	JyMysql.Init()
	jyPushMysql := common.ObjToMap(Sysconfig["pushMysql"])
	JyPushMysql = &mysql.Mysql{
		Address:  (*jyPushMysql)["address"].(string),
		UserName: (*jyPushMysql)["username"].(string),
		PassWord: (*jyPushMysql)["password"].(string),
		DBName:   (*jyPushMysql)["dbName"].(string),
	}
	JyPushMysql.Init()
	jyMysqls := common.ObjToMap(Sysconfig["jyMysql"])
	JyMysqls = &mysql.Mysql{
		Address:  (*jyMysqls)["address"].(string),
		UserName: (*jyMysqls)["username"].(string),
		PassWord: (*jyMysqls)["password"].(string),
		DBName:   (*jyMysqls)["dbName"].(string),
	}
	JyMysqls.Init()
	entMysql := common.ObjToMap(Sysconfig["entMysql"])
	EntMysql = &mysql.Mysql{
		Address:  (*entMysql)["address"].(string),
		UserName: (*entMysql)["username"].(string),
		PassWord: (*entMysql)["password"].(string),
		DBName:   (*entMysql)["dbName"].(string),
	}
	EntMysql.Init()
}

func InitLog() {
	err := log.InitLog(
		log.Path(common.ObjToString(Sysconfig["logpath"])),
		log.Level("debug"),
		log.Compress(true),
		log.MaxSize(20), // M
		log.MaxBackups(10),
		log.MaxAge(10),
		log.Format("text"), // text or json output
	)
	if err != nil {
		fmt.Printf("InitLog failed: %v\n", err)
		os.Exit(1)
	}
}

type ByPinyin []string

func (s ByPinyin) Len() int      { return len(s) }
func (s ByPinyin) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s ByPinyin) Less(i, j int) bool {
	a, _ := UTF82GBK(s[i])
	b, _ := UTF82GBK(s[j])
	bLen := len(b)
	for idx, chr := range a {
		if idx > bLen-1 {
			return false
		}
		if chr != b[idx] {
			return chr < b[idx]
		}
	}
	return true
}

// UTF82GBK : transform UTF8 rune into GBK byte array
func UTF82GBK(src string) ([]byte, error) {
	GB18030 := simplifiedchinese.All[0]
	return io.ReadAll(transform.NewReader(bytes.NewReader([]byte(src)), GB18030.NewEncoder()))
}

// 字段与释义对应关系
var FieldMap = map[string]interface{}{
	"title":            "公告标题",
	"content":          "公告内容",
	"href":             "公告地址",
	"jybxhref":         "剑鱼标讯地址",
	"detail":           "公告正文",
	"area":             "省份",
	"city":             "城市",
	"publishtime":      "发布时间",
	"projectname":      "项目名称",
	"projectcode":      "项目编号",
	"buyer":            "采购单位",
	"buyerperson":      "采购单位联系人",
	"buyertel":         "采购单位联系电话",
	"budget":           "预算（元）",
	"bidopentime":      "开标日期",
	"s_winner":         "中标单位",
	"bidamount":        "中标金额（元）",
	"agency":           "招标代理机构",
	"projectscope":     "项目范围",
	"currency":         "币种",
	"subtype":          "公告类别",
	"toptype":          "公告类别一级",
	"industry":         "行业",
	"matchkey":         "信息匹配词",
	"matchtype":        "匹配字段",
	"priority":         "信息重要程度",
	"department":       "部门名称",
	"excludetype":      "排除类型",
	"winner_organcode": "中标单位征信代码",
	"subscopeclass":    "信息二级分类",
	"bid2017":          "2017年中标数",
	"bid2018":          "2018年中标数",
	"bid2019":          "2019年中标数",
	"bid2020":          "2020年中标数",
	"winner_credit_no": "中标单位统一社会信用代码",
	"buyer_credit_no":  "采购单位统一社会信用代码",
	"stock_name":       "股东名单",
	"business_scope":   "经营范围",
	"establish_date":   "注册时间",
	"capital":          "注册资金",
	"winneraddr":       "中标单位地址",
	"purchasing":       "中标产品信息",
	"rank":             "医院等级",
	"winnertel":        "中标单位联系电话",
	"winnerperson":     "中标单位联系人",
	"district":         "区县",
	"legal_person":     "中标企业联系人",
	"company_phone":    "中标企业联系电话",
	"company_email":    "中标企业联系邮箱",
	"company_address":  "中标企业注册地址",
	"tagname":          "标签",
	"filetext":         "附件内容",
	"buyerclass":       "采购单位类型",
	"rulename":         "规则名称",
	"departname":       "部门名称",
	"projectId":        "项目标识",
	"item":             "分类",
	"winner_type":      "中标人模型",
	"buyer_type":       "行业归属",
	"purchasinglist":   "产品信息",
	"ruleid":           "规则标识",
	"filehref":         "附件链接",
	"winner_style":     "中标类型",
	"agency_fee":       "中标服务费",
	"bidway":           "投标方式",
	"docamount":        "招标文件购买金额",
	"bidtype":          "采购方式",
	"bidendtime":       "投标截止日期",
	"yewu_type":        "业务类型",
	"agencyrate":       "招标代理费率",
	"s_subscopeclass":  "公告行业二级",
	"s_topscopeclass":  "公告行业一级",
	"createtime":       "创建时间",
	"buyeraddr":        "采购单位地址",
	"agencyperson":     "招标代理机构联系人",
	"agencytel":        "招标代理机构联系方式",
	"agencyaddr":       "招标代理机构地址",
	"company_area":     "中标单位省份",
	"company_city":     "中标单位城市",
	"company_district": "中标单位区县",
	"agency_credit_no": "招标代理机构统一社会信用代码",
	"contact":          "中标单位历史联系信息",
	"id":               "唯一标识",
}