package util import ( "app.yhyue.com/moapp/jybase/common" elastic "app.yhyue.com/moapp/jybase/es" "app.yhyue.com/moapp/jybase/log" "app.yhyue.com/moapp/jybase/mongodb" "app.yhyue.com/moapp/jybase/mysql" "app.yhyue.com/moapp/jybase/redis" "bytes" "fmt" "github.com/Chain-Zhang/pinyin" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/transform" "io" "os" "sort" ) var ( Sysconfig map[string]interface{} //配置文件 Mgo *mongodb.MongodbSim MgoCus *mongodb.MongodbSim //企业服务库 MgoEn *mongodb.MongodbSim EsIndex, EsType string EsAllIndex string InterimIndex string TotalIndex string MgoEnC string Subday float64 PreviewHref string //数据预览地址 CustomerApi string LatestId = "" //起始id MgoTag *mongodb.MongodbSim //标签库连接 MgoSave *mongodb.MongodbSim //数据保存库连接 MgoBuyer *mongodb.MongodbSim //医院等级信息 MgoEnps *mongodb.MongodbSim //企业信息 MgoExtract *mongodb.MongodbSim //抽取 MgoBidding *mongodb.MongodbSim //bidding BiddingColl string SaveColl string HospColl string BuyerEntColl string EnpsColl string Es, EsCus elastic.Es //es Index string Itype string IndexCus string TaskTime int64 //定时任务时间 //历史数据 SId string EId string CustomerName string SaveCollProject string SaveCollbid string SaveUserMail string ExtractColl string JyMysql *mysql.Mysql JyPushMysql *mysql.Mysql JyMysqls *mysql.Mysql EntMysql *mysql.Mysql MgoSite *mongodb.MongodbSim SiteColl string IsNewSql int RpcJyServer, RpcProServer string ) var ( TopTypeArr []string //公告一级分类 SubTypeArr []string //公告二级分类 Province []string //省 ProvinceCitys map[string][]string //省:[市1,市2] CityDistricts map[string][]string //市:[区1,区2] BuyerClass []string //采购单位行业 BuyerClassMap map[string][]string //采购单位行业带拼音字母 MatchTypeMap *[]map[string]interface{} //匹配方式 MatchTypeMap2 map[string]interface{} //匹配方式 MatchTypeMap3 *[]map[string]interface{} //匹配方式 ExistFiled *[]map[string]interface{} //存在的字段 ScopeClassMap *[]map[string]interface{} //公告行业 TagFieldMap *[]map[string]interface{} //标签属性值 TaskTypeMap *[]map[string]interface{} //任务类型 BuryClassType map[string]interface{} ProvinceMap map[string]interface{} SiteArr []string //网站名称 ) func InitMgoPool() { Mgo = &mongodb.MongodbSim{ MongodbAddr: common.ObjToString(Sysconfig["mgodb"]), Size: common.IntAll(Sysconfig["dbsize"]), DbName: common.ObjToString(Sysconfig["dbname"]), } Mgo.InitPool() customer := Sysconfig["customer"].(map[string]interface{}) CustomerApi = common.ObjToString(customer["api"]) MgoCus = &mongodb.MongodbSim{ MongodbAddr: common.ObjToString(customer["mgodb"]), Size: common.IntAll(customer["dbsize"]), DbName: common.ObjToString(customer["dbname"]), } MgoCus.InitPool() } func initMgoEn() { MgoEn = &mongodb.MongodbSim{ MongodbAddr: common.ObjToString(Sysconfig["mgodben"]), Size: common.IntAll(Sysconfig["dbsize"]), DbName: common.ObjToString(Sysconfig["dbnameen"]), } MgoEnC = common.ObjToString(Sysconfig["mongoenc"]) MgoEn.InitPool() } func InitOther() { initCitys() initInfoType() initBuyerClass() initMatchType() initExitField() initScopeClass() initTaskType() initEs() InitLog() // initMgoEn() Subday = common.Float64All(Sysconfig["subday"]) PreviewHref = common.ObjToString(Sysconfig["preview_href"]) initTagField() initdb() initBuryClassType() initProvince() initSite() IsNewSql = common.IntAll(Sysconfig["isNewSql"]) RpcJyServer = common.ObjToString(Sysconfig["rpcServer_jy"]) RpcProServer = common.ObjToString(Sysconfig["rpcServer_project"]) } func initCitys() { //map初始化 if ProvinceCitys == nil { ProvinceCitys = make(map[string][]string) } if CityDistricts == nil { CityDistricts = make(map[string][]string) } datas, _ := Mgo.Find("code_city", `{}`, `{"_id":1}`, `{}`, false, -1, -1) for k, data := range *datas { brief := data["brief"].(string) if k != 0 { Province = append(Province, brief) } if data["citys"] != nil { citys := data["citys"].([]interface{}) citytmp := []string{} for _, city := range citys { cityMap := city.(map[string]interface{}) for cname, districts := range cityMap { citytmp = append(citytmp, cname) tmp := districts.([]interface{}) districtarr := common.ObjArrToStringArr(tmp) CityDistricts[cname] = districtarr } } ProvinceCitys[brief] = citytmp } } Province = append(Province, common.ObjToString((*datas)[0]["brief"])) } func initSite() { if SiteArr == nil { SiteArr = []string{} } datas, _ := MgoSite.Find("site", `{}`, `{"_id":1}`, `{}`, false, -1, -1) for _, data := range *datas { name := data["site"].(string) SiteArr = append(SiteArr, name) } //log.Println("网站名称", SiteArr) } func initInfoType() { datas, _ := Mgo.Find("code_infotype", `{}`, `{"_id":1}`, `{}`, false, -1, -1) for _, data := range *datas { name := data["name"].(string) TopTypeArr = append(TopTypeArr, name) child := common.ObjArrToStringArr(data["child"].([]interface{})) SubTypeArr = append(SubTypeArr, child...) } //common.Debug(InfoTypeArr, InfoTypeMap) } func initBuyerClass() { if BuyerClassMap == nil { BuyerClassMap = make(map[string][]string) } datas, _ := Mgo.Find("code_buyerclass", `{}`, `{"name":1}`, `{}`, false, -1, -1) for _, data := range *datas { name := data["name"].(string) BuyerClass = append(BuyerClass, name) } sort.Sort(ByPinyin(BuyerClass)) for _, name := range BuyerClass { str, _ := pinyin.New(name).Mode(pinyin.InitialsInCapitals).Convert() //fmt.Println(str, err) zimu := str[0:1] BuyerClassMap[zimu] = append(BuyerClassMap[zimu], name) } } func initMatchType() { //map初始化 if MatchTypeMap2 == nil { MatchTypeMap2 = make(map[string]interface{}) } MatchTypeMap, _ = Mgo.Find("code_matchtype", `{}`, `{"_id":1}`, `{}`, false, -1, -1) MatchTypeMap3, _ = Mgo.Find("code_matchtype", `{"type": 1}`, nil, nil, false, -1, -1) for _, d := range *MatchTypeMap { code := common.ObjToString(d["code"]) name := common.ObjToString(d["name"]) MatchTypeMap2[code] = name } } func initExitField() { ExistFiled, _ = Mgo.Find("code_field", `{}`, `{"_id":1}`, `{}`, false, -1, -1) for _, data := range *ExistFiled { str, _ := pinyin.New(data["name"].(string)).Mode(pinyin.InitialsInCapitals).Convert() //fmt.Println(str, err) //zimu := str[0:1] data["zimu"] = str } for i := 0; i < len(*ExistFiled); i++ { for j := 0; j < len(*ExistFiled)-i-1; j++ { if (*ExistFiled)[j]["zimu"].(string) > (*ExistFiled)[j+1]["zimu"].(string) { (*ExistFiled)[j], (*ExistFiled)[j+1] = (*ExistFiled)[j+1], (*ExistFiled)[j] } } } //for _, data := range *datas { // name := data["name"].(string) // ExistFiled = append(ExistFiled, name) //} } func initScopeClass() { ScopeClassMap, _ = Mgo.Find("code_scopeclass", `{}`, `{"_id": 1}`, `{}`, false, -1, -1) } func initTaskType() { TaskTypeMap, _ = Mgo.Find("code_task", `{}`, `{"_id": 1}`, `{}`, false, -1, -1) } func initEs() { EsIndex, EsType = common.ObjToString(Sysconfig["elasticsearch_index"]), common.ObjToString(Sysconfig["elasticsearch_type"]) EsAllIndex = common.ObjToString(Sysconfig["elasticsearch_all_index"]) TotalIndex = common.ObjToString(Sysconfig["elasticsearch_total_index"]) InterimIndex = common.ObjToString(Sysconfig["elasticsearch_interim_index"]) // elastic.InitElasticSize(common.ObjToString(Sysconfig["elasticsearch"]), common.IntAllDef(common.Int64All(Sysconfig["elasticPoolSize"]), 30)) es := Sysconfig["es"].(map[string]interface{}) Es = elastic.NewEs(common.ObjToString(es["version"]), common.ObjToString(es["addr"]), common.IntAllDef(es["pool"], 15), common.ObjToString(es["userName"]), common.ObjToString(es["password"])) Index = common.ObjToString(es["index"]) Itype = common.ObjToString(es["itype"]) esCm := Sysconfig["es_customer"].(map[string]interface{}) EsCus = elastic.NewEs(common.ObjToString(esCm["version"]), common.ObjToString(esCm["addr"]), common.IntAllDef(esCm["pool"], 15), common.ObjToString(esCm["userName"]), common.ObjToString(esCm["password"])) IndexCus = common.ObjToString(esCm["index"]) } func initTagField() { TagFieldMap, _ = Mgo.Find("code_tagfield", `{}`, `{"_id": 1}`, `{}`, false, -1, -1) } func initBuryClassType() { infoType := JyMysql.SelectBySql("SELECT id,name FROM infotype") BuryClassType = make(map[string]interface{}) for _, val := range *infoType { k := common.ObjToString(val["name"]) BuryClassType[k] = val["id"] } } func initProvince() { proArr := JyMysql.SelectBySql("SELECT id,name FROM province") ProvinceMap = make(map[string]interface{}) for _, val := range *proArr { k := common.ObjToString(val["name"]) ProvinceMap[k] = val["id"] } } func initdb() { //mgo save save := Sysconfig["save"].(map[string]interface{}) SaveColl = common.ObjToString(save["coll"]) SaveCollProject = common.ObjToString(save["coll_project"]) SaveCollbid = common.ObjToString(save["coll_bid"]) SaveUserMail = common.ObjToString(save["coll_usermail"]) MgoSave = &mongodb.MongodbSim{ MongodbAddr: common.ObjToString(save["addr"]), DbName: common.ObjToString(save["db"]), Size: common.IntAllDef(save["size"], 15), } MgoSave.InitPool() //mgo buyerinfo buyerinfo := Sysconfig["buyerinfo"].(map[string]interface{}) HospColl = common.ObjToString(buyerinfo["hospcoll"]) BuyerEntColl = common.ObjToString(buyerinfo["buyerentcoll"]) MgoBuyer = &mongodb.MongodbSim{ MongodbAddr: common.ObjToString(buyerinfo["addr"]), DbName: common.ObjToString(buyerinfo["db"]), Size: common.IntAllDef(buyerinfo["size"], 15), UserName: common.ObjToString(buyerinfo["username"]), Password: common.ObjToString(buyerinfo["password"]), ReplSet: common.ObjToString(buyerinfo["replSet"]), } MgoBuyer.InitPool() //mgo enterprise enterprise := Sysconfig["enterprise"].(map[string]interface{}) EnpsColl = common.ObjToString(enterprise["coll"]) MgoEnps = &mongodb.MongodbSim{ MongodbAddr: common.ObjToString(enterprise["addr"]), DbName: common.ObjToString(enterprise["db"]), Size: common.IntAllDef(enterprise["size"], 15), UserName: common.ObjToString(buyerinfo["username"]), Password: common.ObjToString(buyerinfo["password"]), ReplSet: common.ObjToString(buyerinfo["replSet"]), } MgoEnps.InitPool() extract := Sysconfig["extract"].(map[string]interface{}) ExtractColl = common.ObjToString(extract["coll"]) MgoExtract = &mongodb.MongodbSim{ MongodbAddr: common.ObjToString(extract["addr"]), DbName: common.ObjToString(extract["db"]), Size: common.IntAllDef(extract["size"], 15), } MgoExtract.InitPool() //bidding库 bidding := Sysconfig["bidding_mgo"].(map[string]interface{}) BiddingColl = common.ObjToString(bidding["coll"]) MgoBidding = &mongodb.MongodbSim{ MongodbAddr: common.ObjToString(bidding["addr"]), DbName: common.ObjToString(bidding["db"]), Size: common.IntAll(bidding["size"]), UserName: common.ObjToString(bidding["username"]), Password: common.ObjToString(bidding["password"]), ReplSet: common.ObjToString(bidding["replSet"]), } MgoBidding.InitPool() site := Sysconfig["site_mgo"].(map[string]interface{}) SiteColl = common.ObjToString(site["coll"]) MgoSite = &mongodb.MongodbSim{ MongodbAddr: common.ObjToString(site["addr"]), Size: common.IntAll(site["size"]), DbName: common.ObjToString(site["db"]), } MgoSite.InitPool() //es // Es = &elastic.Elastic{ // S_esurl: common.ObjToString(es["addr"]), // I_size: common.IntAllDef(es["pool"], 15), // } redis.InitRedis(common.ObjToString(Sysconfig["redis_addrs"])) jyMysql := common.ObjToMap(Sysconfig["jyPushMysql"]) JyMysql = &mysql.Mysql{ Address: (*jyMysql)["address"].(string), UserName: (*jyMysql)["username"].(string), PassWord: (*jyMysql)["password"].(string), DBName: (*jyMysql)["dbName"].(string), } JyMysql.Init() jyPushMysql := common.ObjToMap(Sysconfig["pushMysql"]) JyPushMysql = &mysql.Mysql{ Address: (*jyPushMysql)["address"].(string), UserName: (*jyPushMysql)["username"].(string), PassWord: (*jyPushMysql)["password"].(string), DBName: (*jyPushMysql)["dbName"].(string), } JyPushMysql.Init() jyMysqls := common.ObjToMap(Sysconfig["jyMysql"]) JyMysqls = &mysql.Mysql{ Address: (*jyMysqls)["address"].(string), UserName: (*jyMysqls)["username"].(string), PassWord: (*jyMysqls)["password"].(string), DBName: (*jyMysqls)["dbName"].(string), } JyMysqls.Init() entMysql := common.ObjToMap(Sysconfig["entMysql"]) EntMysql = &mysql.Mysql{ Address: (*entMysql)["address"].(string), UserName: (*entMysql)["username"].(string), PassWord: (*entMysql)["password"].(string), DBName: (*entMysql)["dbName"].(string), } EntMysql.Init() } func InitLog() { err := log.InitLog( log.Path(common.ObjToString(Sysconfig["logpath"])), log.Level("debug"), log.Compress(true), log.MaxSize(10), // M log.MaxBackups(10), log.MaxAge(7), log.Format("text"), // text or json output ) if err != nil { fmt.Printf("InitLog failed: %v\n", err) os.Exit(1) } } type ByPinyin []string func (s ByPinyin) Len() int { return len(s) } func (s ByPinyin) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s ByPinyin) Less(i, j int) bool { a, _ := UTF82GBK(s[i]) b, _ := UTF82GBK(s[j]) bLen := len(b) for idx, chr := range a { if idx > bLen-1 { return false } if chr != b[idx] { return chr < b[idx] } } return true } // UTF82GBK : transform UTF8 rune into GBK byte array func UTF82GBK(src string) ([]byte, error) { GB18030 := simplifiedchinese.All[0] return io.ReadAll(transform.NewReader(bytes.NewReader([]byte(src)), GB18030.NewEncoder())) } // 字段与释义对应关系 var FieldMap = map[string]interface{}{ "title": "公告标题", "content": "公告内容", "href": "公告地址", "jybxhref": "剑鱼标讯地址", "detail": "公告正文", "area": "省份", "city": "城市", "publishtime": "发布时间", "projectname": "项目名称", "projectcode": "项目编号", "buyer": "采购单位", "buyerperson": "采购单位联系人", "buyertel": "采购单位联系电话", "budget": "预算(元)", "bidopentime": "开标日期", "s_winner": "中标单位", "bidamount": "中标金额(元)", "agency": "招标代理机构", "projectscope": "项目范围", "currency": "币种", "subtype": "公告类别", "toptype": "公告类别一级", "industry": "行业", "matchkey": "信息匹配词", "matchtype": "匹配字段", "priority": "信息重要程度", "department": "部门名称", "excludetype": "排除类型", "winner_organcode": "中标单位征信代码", "subscopeclass": "信息二级分类", "bid2017": "2017年中标数", "bid2018": "2018年中标数", "bid2019": "2019年中标数", "bid2020": "2020年中标数", "winner_credit_no": "中标单位统一社会信用代码", "buyer_credit_no": "采购单位统一社会信用代码", "stock_name": "股东名单", "business_scope": "经营范围", "establish_date": "注册时间", "capital": "注册资金", "winneraddr": "中标单位地址", "purchasing": "中标产品信息", "rank": "医院等级", "winnertel": "中标单位联系电话", "winnerperson": "中标单位联系人", "district": "区县", "legal_person": "中标企业联系人", "company_phone": "中标企业联系电话", "company_email": "中标企业联系邮箱", "company_address": "中标企业注册地址", "tagname": "标签", "filetext": "附件内容", "buyerclass": "采购单位类型", "rulename": "规则名称", "departname": "部门名称", "projectId": "项目标识", "item": "分类", "winner_type": "中标人模型", "buyer_type": "行业归属", "purchasinglist": "产品信息", "ruleid": "规则标识", "filehref": "附件链接", "winner_style": "中标类型", "agency_fee": "中标服务费", "bidway": "投标方式", "docamount": "招标文件购买金额", "bidtype": "采购方式", "bidendtime": "投标截止日期", "yewu_type": "业务类型", "agencyrate": "招标代理费率", "s_subscopeclass": "公告行业二级", "s_topscopeclass": "公告行业一级", "createtime": "创建时间", "buyeraddr": "采购单位地址", "agencyperson": "招标代理机构联系人", "agencytel": "招标代理机构联系方式", "agencyaddr": "招标代理机构地址", "company_area": "中标单位省份", "company_city": "中标单位城市", "company_district": "中标单位区县", "agency_credit_no": "招标代理机构统一社会信用代码", "contact": "中标单位历史联系信息", "id": "唯一标识", }