jy-data-extract/qyxy/qyxy_std/init.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"mongodb"
	"qfw/util"
	"qfw/util/elastic"
)

var (
	Sysconfig  map[string]interface{}
	MongoTool  *mongodb.MongodbSim
	MongoTool1 *mongodb.MongodbSim
	MysqlTool  *Mysql
	DbSave     string

	updatePool chan []map[string]interface{}
	updateSp   chan bool
	saveSize   int
	savePool   chan map[string]interface{}
	saveSp     chan bool

	QyStypeMap       map[string]string
	CompanyStatusMap map[string]string
	AddressMap       map[string]*City
	AddressOldMap    map[string]*City

	Es *elastic.Elastic
)

func init() {
	util.ReadConfig(&Sysconfig)

	DbSave = util.ObjToString(Sysconfig["dbSave"])
	MongoTool = &mongodb.MongodbSim{
		MongodbAddr: util.ObjToString(Sysconfig["dbServer"]),
		Size:        util.IntAll(Sysconfig["dbSize"]),
		DbName:      util.ObjToString(Sysconfig["dbName"]),
		UserName:    util.ObjToString(Sysconfig["uname"]),
		Password:    util.ObjToString(Sysconfig["upwd"]),
	}
	MongoTool.InitPool()

	MongoTool1 = &mongodb.MongodbSim{
		MongodbAddr: "172.17.4.181:27001", // 172.17.4.181:27001
		Size:        10,
		DbName:      "mixdata",
	}
	MongoTool1.InitPool()

	//MysqlTool = &Mysql{
	//	Address: "172.17.4.181:3308",
	//	UserName: "root",
	//	PassWord: "root123",
	//	DBName: "ic",
	//}
	//MysqlTool.Init()

	Es = &elastic.Elastic{
		S_esurl: "http://127.0.0.1:9800", //http://172.17.145.170:9800
		I_size:  10,
	}
	Es.InitElasticSize()

	saveSize = 200
	updatePool = make(chan []map[string]interface{}, 5000)
	updateSp = make(chan bool, 5)
	savePool = make(chan map[string]interface{}, 5000)
	saveSp = make(chan bool, 5)

	InitQyStype()
	InitCompanyStatus()
	InitAddress()
	err := seg.LoadDictionary("dict.txt")
	if err != nil {
		log.Fatal("结巴分词出错...")
		return
	}
	//redis.InitRedis1("qyxy=172.17.4.189:8379", 3)
}

func InitQyStype() {
	defer util.Catch()
	util.Debug("Init QyStype...")
	QyStypeMap = map[string]string{}
	qystype, _ := MongoTool.Find("qystype", nil, nil, nil, false, -1, -1)
	for _, tmp := range *qystype {
		name := util.ObjToString(tmp["name"])
		prename := util.ObjToString(tmp["prename"])
		QyStypeMap[name] = prename
	}
}

func InitCompanyStatus() {
	defer util.Catch()
	util.Debug("Init CompanyStatus...")
	CompanyStatusMap = map[string]string{}
	status, _ := MongoTool.Find("company_status", nil, nil, nil, false, -1, -1)
	for _, tmp := range *status {
		old_status := util.ObjToString(tmp["old"])
		new_status := util.ObjToString(tmp["new"])
		CompanyStatusMap[old_status] = new_status
	}
}

type City struct {
	Code     string `json:"code"`
	Province string `json:"province"`
	City     string `json:"city"`
	District string `json:"district"`
}

// CodeMap 区域code补全
var CodeMap = map[int]string{
	2: "0000",
	4: "00",
}

func InitAddress() {
	defer util.Catch()
	util.Debug("Init Address...")
	AddressMap = map[string]*City{}
	AddressOldMap = map[string]*City{}

	sess := MongoTool.GetMgoConn()
	defer MongoTool.DestoryMongoConn(sess)
	result := sess.DB("mixdata").C("address_new_2020").Find(nil).Iter()
	count := 0
	for tmp := make(map[string]interface{}); result.Next(&tmp); count++ {
		if count%50000 == 0 {
			util.Debug("current---", count)
		}
		code := util.ObjToString(tmp["code"])
		codeLen := len(code)
		if codeLen > 6 {
			continue
		}
		if t_code := CodeMap[codeLen]; t_code != "" {
			code = code + t_code
		}
		remark := fmt.Sprint(tmp["Remarks"])
		city := &City{}
		tmpjson, err := json.Marshal(tmp)
		if err == nil {
			json.Unmarshal(tmpjson, city)
		}
		if remark == "已作废" {
			AddressOldMap[code] = city
		} else {
			AddressMap[code] = city
		}
	}
	util.Debug("Init Address end...", len(AddressMap), len(AddressOldMap))
}