package main import ( "fmt" "jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic" "jygit.jydev.jianyu360.cn/data_processing/common_utils/log" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mysqldb" "os" "proposed_project/config" "sync" ) var ( MgoBid, MgoPro *mongodb.MongodbSim //mongodb连接 MysqlTool, MysqlTool1 *mysqldb.Mysql saveSize int LastId string //增量拟建数据最后id TaskSingle bool id string pici int64 // 拟建增量数据pici时间 updatePool chan []map[string]interface{} updateSp chan bool savePool chan map[string]interface{} saveSp chan bool AllPidMap map[string]*ID AllPidMapLock sync.Mutex AllPnMap map[string]*Pname AllPnMapLock sync.Mutex AllCodeMap map[string]string AllCodeMapLock sync.Mutex ) func init() { config.Init("./common.toml") InitLog() InitMgo() TaskSingle = true saveSize = 200 AllPidMap = make(map[string]*ID, 20000) AllPnMap = make(map[string]*Pname, 20000) AllCodeMap = make(map[string]string, 50000) updatePool = make(chan []map[string]interface{}, 5000) updateSp = make(chan bool, 1) savePool = make(chan map[string]interface{}, 5000) saveSp = make(chan bool, 2) } func InitLog() { logcfg := config.Conf.Log err := log.InitLog( log.Path(logcfg.LogPath), log.Level(logcfg.LogLevel), log.Compress(logcfg.Compress), log.MaxSize(logcfg.MaxSize), log.MaxBackups(logcfg.MaxBackups), log.MaxAge(logcfg.MaxAge), log.Format(logcfg.Format), ) if err != nil { fmt.Printf("InitLog failed: %v\n", err) os.Exit(1) } } func InitMgo() { MgoBid = &mongodb.MongodbSim{ MongodbAddr: config.Conf.DB.MongoB.Addr, DbName: config.Conf.DB.MongoB.Dbname, Size: config.Conf.DB.MongoB.Size, UserName: config.Conf.DB.MongoB.User, Password: config.Conf.DB.MongoB.Password, } MgoBid.InitPool() MgoPro = &mongodb.MongodbSim{ MongodbAddr: config.Conf.DB.MongoP.Addr, DbName: config.Conf.DB.MongoP.Dbname, Size: config.Conf.DB.MongoP.Size, UserName: config.Conf.DB.MongoP.User, Password: config.Conf.DB.MongoP.Password, } MgoPro.InitPool() } func InitMysql() { dbcfg := config.Conf.DB.Mysql MysqlTool = &mysqldb.Mysql{ Address: dbcfg.Addr, DBName: dbcfg.DbnameMedical, UserName: dbcfg.User, PassWord: dbcfg.Password, } MysqlTool.Init() MysqlTool1 = &mysqldb.Mysql{ Address: dbcfg.Addr, DBName: dbcfg.DbnameBasic, UserName: dbcfg.User, PassWord: dbcfg.Password, } MysqlTool1.Init() } func InitEs() { Es = &elastic.Elastic{ S_esurl: config.Conf.DB.Es.Addr, I_size: config.Conf.DB.Es.Size, Username: config.Conf.DB.Es.User, Password: config.Conf.DB.Es.Password, } Es.InitElasticSize() } type Pname struct { Id map[string]string Lock sync.Mutex } type Key struct { Id string Lock sync.Mutex } type ID struct { Lock sync.Mutex P *Project } // Project 内存 项目信息 type Project struct { Id string `json:"_id"` Ids []string `json:"ids,omitempty"` FirstTime int64 `json:"firsttime,omitempty"` //项目的最早时间 LastTime int64 `json:"lasttime,omitempty"` //项目的最后时间 ProjectName string `json:"projectname,omitempty"` //项目名称 Owner string `json:"owner"` ApproveCode string `json:"approvecode"` // 审批代码 ApproveNumber string `json:"approvenumber"` ApproveDept string `json:"approvedept"` TotalInvestment string `json:"total_investment"` ProjectAddr string `json:"projectaddr"` Funds string `json:"funds"` ProjectPeriod string `json:"projectperiod"` ProjectScale string `json:"project_scale"` ProjectStartDate int64 `json:"project_startdate"` ProjectCompleteDate int64 `json:"project_completedate"` ConstructionArea string `json:"construction_area"` FollowNum int `json:"follow_num"` FloorArea string `json:"floor_area"` ProjectPhone string `json:"project_phone"` ProjectPerson string `json:"project_person"` Area string `json:"area"` //地区 City string `json:"city"` //地市 District string `json:"district"` //区县 Site string `json:"site"` } // Info 拟在建数据信息 type Info struct { Id string `json:"_id"` Href string `json:"href"` Publishtime int64 `json:"publishtime"` Title string `json:"title"` ProjectName string `json:"projectname"` Area string `json:"area"` City string `json:"city"` District string `json:"district"` Infoformat int `json:"infoformat"` Site string `json:"site"` Topscopeclass []string `json:"topscopeclass"` Subscopeclass []string `json:"subscopeclass"` Buyerclass string `json:"buyerclass"` Owner string `json:"owner"` ApproveCode string `json:"approvecode"` ApproveDept string `json:"approvedept"` ApproveStatus string `json:"approvestatus"` ApproveTime string `json:"approvetime"` ApproveNumber string `json:"approvenumber"` ApproveContent string `json:"approvecontent"` ProjectType string `json:"projecttype"` ApproveCity string `json:"approvecity"` ProjectScale string `json:"project_scale"` TotalInvestment string `json:"total_investment"` ProjectAddr string `json:"projectaddr"` ProjectPeriod string `json:"projectperiod"` Funds string `json:"funds"` ProjectStartDate int64 `json:"project_startdate"` ProjectCompleteDate int64 `json:"project_completedate"` ConstructionArea string `json:"construction_area"` FloorArea string `json:"floor_area"` ProjectPhone string `json:"project_phone"` ProjectPerson string `json:"project_person"` } var Project_Stage = []string{"可研", "初设", "环评", "审批备案", "设计", "施工准备", "施工", "竣工验收"}