package main import ( "encoding/json" "fmt" "github.com/garyburd/redigo/redis" hisRedis "github.com/go-redis/redis" "gopkg.in/mgo.v2/bson" es "gopkg.in/olivere/elastic.v1" "log" mu "mfw/util" "net" "qfw/common/src/qfw/util/elastic" "qfw/util" "qfw/util/mongodb" "regexp" "strconv" "strings" "time" ) var ( Config = make(map[string]string) Fields, BuyerFields, AgencyFields []string SourceClient, FClient *mongodb.MongodbSim RedisPool redis.Pool HisRedisPool *hisRedis.Client Addrs = make(map[string]interface{}, 0) //省市县 udpclient mu.UdpClient //udp对象 ElasticClientIndex, ElasticClientType string Reg_xing = regexp.MustCompile(`\*{1,}`) Reg_person = regexp.MustCompile("[\u4E00-\u9FA5\\s]+") Reg_tel = regexp.MustCompile(`^[0-9\-\s]*$`) EsConn *es.Client Updport int CPool chan bool //his_redis db redis_winner_db, redis_buyer_db, redis_agency_db int //异常表正则匹配处理 WinnerRegOk, WinnerRegErr, AgencyRegOk, AgencyRegErr, BuerRegOk, BuyerRegErr []regexp.Regexp ) /** 新增 初始化 */ func init() { log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) util.ReadConfig(&Config) log.Println(Config) var err error cpnum, err := strconv.Atoi(Config["chan_pool_num"]) if err != nil { log.Fatalln(err) } CPool = make(chan bool, cpnum) Fields = []string{"_id", "contact", "partners", "business_scope", "company_address", "capital", "establish_date", "legal_person", "company_type", "district", "city", "province", "area_code", "credit_no", "company_name", "history_name", "wechat_accounts", "alias", "website", "report_websites", "industry"} BuyerFields = []string{"_id", "contact", "type", "ranks", "buyerclass", "address", "district", "city", "province", "area_code", "credit_no", "buyer_name", "history_name", "wechat_accounts", "website", "report_websites"} AgencyFields = []string{"_id", "contact", "type", "ranks", "address", "district", "city", "province", "area_code", "credit_no", "agency_name", "history_name", "wechat_accounts", "website", "report_websites"} pool_size, _ := strconv.Atoi(Config["pool_size"]) //mongo init SourceClient = new(mongodb.MongodbSim) SourceClient.MongodbAddr = Config["mgoinit"] SourceClient.Size = pool_size SourceClient.DbName = Config["mgodb_bidding"] //mongodbSim.DbName = "qfw" SourceClient.InitPool() FClient = new(mongodb.MongodbSim) FClient.MongodbAddr = Config["mgourl"] FClient.Size = pool_size FClient.DbName = Config["mgodb_extract_kf"] //mongodbSim.DbName = "qfw" FClient.InitPool() FClientmgoConn := FClient.GetMgoConn() defer FClient.DestoryMongoConn(FClientmgoConn) //加载省市县代码 cursor2 := FClientmgoConn.DB(Config["mgodb_extract_kf"]).C("address").Find(bson.M{}).Select(bson.M{"province": 1, "code": 1, "city": 1, "district": 1}).Iter() //defer FClient.Connect(cc) if cursor2 == nil { log.Fatalln(cursor2) } tmp := make(map[string]interface{}) for cursor2.Next(&tmp) { code := tmp["code"] if code != nil && strings.TrimSpace(code.(string)) != "" { Addrs[fmt.Sprint(code)] = tmp } } log.Println(len(Addrs)) //es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false)) //es init elastic.InitElasticSize(Config["elasticsearch"], 50) EsConn = elastic.GetEsConn() defer elastic.DestoryEsConn(EsConn) //redis RedisPool = redis.Pool{ MaxIdle: 50, IdleTimeout: 10 * time.Second, Dial: func() (redis.Conn, error) { conn, e := redis.Dial("tcp", Config["redis"]) if e != nil { return conn, e } _, err = conn.Do("SELECT", "1") if err != nil { return nil, err } return conn, nil }} c := RedisPool.Get() if _, err := c.Do("PING"); err != nil { log.Fatalln("redis err:", err) } c.Close() HisRedisPool = hisRedis.NewClient(&hisRedis.Options{ Addr: Config["his_redis"], DB: 1, DialTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 30 * time.Second, PoolSize: 30, MinIdleConns: 20, PoolTimeout: 30 * time.Second, }) redis_winner_db, _ = strconv.Atoi(Config["redis_winner_db"]) redis_buyer_db, _ = strconv.Atoi(Config["redis_buyer_db"]) redis_agency_db, _ = strconv.Atoi(Config["redis_agency_db"]) iniReg() } func iniReg() { FClientmgoConnReg := FClient.GetMgoConn() defer FClient.DestoryMongoConn(FClientmgoConnReg) findReg, b := FClient.Find(Config["mgo_qyk_reg"], bson.M{"delete": false, "isuse": true}, bson.M{"_id": 1}, nil, false, -1, 0) if !b { log.Fatalln("查询正则失败") } for _, v := range (*findReg) { s_field, ok := v["s_field"].(string) //字段 s_rule, ok2 := v["s_rule"].(string) //正则 s_type, ok3 := v["s_type"].(string) //ok or err if !ok || !ok2 || !ok3 || s_field == "" || s_rule == "" || s_type == "" { continue } regtmp := regexp.MustCompile(s_rule) if s_field == "winner" { if s_type=="ok"{ WinnerRegOk = append(WinnerRegOk, *regtmp) }else if s_type=="err"{ WinnerRegErr = append(WinnerRegErr, *regtmp) } } else if s_field == "buyer" { if s_type=="ok"{ BuerRegOk = append(BuerRegOk, *regtmp) }else if s_type=="err"{ BuyerRegErr = append(BuyerRegErr, *regtmp) } } else if s_field == "agency" { if s_type=="ok"{ AgencyRegOk = append(AgencyRegOk, *regtmp) }else if s_type=="err"{ AgencyRegErr = append(AgencyRegErr, *regtmp) } } } log.Println(len(WinnerRegOk), len(WinnerRegErr), len(BuerRegOk), len(BuyerRegErr), len(AgencyRegOk), len(AgencyRegErr)) } func main() { //udp updport := Config["udpport"] Updport, _ = strconv.Atoi(Config["port"]) udpclient = mu.UdpClient{Local: updport, BufSize: 1024} udpclient.Listen(processUdpMsg) log.Println("Udp服务监听", updport) log.Println("发送端口port:", Updport) go TimedTaskWinner() //定时任务 go TimedTaskBuyer() //定时任务 go TimedTaskAgency() //定时任务 c := make(chan int, 1) <-c } func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) { log.Println(act, string(data), ra) switch act { case mu.OP_TYPE_DATA: //上个节点的数据 //从表中开始处理生成企业数据 tmp := new(map[string]interface{}) err := json.Unmarshal(data, &tmp) if err != nil { log.Println("err:", err) udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra) return } else if tmp != nil { if key, ok := (*tmp)["key"].(string); ok { udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra) } else { udpclient.WriteUdp([]byte("udpok"), mu.OP_NOOP, ra) } //data_type:winner data_type:buyer data_type:agency //data_info:save//存量 data_info:add //增量 if key, ok := (*tmp)["data_type"].(string); ok { //阻塞 CPool <- true if key == "winner" { go TaskWinner(tmp) } else if key == "buyer" { go TaskBuyer(tmp) } else if key == "agency" { go TaskAgency(tmp) } } } case mu.OP_NOOP: //下个节点回应 log.Println("发送成功", string(data)) } }