jy-data-extract/udp_winner/main.go

package main

import (
	"encoding/json"
	"fmt"
	"github.com/garyburd/redigo/redis"
	hisRedis "github.com/go-redis/redis"
	"gopkg.in/mgo.v2/bson"
	es "gopkg.in/olivere/elastic.v1"
	"log"
	mu "mfw/util"
	"net"
	"qfw/common/src/qfw/util/elastic"
	"qfw/util"
	"qfw/util/mongodb"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	Config                                = make(map[string]string)
	Fields, BuyerFields, AgencyFields     []string
	SourceClient, FClient                 *mongodb.MongodbSim
	RedisPool                             redis.Pool
	HisRedisPool                          *hisRedis.Client
	Addrs                                 = make(map[string]interface{}, 0) //省市县
	udpclient                             mu.UdpClient                      //udp对象
	ElasticClientIndex, ElasticClientType string
	Reg_xing                              = regexp.MustCompile(`\*{1,}`)
	Reg_person                            = regexp.MustCompile("[\u4E00-\u9FA5\\s]+")
	Reg_tel                               = regexp.MustCompile(`^[0-9\-\s]*$`)
	EsConn                                *es.Client
	Updport                               int
	CPool                                 chan bool
	//his_redis db
	redis_winner_db, redis_buyer_db, redis_agency_db int
	//异常表正则匹配处理
	WinnerRegOk, WinnerRegErr, AgencyRegOk, AgencyRegErr, BuerRegOk, BuyerRegErr []regexp.Regexp
)

/**
新增
初始化
*/
func init() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	util.ReadConfig(&Config)
	log.Println(Config)
	var err error
	cpnum, err := strconv.Atoi(Config["chan_pool_num"])
	if err != nil {
		log.Fatalln(err)
	}
	CPool = make(chan bool, cpnum)
	Fields = []string{"_id", "contact", "partners", "business_scope", "company_address",
		"capital", "establish_date", "legal_person", "company_type",
		"district", "city", "province", "area_code", "credit_no",
		"company_name", "history_name", "wechat_accounts",
		"alias", "website", "report_websites", "industry"}

	BuyerFields = []string{"_id", "contact", "type", "ranks", "buyerclass",
		"address", "district", "city", "province", "area_code", "credit_no", "buyer_name",
		"history_name", "wechat_accounts", "website", "report_websites"}

	AgencyFields = []string{"_id", "contact", "type", "ranks",
		"address", "district", "city", "province", "area_code", "credit_no", "agency_name",
		"history_name", "wechat_accounts", "website", "report_websites"}
	pool_size, _ := strconv.Atoi(Config["pool_size"])

	//mongo init
	SourceClient = new(mongodb.MongodbSim)
	SourceClient.MongodbAddr = Config["mgoinit"]
	SourceClient.Size = pool_size
	SourceClient.DbName = Config["mgodb_bidding"]
	//mongodbSim.DbName = "qfw"
	SourceClient.InitPool()

	FClient = new(mongodb.MongodbSim)
	FClient.MongodbAddr = Config["mgourl"]
	FClient.Size = pool_size
	FClient.DbName = Config["mgodb_extract_kf"]
	//mongodbSim.DbName = "qfw"
	FClient.InitPool()
	FClientmgoConn := FClient.GetMgoConn()
	defer FClient.DestoryMongoConn(FClientmgoConn)
	//加载省市县代码
	cursor2 := FClientmgoConn.DB(Config["mgodb_extract_kf"]).C("address").Find(bson.M{}).Select(bson.M{"province": 1, "code": 1, "city": 1, "district": 1}).Iter()
	//defer FClient.Connect(cc)
	if cursor2 == nil {
		log.Fatalln(cursor2)
	}
	tmp := make(map[string]interface{})
	for cursor2.Next(&tmp) {
		code := tmp["code"]
		if code != nil && strings.TrimSpace(code.(string)) != "" {
			Addrs[fmt.Sprint(code)] = tmp
		}
	}
	log.Println(len(Addrs))
	//es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false))
	//es init
	elastic.InitElasticSize(Config["elasticsearch"], 50)
	EsConn = elastic.GetEsConn()
	defer elastic.DestoryEsConn(EsConn)
	//redis
	RedisPool = redis.Pool{
		MaxIdle:     50,
		IdleTimeout: 10 * time.Second,
		Dial: func() (redis.Conn, error) {
			conn, e := redis.Dial("tcp", Config["redis"])
			if e != nil {
				return conn, e
			}
			_, err = conn.Do("SELECT", "1")
			if err != nil {
				return nil, err
			}
			return conn, nil
		}}
	c := RedisPool.Get()
	if _, err := c.Do("PING"); err != nil {
		log.Fatalln("redis err:", err)
	}
	c.Close()
	HisRedisPool = hisRedis.NewClient(&hisRedis.Options{
		Addr:         Config["his_redis"],
		DB:           1,
		DialTimeout:  10 * time.Second,
		ReadTimeout:  30 * time.Second,
		WriteTimeout: 30 * time.Second,
		PoolSize:     30,
		MinIdleConns: 20,
		PoolTimeout:  30 * time.Second,
	})
	redis_winner_db, _ = strconv.Atoi(Config["redis_winner_db"])
	redis_buyer_db, _ = strconv.Atoi(Config["redis_buyer_db"])
	redis_agency_db, _ = strconv.Atoi(Config["redis_agency_db"])
	iniReg()

}

func iniReg() {
	FClientmgoConnReg := FClient.GetMgoConn()
	defer FClient.DestoryMongoConn(FClientmgoConnReg)
	findReg, b := FClient.Find(Config["mgo_qyk_reg"], bson.M{"delete": false, "isuse": true}, bson.M{"_id": 1}, nil, false, -1, 0)
	if !b {
		log.Fatalln("查询正则失败")
	}
	for _, v := range (*findReg) {
		s_field, ok := v["s_field"].(string) //字段
		s_rule, ok2 := v["s_rule"].(string)  //正则
		s_type, ok3 := v["s_type"].(string)  //ok or err
		if !ok || !ok2 || !ok3 || s_field == "" || s_rule == "" || s_type == "" {
			continue
		}
		regtmp := regexp.MustCompile(s_rule)
		if s_field == "winner" {
			if s_type=="ok"{
				WinnerRegOk = append(WinnerRegOk, *regtmp)
			}else if s_type=="err"{
				WinnerRegErr = append(WinnerRegErr, *regtmp)
			}
		} else if s_field == "buyer" {
			if s_type=="ok"{
				BuerRegOk = append(BuerRegOk, *regtmp)
			}else if s_type=="err"{
				BuyerRegErr = append(BuyerRegErr, *regtmp)
			}
		} else if s_field == "agency" {
			if s_type=="ok"{
				AgencyRegOk = append(AgencyRegOk, *regtmp)
			}else if s_type=="err"{
				AgencyRegErr = append(AgencyRegErr, *regtmp)
			}
		}
	}
	log.Println(len(WinnerRegOk), len(WinnerRegErr), len(BuerRegOk), len(BuyerRegErr), len(AgencyRegOk), len(AgencyRegErr))
}

func main() {
	//udp
	updport := Config["udpport"]
	Updport, _ = strconv.Atoi(Config["port"])
	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
	udpclient.Listen(processUdpMsg)
	log.Println("Udp服务监听", updport)
	log.Println("发送端口port：", Updport)
	go TimedTaskWinner() //定时任务
	go TimedTaskBuyer()  //定时任务
	go TimedTaskAgency() //定时任务
	c := make(chan int, 1)
	<-c

}

func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
	log.Println(act, string(data), ra)
	switch act {
	case mu.OP_TYPE_DATA: //上个节点的数据
		//从表中开始处理生成企业数据
		tmp := new(map[string]interface{})
		err := json.Unmarshal(data, &tmp)
		if err != nil {
			log.Println("err:", err)
			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
			return
		} else if tmp != nil {
			if key, ok := (*tmp)["key"].(string); ok {
				udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
			} else {
				udpclient.WriteUdp([]byte("udpok"), mu.OP_NOOP, ra)
			}
			//data_type:winner data_type:buyer data_type:agency
			//data_info:save//存量   data_info:add //增量
			if key, ok := (*tmp)["data_type"].(string); ok {
				//阻塞
				CPool <- true
				if key == "winner" {
					go TaskWinner(tmp)
				} else if key == "buyer" {
					go TaskBuyer(tmp)
				} else if key == "agency" {
					go TaskAgency(tmp)
				}
			}
		}
	case mu.OP_NOOP: //下个节点回应
		log.Println("发送成功", string(data))
	}
}