jy-data-extract/standardata/src/main.go

package main

import (
	"dbutil/mongo"
	"dbutil/redis"
	"fmt"
	"log"
	mu "mfw/util"
	"os"
	qu "qfw/util"
	"regexp"
	"strconv"
	"strings"
)

var (
	MongoFrom /*抽取原*/, MongoTo /*保存库*/, MongoEnt/*企业库*/ *mongo.MongodbSim
	sysconfig                   map[string]interface{}
	extractcoll                 string
	winnerent, winnererr        string
	buyerent, buyererr          string
	agencyent, agencyerr        string
	winnerbd, buyerbd, agencybd int
	Addrs                       = make(map[string]interface{}, 0) //省市县
	winchanbool                 = make(chan bool, 3)
	buyerchanbool               = make(chan bool, 3)
	agencychanbool              = make(chan bool, 3)
	gochan                      = make(chan bool, 3)
	udpclient                   mu.UdpClient
	//异常表正则匹配处理
	WinnerRegOk, WinnerRegErr, AgencyRegOk, AgencyRegErr, BuyerRegOk, BuyerRegErr []regexp.Regexp
)

func init() {
	qu.ReadConfig(&sysconfig)
	extractcoll = qu.ObjToString(sysconfig["extractcoll"])
	if standardata, ok := sysconfig["standardata"].(map[string]interface{}); ok {
		winner, _ := standardata["winner"].(map[string]interface{})
		winnerent = qu.ObjToString(winner["standarent"])
		winnererr = qu.ObjToString(winner["standarerr"])
		winnerbd = qu.IntAll(winner["redisdb"])
		buyer, _ := standardata["buyer"].(map[string]interface{})
		buyerent = qu.ObjToString(buyer["standarent"])
		buyererr = qu.ObjToString(buyer["standarerr"])
		buyerbd = qu.IntAll(buyer["redisdb"])
		agency, _ := standardata["agency"].(map[string]interface{})
		agencyent = qu.ObjToString(agency["standarent"])
		agencyerr = qu.ObjToString(agency["standarerr"])
		agencybd = qu.IntAll(agency["redisdb"])
	} else {
		os.Exit(0)
	}

	MongoFrom = &mongo.MongodbSim{
		MongodbAddr: qu.ObjToString(sysconfig["mgofrom"]),
		Size:        qu.IntAll(sysconfig["mgofromsize"]),
		DbName:      qu.ObjToString(sysconfig["mgofromdb"]),
	}
	MongoFrom.InitPool()

	MongoTo = &mongo.MongodbSim{
		MongodbAddr: qu.ObjToString(sysconfig["mgoto"]),
		Size:        qu.IntAll(sysconfig["mgotosize"]),
		DbName:      qu.ObjToString(sysconfig["mgotodb"]),
	}
	MongoTo.InitPool()

	MongoEnt = &mongo.MongodbSim{
		MongodbAddr: qu.ObjToString(sysconfig["mgoent"]),
		Size:        qu.IntAll(sysconfig["mgoentsize"]),
		DbName:      qu.ObjToString(sysconfig["mgoentdb"]),
	}
	MongoEnt.InitPool()
	redis.InitRedis(qu.ObjToString(sysconfig["redis"]))

	//加载省市县代码
	list, _ := MongoTo.Find("address", map[string]interface{}{}, nil, nil)
	for _, v := range list {
		code := v["code"]
		if code != nil && strings.TrimSpace(code.(string)) != "" {
			Addrs[fmt.Sprint(code)] = v
		}
	}
	log.Println("加载省市县代码加载完成", len(Addrs))
	initReg() //初始标记规则
}

func initReg() {
	list, _ := MongoTo.Find("rc_rule", map[string]interface{}{"delete": false, "isuse": true}, nil, nil)
	for _, v := range list {
		s_field, ok := v["s_field"].(string) //字段
		s_rule, ok2 := v["s_rule"].(string)  //正则
		s_type, ok3 := v["s_type"].(string)  //ok or err
		if !ok || !ok2 || !ok3 || s_field == "" || s_rule == "" || s_type == "" {
			continue
		}
		var pattern string
		if strings.Contains(s_rule, "\\u") {
			s_rule = strings.Replace(s_rule, "\\", "\\\\", -1)
			s_rule = strings.Replace(s_rule, "\\\\u", "\\u", -1)
			pattern, _ = strconv.Unquote(`"` + s_rule + `"`)
		} else {
			pattern = s_rule
		}
		regtmp := regexp.MustCompile(pattern)
		if s_field == "winner" {
			if s_type == "ok" {
				WinnerRegOk = append(WinnerRegOk, *regtmp)
			} else if s_type == "err" {
				WinnerRegErr = append(WinnerRegErr, *regtmp)
			}
		} else if s_field == "buyer" {
			if s_type == "ok" {
				BuyerRegOk = append(BuyerRegOk, *regtmp)
			} else if s_type == "err" {
				BuyerRegErr = append(BuyerRegErr, *regtmp)
			}
		} else if s_field == "agency" {
			if s_type == "ok" {
				AgencyRegOk = append(AgencyRegOk, *regtmp)
			} else if s_type == "err" {
				AgencyRegErr = append(AgencyRegErr, *regtmp)
			}
		}
	}
	log.Println(len(WinnerRegOk), len(WinnerRegErr), len(BuyerRegOk), len(BuyerRegErr), len(AgencyRegOk), len(AgencyRegErr))
}

func main() {
	//go historywinner(qu.ObjToString(sysconfig["mgofromdb"]), extractcoll)
	//go historybuyer(qu.ObjToString(sysconfig["mgofromdb"]), extractcoll)
	//go historyagency(qu.ObjToString(sysconfig["mgofromdb"]), extractcoll)

	go task_standarData()
	c := make(chan int, 1)
	<-c
}