jy-data-extract/udp_winner/main.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"go.mongodb.org/mongo-driver/bson"
	"go.mongodb.org/mongo-driver/bson/primitive"
	"go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/options"
	es "gopkg.in/olivere/elastic.v1"
	"log"
	mu "mfw/util"
	"net"
	"qfw/common/src/qfw/util/elastic"
	"qfw/util"
	"regexp"
	"strconv"

	"sort"
	"strings"
	"time"
)

var
(
	Config                                = make(map[string]string)
	Fields                                []string
	SourceClient, FClient                 *mongo.Client
	RedisPool                             redis.Pool
	Addrs                                 = make(map[string]interface{}, 0) //省市县
	udpclient                             mu.UdpClient                      //udp对象
	ElasticClientIndex, ElasticClientType string
	Reg_xing                              = regexp.MustCompile(`\*{1,}`)
	Reg_person                            = regexp.MustCompile("[\u4E00-\u9FA5\\s]+")
	Reg_tel                               = regexp.MustCompile(`^[0-9\-\s]*$`)
	EsConn                                *es.Client
	Updport                               int
)
/**
新增
初始化
 */
func init() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	util.ReadConfig(&Config)
	log.Println(Config)
	Fields = []string{"_id", "contact", "partners", "business_scope", "company_address", "capital",
		"establish_date", "legal_person", "company_type", "district", "city", "province", "area_code", "credit_no",
		"company_name", "history_name", "topscopeclass", "wechat_accounts", "alias", "website", "report_websites"}
	var err error
	pool_size, _ := strconv.Atoi(Config["pool_size"])

	//mongo init
	SourceClient, err = mongo.NewClient(options.Client().ApplyURI("mongodb://" + Config["mgoinit"]).SetMaxPoolSize(uint16(pool_size)).SetMaxConnIdleTime(time.Hour * 24))
	if err != nil {
		log.Fatalln(err)
	}
	c1, _ := context.WithTimeout(context.Background(), 9999*time.Hour)

	err = SourceClient.Connect(c1)
	//defer SourceClient.Disconnect(c1)
	if err != nil {
		log.Fatalln(err)
	}
	FClient, err = mongo.NewClient(options.Client().ApplyURI("mongodb://" + Config["mgourl"]).SetMaxPoolSize(uint16(pool_size)).SetMaxConnIdleTime(time.Hour * 999999))
	if err != nil {
		log.Fatalln(err)
	}
	err = FClient.Connect(c1)
	//defer FClient.Disconnect(cc)
	if err != nil {
		log.Fatalln(err)
	}
	//加载省市县代码
	cursor2, err := FClient.Database(Config["mgodb_extract_kf"]).Collection("address").Find(c1, bson.M{},
		options.Find().SetProjection(bson.M{"province": 1, "code": 1, "city": 1, "district": 1}))
	//defer FClient.Connect(cc)
	if err != nil {
		log.Fatalln(err)
	}
	for cursor2.Next(context.TODO()) {
		if err := cursor2.Err(); err != nil {
			log.Println("cursor.Err();", err)
		}
		tmp := make(map[string]interface{})
		if err := cursor2.Decode(&tmp); err != nil {
			log.Println(err)
			continue
		} else {
			code := tmp["code"]
			if code != nil && strings.TrimSpace(code.(string)) != "" {
				Addrs[fmt.Sprint(code)] = tmp
			}
		}
	}
	defer cursor2.Close(context.TODO())
	log.Println(len(Addrs))
	//es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false))
	//es init
	elastic.InitElasticSize(Config["elasticsearch"], 50)
	EsConn = elastic.GetEsConn()
	defer elastic.DestoryEsConn(EsConn)
	//redis
	RedisPool = redis.Pool{
		MaxIdle:     50,
		IdleTimeout: 10 * time.Second,
		Dial: func() (redis.Conn, error) {
			conn, e := redis.Dial("tcp", Config["redis"])
			if e != nil {
				return conn, e
			}
			_, err = conn.Do("SELECT", "1")
			if err != nil {
				return nil, err
			}
			return conn, nil
		},}
	c := RedisPool.Get()
	if _, err := c.Do("PING"); err != nil {
		log.Fatalln("redis err:", err)
	}
	c.Close()
}

func main() {
	//udp
	updport := Config["udpport"]
	Updport, _ = strconv.Atoi(Config["port"])
	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
	udpclient.Listen(processUdpMsg)
	log.Println("Udp服务监听", updport)
	log.Println("发送端口port：",Updport)
	go TimedTask() //定时任务
	c := make(chan int, 1)
	<-c

}

func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
	log.Println(act, string(data), ra)
	switch act {
	case mu.OP_TYPE_DATA: //上个节点的数据
		//从表中开始处理生成企业数据
		tmp := new(map[string]interface{})
		err := json.Unmarshal(data, &tmp)
		if err != nil {
			log.Println("err:", err)
			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
			return
		} else if tmp != nil {
			udpclient.WriteUdp([]byte("ok,run"), mu.OP_NOOP, ra)
			go task(tmp)
		}
	case mu.OP_NOOP: //下个节点回应
		log.Println("发送成功", string(data))
	}
}
func task(mapinfo *map[string]interface{}) {
	defer util.Catch()
	gtid, lteid := util.ObjToString((*mapinfo)["gtid"]), util.ObjToString((*mapinfo)["lteid"])
	if gtid == "" || lteid == "" {
		log.Println(gtid, lteid, "参数错误")
		return
	}
	GId, err := primitive.ObjectIDFromHex(gtid)
	LtId, err2 := primitive.ObjectIDFromHex(lteid)
	if err != nil || err2 != nil {
		log.Println(gtid, lteid, "转换_id错误")
		return
	}
	//udp的id区间查询bidding  中标人 中标联系人 中标联系电话
	// topscopeclass项目类型-industry行业类型&&topscopeclass联系人项目类型
	// (area地区-province省份 city城市-city城市 district区县-district区县)
	// winneraddr-company_address企业地址
	cursor, err := SourceClient.Database(Config["mgodb_bidding"]).Collection(Config["mgodb_mgoinit_c"]).Find(context.Background(), bson.M{
		"_id": bson.M{
			"$gte": GId,
			"$lte": LtId,
		},
	}, options.Find().SetProjection(bson.M{"winner": 1, "winnertel": 1, "winnerperson": 1,
		"topscopeclass": 1, "winneraddr": 1}).SetSort(bson.M{"_id": 1}).SetBatchSize(60000000).SetMaxTime(time.Hour*24))
	if err != nil {
		log.Println(err)
		return
	}
	overid := ""
	c2, _ := context.WithTimeout(context.Background(), 9999*time.Hour)
	for cursor.Next(c2) {
		if err := cursor.Err(); err != nil {
			log.Println("cursor.Err();", err)
		}
		tmp := map[string]interface{}{}
		if err := cursor.Decode(&tmp); err == nil {
			if tmp["winner"] == nil || tmp["winner"] == "" {
				continue
			}
			//redis查询是否存在
			rdb := RedisPool.Get()
			if reply, err := redis.String(rdb.Do("GET", tmp["winner"])); err != nil {
				//redis不存在存到临时表,定时任务处理
				if _, err := FClient.Database(Config["mgodb_extract_kf"]).Collection("winner_new").InsertOne(context.TODO(), tmp); err != nil {
					log.Println(err, tmp)
				}
				//log.Println("get redis id err:定时任务处理", err, tmp)
				if err := rdb.Close(); err != nil {
					log.Println(err)
				}
				continue
			} else {
				//log.Println("redis get :", reply)
				//redis存在
				//log.Println(reply)
				//reply = "5e0316b998a9abaf6535df3d"
				id, err := primitive.ObjectIDFromHex(reply)
				if err != nil {
					log.Println("get redis id  Hex err:", err, tmp)
					if err := rdb.Close(); err != nil {
						log.Println(err)
					}
					continue
				}
				if err := rdb.Close(); err != nil {
					log.Println(err)
				}
				//拿到合并后的qyk
				oldTmp := make(map[string]interface{})
				err = FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
					FindOne(context.TODO(), bson.M{"_id": id}).Decode(&oldTmp)
				if err != nil {
					log.Println("qyk id err:", err, id)
					continue
				}
				//比较合并
				//行业类型
				tmpTopscopeclass := []string{}
				tmpTopscopeclassMap := make(map[string]bool)
				log.Println(tmp["_id"])
				overid = tmp["_id"].(primitive.ObjectID).Hex()
				if oldTmp["industry"] == nil {
					//log.Println(reflect.ValueOf(tmp["topscopeclass"]))
					if v, ok := tmp["topscopeclass"].(primitive.A); ok {
						for _, vv := range v {
							if vvv, ok := vv.(string); ok && len(vvv) > 1 {
								tmpTopscopeclassMap[vvv[:len(vvv)-1]] = true
							}
						}
						for k := range tmpTopscopeclassMap {
							tmpTopscopeclass = append(tmpTopscopeclass, k)
						}
					}
				} else {
					if v, ok := oldTmp["industry"].(primitive.A); ok {
						for _, vv := range v {
							if vvv, ok := vv.(string); ok {
								tmpTopscopeclassMap[vvv] = true
							}
						}
					}
					if v, ok := tmp["topscopeclass"].(primitive.A); ok {
						for _, vv := range v {
							if vvv, ok := vv.(string); ok && len(vvv) > 1 {
								tmpTopscopeclassMap[vvv[:len(vvv)-1]] = true
							}
						}
						for k := range tmpTopscopeclassMap {
							tmpTopscopeclass = append(tmpTopscopeclass, k)
						}
					}
				}
				sort.Strings(tmpTopscopeclass)
				oldTmp["industry"] = tmpTopscopeclass
				esId := oldTmp["_id"].(primitive.ObjectID).Hex()
				//更新行业类型
				if tmp["winnerperson"] == nil || tmp["winnerperson"] == "" || Reg_xing.MatchString(util.ObjToString(tmp["winnerperson"])) {
					oldTmp["updatatime"] = time.Now().Unix()
					//mongo更新
					if _, err := FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
						UpdateOne(context.TODO(), bson.M{"_id": oldTmp["_id"]}, bson.M{"$set": oldTmp}); err != nil {
						log.Println("mongo更新err:", err)
					}
					//es更新
					delete(oldTmp, "_id")
					//esConn := elastic.GetEsConn()
					//defer elastic.DestoryEsConn(esConn)
					if _, err := EsConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do(); err != nil {
						log.Println("update es err:", err)
					}
					//log.Println( err2,err3)
					continue
				}
				//联系方式合并
				var tmpperson, winnertel string
				tmpperson = tmp["winnerperson"].(string)
				if tmp["winnertel"] == nil || tmp["winnertel"] == "" {
					winnertel = ""
				} else {
					if Reg_xing.MatchString(util.ObjToString(tmp["winnertel"])) || !Reg_tel.MatchString(util.ObjToString(tmp["winnertel"])) {
						winnertel = ""
					} else {
						winnertel = util.ObjToString(tmp["winnertel"])
					}
				}
				contactMaps := make([]interface{}, 0)
				if oldTmp["contact"] == nil {
					tmpContact := make(map[string]interface{})
					tmpContact["contact_person"] = tmpperson
					tmpContact["contact_type"] = "项目联系人"
					tmpContact["phone"] = winnertel
					tmpContact["topscopeclass"] = strings.Join(tmpTopscopeclass, ";")
					tmpContact["updatetime"] = time.Now().Unix()
					contactMaps = append(contactMaps, tmpContact)
				} else {
					//对比前四项,相等丢弃
					if v, ok := oldTmp["contact"].(primitive.A); ok {
						var isNotUpdate bool
						for _, vv := range v {
							if vvv, ok := vv.(map[string]interface{}); ok {
								if vvv["contact_person"] == tmpperson && vvv["contact_type"] == "项目联系人" &&
									vvv["phone"] == winnertel && vvv["topscopeclass"] == strings.Join(tmpTopscopeclass, ";") {
									isNotUpdate = true
									vvv["updatetime"] = time.Now().Unix()
								}
								contactMaps = append(contactMaps, vvv)
							}
						}
						if !isNotUpdate {
							vvv := make(map[string]interface{})
							vvv["contact_person"] = tmp["winnerperson"]
							vvv["contact_type"] = "项目联系人"
							vvv["phone"] = winnertel
							vvv["topscopeclass"] = strings.Join(tmpTopscopeclass, ";")
							vvv["updatetime"] = time.Now().Unix()
							contactMaps = append(contactMaps, vvv)
						}
					}
				}
				oldTmp["contact"] = contactMaps
				//mongo更新
				oldTmp["updatatime"] = time.Now().Unix()
				if _, err := FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
					UpdateOne(context.TODO(), bson.M{"_id": oldTmp["_id"]}, bson.M{"$set": oldTmp}); err != nil {
					log.Println("mongo更新 err :", err)
				}
				//es更新
				delete(oldTmp, "_id")
				//esConn := elastic.GetEsConn()
				//defer elastic.DestoryEsConn(esConn)
				if _, err := EsConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do(); err != nil {
					log.Println("EsConn err :", err)
				}
				//log.Println( err2,err3)
			}
		} else {
			log.Println(tmp)
			continue
		}
	}
	defer cursor.Close(c2)
	log.Println("合并执行完成", gtid, lteid ,overid)
	if gtid != lteid{
		by, _ := json.Marshal(map[string]interface{}{
			"gtid":  overid,
			"lteid": lteid,
			"stype": "",
		})
		if e := udpclient.WriteUdp(by, mu.OP_TYPE_DATA, &net.UDPAddr{
			IP:   net.ParseIP("127.0.0.1"),
			Port: Updport,
		});e != nil{
			log.Println(e)
		}
		log.Println("重新发送udp:",string(by))
	}
	log.Println("合并执行完成 ok", gtid, lteid,overid)

}