data_ent_wuye/ent_util/init.go

package ent_util

import (
	"context"
	"fmt"
	"github.com/ClickHouse/clickhouse-go/v2"
	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
	log "github.com/donnie4w/go-logger/logger"
	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
	"runtime"
	"time"
)

var (
	SourceMgo, QyxyMgo *MongodbSim
	SpiMgo, ExtMgo     *MongodbSim
	MysqlGlobalTool    *Mysql
	IsLocal, IsDev     bool
	ClickHouseConn     driver.Conn
	EsClinet           *elastic.Elastic
	BuyerClassData     = map[string]string{}
	RegionCodeData     = map[string]string{}
	TimeLayout         = "2006年01月02日"
	TimeLayout_New     = "2006-01-02 15:04:05"
	Url                = "https://www.jianyu360.cn/article/content/%s.html"
)

const (
	G_Units_Baseinfo = "dws_f_ent_baseinfo"
	G_Units_Contact  = "dws_f_ent_contact"
	G_Units_Tags     = "dws_f_ent_tags"
	G_Units_Warning  = "dws_f_ent_warning"
)

func InitGlobalVar() {
	IsLocal = true
	initMgo()
	//initMysql()
	//initVCode()
	//initClickHouse()
	//initEs()
}

func initEs() {
	url := "http://127.0.0.1:13003"
	if !IsLocal {
		url = "http://172.17.4.184:19805"
	}
	EsClinet = &elastic.Elastic{
		S_esurl:  url,
		I_size:   10,
		Username: "es_all",
		Password: "TopJkO2E_d1x",
	}
	EsClinet.InitElasticSize()
}

func initClickHouse() {
	ClickHouseConn, _ = connectClickhouse()
}

// 初始化mgo
func initMgo() {
	if IsLocal {
		SourceMgo = &MongodbSim{
			MongodbAddr: "127.0.0.1:12005",
			DbName:      "qfw",
			Size:        10,
			UserName:    "zhengkun",
			Password:    "zk@123123",
		}
		SourceMgo.InitPoolDirect()

		QyxyMgo = &MongodbSim{
			MongodbAddr: "127.0.0.1:12005",
			DbName:      "mixdata",
			Size:        10,
			UserName:    "zhengkun",
			Password:    "zk@123123",
		}
		QyxyMgo.InitPoolDirect()

		SpiMgo = &MongodbSim{
			MongodbAddr: "127.0.0.1:12004",
			DbName:      "zhengkun",
			Size:        10,
			UserName:    "",
			Password:    "",
		}
		SpiMgo.InitPoolDirect()

		ExtMgo = &MongodbSim{
			MongodbAddr: "127.0.0.1:12001",
			DbName:      "qfw",
			Size:        10,
			UserName:    "",
			Password:    "",
		}
		ExtMgo.InitPoolDirect()
	} else {
		SourceMgo = &MongodbSim{
			MongodbAddr: "172.17.189.140:27080,172.17.189.141:27081",
			DbName:      "qfw",
			Size:        10,
			UserName:    "zhengkun",
			Password:    "zk@123123",
		}
		SourceMgo.InitPool()

		QyxyMgo = &MongodbSim{
			MongodbAddr: "172.17.189.140:27080,172.17.189.141:27081",
			DbName:      "mixdata",
			Size:        10,
			UserName:    "zhengkun",
			Password:    "zk@123123",
		}
		QyxyMgo.InitPool()

		SpiMgo = &MongodbSim{
			MongodbAddr: "172.17.4.181:27001",
			DbName:      "mixdata",
			Size:        10,
			UserName:    "",
			Password:    "",
		}
		SpiMgo.InitPool()

		ExtMgo = &MongodbSim{
			MongodbAddr: "172.17.4.85:27080",
			DbName:      "qfw",
			Size:        10,
			UserName:    "",
			Password:    "",
		}
		ExtMgo.InitPool()
	}
}

func initMysql() {

	username, password := "zhengkun", "Zk#20220824"
	address := "127.0.0.1:15001"
	if !IsLocal {
		address = "172.17.4.242:4000"
	}
	if IsDev {
		username, password = "root", "=PDT49#80Z!RVv52_z"
		address = "192.168.3.217:4000"
	}
	MysqlGlobalTool = &Mysql{
		Address:  address,
		UserName: username,
		PassWord: password,
		DBName:   "global_common_data",
	}
	MysqlGlobalTool.Init()
}

// 加载代码表
func initVCode() {
	data_types := MysqlGlobalTool.Find("code_buyerclass", nil, "", "", -1, -1)
	//先构建所有一级数据
	for _, v := range *data_types {

		name := qu.ObjToString(v["name"])
		code := qu.ObjToString(v["code"])
		BuyerClassData[name] = code
	}
	BuyerClassData["其它"] = "00"
	log.Debug("招标行业分类表~", len(BuyerClassData))

	data_regions := MysqlGlobalTool.Find("code_area", nil, "", "", -1, -1)
	//先构建所有一级数据
	for _, v := range *data_regions {
		area := qu.ObjToString(v["area"])
		city := qu.ObjToString(v["city"])
		district := qu.ObjToString(v["district"])
		code := qu.ObjToString(v["code"])
		key := area + "~" + city + "~" + district + "~"
		RegionCodeData[key] = code
	}
	log.Debug("地域信息数量~", len(RegionCodeData))
}

// 创建clickhouse连接
func connectClickhouse() (driver.Conn, error) {
	var (
		ctx = context.Background()
		/*
			外网地址：cc-2ze9tv451wov14w9e.public.clickhouse.ads.aliyuncs.com:9000
			内网地址：cc-2ze9tv451wov14w9e.clickhouse.ads.aliyuncs.com:9000
		*/

		conn, err = clickhouse.Open(&clickhouse.Options{
			Addr:         []string{"192.168.3.207:19000"},
			DialTimeout:  10 * time.Second,
			MaxIdleConns: 3,
			MaxOpenConns: 30,
			Auth: clickhouse.Auth{
				Database: "information",
				Username: "jytop",
				Password: "pwdTopJy123",
			},
			Debugf: func(format string, v ...interface{}) {
				fmt.Printf(format, v)
			},
		})

		//conn, err = clickhouse.Open(&clickhouse.Options{
		//	Addr:         []string{"cc-2ze9tv451wov14w9e.public.clickhouse.ads.aliyuncs.com:9000"},
		//	DialTimeout:  10 * time.Second,
		//	MaxIdleConns: 3,
		//	MaxOpenConns: 30,
		//	Auth: clickhouse.Auth{
		//		Database: "information",
		//		Username: "biservice",
		//		Password: "Bi_top95215#",
		//	},
		//	Debugf: func(format string, v ...interface{}) {
		//		fmt.Printf(format, v)
		//	},
		//})
	)
	if err != nil {
		return nil, err
	}
	if err := conn.Ping(ctx); err != nil {
		if exception, ok := err.(*clickhouse.Exception); ok {
			fmt.Printf("Exception [%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
		}
		return nil, err
	}
	return conn, nil
}

// ...
func GetNewInfo(index, query string) map[string]interface{} {
	//log.Println("query  -- ", query)
	client := EsClinet.GetEsConn()
	defer EsClinet.DestoryEsConn(client)
	res := map[string]interface{}{}
	if client != nil {
		defer func() {
			if r := recover(); r != nil {
				log.Debug("[E]", r)
				for skip := 1; ; skip++ {
					_, file, line, ok := runtime.Caller(skip)
					if !ok {
						break
					}
					go log.Debug("%v,%v\n", file, line)
				}
			}
		}()
		searchResult, err := client.Search().Index(index).Source(query).Do(context.TODO())
		if err != nil {
			log.Debug("从ES查询出错", err.Error())
			return nil
		}
		if searchResult.Hits != nil {
			resNum := searchResult.Hits.TotalHits.Value
			res["total"] = resNum
		}
		if searchResult.Aggregations != nil {
			bidamount_sum, _ := searchResult.Aggregations.Sum("bidamount_sum")
			res["bidamount_sum"] = *bidamount_sum.Value
		}
	}
	return res
}