备份 ... 正式版

data_mgo_to_tidb/bidding.go

@@ -20,12 +20,17 @@ var (
 	regLetter = regexp.MustCompile("[a-z]*")
 )
 
-func taskB() {
+func TaskBidding() {
 	sess := MongoB.GetMgoConn()
 	defer MongoB.DestoryMongoConn(sess)
 	ch := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
-	q := map[string]interface{}{"_id": map[string]interface{}{"$gt": mongodb.StringTOBsonId("132d42d667a6b0a2861eef92")}}
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  mongodb.StringTOBsonId("100000000000000000000000"),
+			"$lte": mongodb.StringTOBsonId("900000000000000000000000"),
+		},
+	}
 	query := sess.DB(config.Conf.DB.MongoB.Dbname).C("bidding").Find(q).Sort("_id").Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {

data_mgo_to_tidb/common.toml

@@ -3,24 +3,24 @@ locport = ":1166"
 [db]
 
 [db.mysql]
-addr = "192.168.3.217:4000"
+addr = "172.17.4.242:4000"
 dbname = "global_common_data"
 size = 5
-user = "root"
-password = "=PDT49#80Z!RVv52_z"
+user = "zhengkun"
+password = "Zk#20220824"
 
 [db.mongob]
-addr = "192.168.3.166:27082"
+addr = "172.17.189.140:27080,172.17.189.141:27081"
 dbname = "qfw"
 size = 5
-user = ""
-password = ""
+user = "zhengkun"
+password = "zk@123123"
 [db.mongop]
-addr = "192.168.3.166:27082"
-dbname = "qfw"
+addr = "172.17.189.140:27080,172.17.189.141:27081"
+dbname = "mixdata"
 size = 5
-user = ""
-password = ""
+user = "zhengkun"
+password = "zk@123123"
 
 [mail]
 send = false

data_mgo_to_tidb/init.go

@@ -69,6 +69,15 @@ var ( //项目
 	FileTypeArr      = []string{"pdf", "doc", "docx", "xlsx", "xls", "jpg", "zip", "rar", "txt", "gif", "png", "bmp", "swf", "html"}
 )
 
+func InitInfo() {
+	config.Init("./common.toml")
+	InitLog()
+	InitMgo()
+	InitMysql()
+	InitField()
+	log.Info("init success")
+}
+
 // InitLog @Description
 // @Author J 2022/7/26 15:30
 func InitLog() {

data_mgo_to_tidb/main.go

@@ -1,113 +1,13 @@
 package main
 
-import (
-	"data_tidb/config"
-	"github.com/spf13/cobra"
-	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
-	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
-)
-
-var (
-	UdpClient udp.UdpClient
-)
-
 func init() {
-	config.Init("./common.toml")
-	InitLog()
-	InitMgo()
-	InitMysql()
-	InitField()
-	//InitOss(false)
-	//redis.InitRedis1("qyxy_id=127.0.0.1:8379", 1)
-	//redis.InitRedis1("qyxy_id=192.168.3.166:4379", 1)
-	log.Info("init success")
-	//启动 nohup ./MTB bidding &
+	InitInfo()
 }
 
 func main() {
-	//go SaveFunc()
-	//go SaveExpandFunc()
-	//go SaveDetailFunc()
-	//go SaveAttrFunc()
-	//go SaveIntentFunc()
-	//go SaveBidderFunc()
-	//go SaveGoodsFunc()
-	//go saveErrMethod()
-	taskB()
-
-	//rootCmd := &cobra.Command{Use: "my cmd"}
-	//rootCmd.AddCommand(bidding())
-	//rootCmd.AddCommand(project())
-	//rootCmd.AddCommand(projectAdd())
-	//if err := rootCmd.Execute(); err != nil {
-	//	fmt.Println("rootCmd.Execute failed", err.Error())
-	//}
+	RunSaveFunc()
+	TaskBidding()
 
-	//UdpClient = udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
-	//UdpClient.Listen(processUdpMsg)
-	//log.Info("Udp服务监听", zap.String("port:", config.Conf.Udp.LocPort))
 	c := make(chan bool, 1)
 	<-c
-
-}
-
-// @Description 标讯数据
-// @Author J 2022/9/20 17:52
-func bidding() *cobra.Command {
-	cmdClient := &cobra.Command{
-		Use:   "bidding",
-		Short: "Start processing bidding data",
-		Run: func(cmd *cobra.Command, args []string) {
-			//go SaveFunc()
-			//go SaveTagFunc()
-			//go SaveExpandFunc()
-			//go SaveAttrFunc()
-			//go SaveImfFunc()
-			//go SaveIntentFunc()
-			//go SaveWinnerFunc()
-			//go SavePackageFunc()
-			//go SavePurFunc()
-			//go saveErrMethod()
-			taskB()
-		},
-	}
-	//cmdClient.Flags().StringVarP(&cfg, "conf", "c", "", "server config [toml]")
-	return cmdClient
-}
-
-// @Description 项目数据
-// @Author J 2022/9/20 17:52
-func project() *cobra.Command {
-	cmdClient := &cobra.Command{
-		Use:   "project",
-		Short: "Start processing project data",
-		Run: func(cmd *cobra.Command, args []string) {
-			//go SaveProFunc()
-			//go SaveProTagFunc()
-			//go SaveProbFunc()
-			//go SaveRelationFunc()
-			taskP()
-		},
-	}
-	//cmdClient.Flags().StringVarP(&cfg, "conf", "c", "", "server config [toml]")
-	return cmdClient
-}
-
-// @Description 项目数据
-// @Author J 2022/9/20 17:52
-func projectAdd() *cobra.Command {
-	var pici int64
-	cmdClient := &cobra.Command{
-		Use:   "project",
-		Short: "Start processing project data",
-		Run: func(cmd *cobra.Command, args []string) {
-			//go SaveProFunc()
-			//go SaveProTagFunc()
-			//go SaveProbFunc()
-			go SaveRelationFunc()
-			taskPAdd(pici)
-		},
-	}
-	cmdClient.Flags().Int64VarP(&pici, "pici", "p", 0, "")
-	return cmdClient
 }

data_mgo_to_tidb/mark

@@ -0,0 +1,54 @@
+//InitOss(false)
+//redis.InitRedis1("qyxy_id=127.0.0.1:8379", 1)
+//redis.InitRedis1("qyxy_id=192.168.3.166:4379", 1)
+
+
+//rootCmd := &cobra.Command{Use: "my cmd"}
+//rootCmd.AddCommand(bidding())
+//rootCmd.AddCommand(project())
+//rootCmd.AddCommand(projectAdd())
+//if err := rootCmd.Execute(); err != nil {
+//	fmt.Println("rootCmd.Execute failed", err.Error())
+//}
+
+//UdpClient = udp.UdpClient{Local: config.Conf.Udp.LocPort, BufSize: 1024}
+//UdpClient.Listen(processUdpMsg)
+//log.Info("Udp服务监听", zap.String("port:", config.Conf.Udp.LocPort))
+
+
+// @Description 项目数据
+// @Author J 2022/9/20 17:52
+func project() *cobra.Command {
+	cmdClient := &cobra.Command{
+		Use:   "project",
+		Short: "Start processing project data",
+		Run: func(cmd *cobra.Command, args []string) {
+			//go SaveProFunc()
+			//go SaveProTagFunc()
+			//go SaveProbFunc()
+			//go SaveRelationFunc()
+			taskP()
+		},
+	}
+	//cmdClient.Flags().StringVarP(&cfg, "conf", "c", "", "server config [toml]")
+	return cmdClient
+}
+
+// @Description 项目数据
+// @Author J 2022/9/20 17:52
+func projectAdd() *cobra.Command {
+	var pici int64
+	cmdClient := &cobra.Command{
+		Use:   "project",
+		Short: "Start processing project data",
+		Run: func(cmd *cobra.Command, args []string) {
+			//go SaveProFunc()
+			//go SaveProTagFunc()
+			//go SaveProbFunc()
+			//go SaveRelationFunc()
+			taskPAdd(pici)
+		},
+	}
+	cmdClient.Flags().Int64VarP(&pici, "pici", "p", 0, "")
+	return cmdClient
+}

data_mgo_to_tidb/save.go

@@ -2,6 +2,17 @@ package main
 
 import "time"
 
+func RunSaveFunc() {
+	//go SaveFunc()
+	//go SaveExpandFunc()
+	//go SaveDetailFunc()
+	//go SaveAttrFunc()
+	//go SaveIntentFunc()
+	//go SaveBidderFunc()
+	//go SaveGoodsFunc()
+	//go saveErrMethod()
+}
+
 func SaveFunc() {
 	arru := make([]map[string]interface{}, saveSize)
 	indexu := 0
@@ -176,6 +187,7 @@ func SaveIntentFunc() {
 		}
 	}
 }
+
 func SaveBidderFunc() {
 	arru := make([]map[string]interface{}, saveSize)
 	indexu := 0
@@ -246,6 +258,9 @@ func SaveGoodsFunc() {
 	}
 }
 
+/*
+以下项目相关涉及
+*/
 func SaveProFunc() {
 	arru := make([]map[string]interface{}, saveSize)
 	indexu := 0

data_mgo_to_tidb/udp.go

@@ -10,6 +10,10 @@ import (
 	"net"
 )
 
+var (
+	UdpClient udp.UdpClient
+)
+
 func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 	defer util.Catch()
 	switch act {