配置文件测试

qyxy_change_new/main.go

@@ -25,6 +25,7 @@ var (
 
 func init() {
 	util.ReadConfig(&Sysconfig)
+	//util.ReadConfig("./test.json", &Sysconfig)
 
 	localPort = Sysconfig["local_port"].(string) //udp 本地监听地址
 	UdpClient = udp.UdpClient{Local: localPort, BufSize: 1024}

qyxy_change_new/test.json

@@ -9,7 +9,7 @@
   "local_port": ":18891",
   "lastId": 0,
   "company_server": "192.168.3.207:27092",
-  "company_db": "mixdata",
+  "company_db": "wcc_mixdata",
   "changeType": [
     {
       "change_code": "100001",

qyxy_es_new/main.go

@@ -29,6 +29,7 @@ var SP = make(chan bool, 5)
 
 func init() {
 	utils.ReadConfig(&Sysconfig)
+	//utils.ReadConfig("test.json", &Sysconfig)
 	Dbname = Sysconfig["dbname"].(string) //
 	Dbcoll = Sysconfig["dbcoll"].(string) //qyxy_std
 	Mgo = &mongodb.MongodbSim{
@@ -85,8 +86,8 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 
 		//拿到同步信号，开始同步数据
 		if _, ok := mapInfo["start"]; ok {
-			go SaveEs()
-			go StdAdd()
+			go SaveEs() // 写入es
+			go StdAdd() //读取qyxy_std 数据，放入es 数组
 		}
 
 		if err != nil {

qyy_inc_data_new/main.go → qyxy_inc_data_new/main.go

@@ -8,10 +8,8 @@ import (
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
-	"go.uber.org/zap"
 	"io"
 	"io/ioutil"
-	"log"
 	"net"
 	"os"
 	"strings"
@@ -42,6 +40,7 @@ var (
 )
 
 func init() {
+	//util.ReadConfig("./test.json", &SysConfig)
 	util.ReadConfig(&SysConfig)
 	localPort = util.ObjToString(SysConfig["local_port"])
 	changePort = util.ObjToString(SysConfig["qyxy_change_port"])
@@ -73,7 +72,6 @@ func main() {
 	UdpClient = udp.UdpClient{Local: localPort, BufSize: 1024}
 	UdpClient.Listen(processUdpMsg)
 
-	log.Println("Udp服务监听=======", zap.String("port:", localPort))
 	util.Debug("Udp服务监听======= port: ", localPort)
 
 	ch := make(chan bool, 1)

qyy_inc_data_new/test.json → qyxy_inc_data_new/test.json

@@ -4,5 +4,5 @@
   "target_ip": "127.0.0.1",
   "path": "",
   "mgoAddr": "192.168.3.207:27092",
-  "db_name": "mixdata"
+  "db_name": "wcc_mixdata"
 }

qyxy_listen/config.json

@@ -6,5 +6,6 @@
   "qyxy_change_port": "18891",
   "target_ip": "127.0.0.1",
   "watch_path": "/Users/wangchengcheng/Desktop/jianyu/upload/",
-  "manual_path": ""
+  "manual_path": "",
+  "spec": "0 */1 * * * *"
 }

qyxy_listen/go.mod

@@ -3,13 +3,14 @@ module qyxy_listen
 go 1.18
 
 require (
-	app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005 // indirect
+	app.yhyue.com/data_processing/common_utils v0.0.0-20221205033056-885644941005
+	github.com/robfig/cron/v3 v3.0.1
+)
+
+require (
 	github.com/PuerkitoBio/goquery v1.8.0 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/dchest/captcha v1.0.0 // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/robfig/cron/v3 v3.0.1 // indirect
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
-	golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 )

qyxy_listen/go.sum

@@ -11,15 +11,15 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
 github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
-github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
-github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
@@ -64,8 +64,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY=
-golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -80,11 +78,13 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/olivere/elastic.v2 v2.0.61/go.mod h1:CTVyl1gckiFw1aLZYxC00g3f9jnHmhoOKcWF7W3c6n4=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

qyxy_listen/main.go

@@ -13,8 +13,8 @@ import (
 	"time"
 )
 
-const pathTableSize = 3 //目录下数据表文件夹个数
-const tableDateSize = 7 //表文件夹下一周的文件夹个数
+const pathTableSize = 45 //目录下数据表文件夹个数
+const tableDateSize = 7  //表文件夹下一周的文件夹个数
 
 var (
 	UdpClient      udp.UdpClient
@@ -64,21 +64,15 @@ func main() {
 	cronServer := cron.New(cron.WithLocation(local), cron.WithSeconds())
 
 	//spec := "0 */1 * * * *" //秒，分，时，日，月，星期
-	spec := "0 57 09 * * 1"
+	//spec := "0 10 09 * * 1" //每周一的9点十分执行
+	spec := util.ObjToString(Sysconfig["spec"])
 	cronServer.AddFunc(spec, dealWatch)
-	//c.AddFunc("0 9 * * 1", dealWatch)
 
 	cronServer.Start()
 
 	select {}
 }
 
-func test() {
-	fmt.Println("test -----")
-	log.Println("start dealWatch")
-	fmt.Println("test >>>>>>")
-}
-
 //dealWatch 处理监听程序，获取最新的文件夹路径
 func dealWatch() {
 	log.Println("start dealWatch")

qyxy_std_new/init.go

@@ -22,6 +22,7 @@ var (
 
 func init() {
 	util.ReadConfig(&Sysconfig)
+	//util.ReadConfig("./test.json", &Sysconfig)
 
 	MongoTool = &mongodb.MongodbSim{
 		MongodbAddr: util.ObjToString(Sysconfig["dbServer"]),

qyxy_std_new/main.go

@@ -152,7 +152,7 @@ func dealinfo(path string) {
 			}
 		}
 		// 本行真正有数据，就处理
-		if len(line) > 500 {
+		if len(line) > 0 {
 			count = hookfn(line, count)
 		}