定时更新es

go.mod

@@ -3,7 +3,7 @@ module app.yhyue.com/moapp/jy_docs
 go 1.14
 
 require (
-	app.yhyue.com/moapp/jybase v0.0.0-20210318061912-f5594310c8fe
+	app.yhyue.com/moapp/jybase v0.0.0-20210319015107-fe59d2046cf8
 	app.yhyue.com/moapp/jyfs v0.0.0-20210309070327-ecad67beabb8
 	github.com/golang/protobuf v1.4.3
 	github.com/mailru/easyjson v0.7.7 // indirect

go.sum

@@ -1,5 +1,7 @@
 app.yhyue.com/moapp/jybase v0.0.0-20210318061912-f5594310c8fe h1:RSQIrriyi534CCrI/sSLGW+Rd0X/0W4VDDXIalHVo8c=
 app.yhyue.com/moapp/jybase v0.0.0-20210318061912-f5594310c8fe/go.mod h1:29ShuI8y7qEyg2KviHSx1iamiCioBKdTMm2ndVzWAhk=
+app.yhyue.com/moapp/jybase v0.0.0-20210319015107-fe59d2046cf8 h1:0wnq1iFfpPrASDJzO24+Lr2pMO18ISNDf1n4TU3faTQ=
+app.yhyue.com/moapp/jybase v0.0.0-20210319015107-fe59d2046cf8/go.mod h1:29ShuI8y7qEyg2KviHSx1iamiCioBKdTMm2ndVzWAhk=
 app.yhyue.com/moapp/jyfs v0.0.0-20210309070327-ecad67beabb8 h1:Mg5E4O+5mCjRyW2b4/bNRQMz8ilzWfa7qaOjKT7SNRY=
 app.yhyue.com/moapp/jyfs v0.0.0-20210309070327-ecad67beabb8/go.mod h1:iflE6z/Ee5f7YZy2DSj7S6asyL1yCbu3BjK/xMYMiDY=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=

timetask/config.json

@@ -0,0 +1,17 @@
+{
+	"elasticsearch": {
+		"address": "http://192.168.3.241:9200,http://192.168.3.242:9200",
+    	"size": 1,
+		"userName": "elastic",
+        "passWord": "elastic"
+    },
+    "mysql": {
+    	"driverName": "mysql",
+        "url": "root:Topnet123@tcp(192.168.3.11:3366)/jydocs?charset=utf8mb4&parseTime=true&loc=Local",
+		"maxOpenConns": 1,
+		"maxIdleConns": 1,
+ 		"maxConnLifeTime": 100
+	},
+	"durationMinute": 10,
+	"blukSize": 100
+}

timetask/main.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"time"
+
+	"app.yhyue.com/moapp/jybase/common"
+	"app.yhyue.com/moapp/jybase/date"
+	"app.yhyue.com/moapp/jybase/esv7"
+	. "app.yhyue.com/moapp/jybase/mysql"
+	"gorm.io/gorm"
+)
+
+type config struct {
+	Elasticsearch struct {
+		Address  string
+		Size     int
+		UserName string
+		PassWord string
+	}
+	Mysql struct {
+		DriverName      string
+		Url             string
+		MaxOpenConns    int
+		MaxIdleConns    int
+		MaxConnLifeTime int
+	}
+	DurationMinute int
+	BlukSize       int
+}
+type timeTask struct {
+	Time string `json:"time"`
+}
+type DocStatistics struct {
+	DocId     string `json:"docId" gorm:"column:docId"`
+	DownTimes int    `json:"downTimes" gorm:"column:downTimes"`
+	ViewTimes int    `json:"viewTimes" gorm:"column:viewTimes"`
+}
+
+const (
+	Es_Jydoc = "jydoc"
+)
+
+var (
+	Config   *config
+	TimeTask *timeTask
+	Mysql    *gorm.DB
+)
+
+func init() {
+	common.ReadConfig(&Config)
+	if Config.DurationMinute == 0 {
+		log.Fatalln("config.json中durationMinute配置项异常")
+	}
+	common.ReadConfig("./timetask.json", &TimeTask)
+	elastic.InitElasticSizeByAuth(Config.Elasticsearch.Address, Config.Elasticsearch.Size, Config.Elasticsearch.UserName, Config.Elasticsearch.PassWord)
+	log.Printf("初始化 elasticsearch")
+	Mysql = GormMysql(Config.Mysql.Url, Config.Mysql.DriverName, Config.Mysql.MaxOpenConns, Config.Mysql.MaxIdleConns, nil)
+	if Mysql != nil {
+		log.Printf("初始化 mysql")
+	} else {
+		log.Fatalf("mysql初始化失败")
+	}
+}
+func main() {
+	run()
+	<-chan bool(nil)
+}
+
+//
+func run() {
+	now := date.NowFormat(date.Date_Full_Layout)
+	log.Println("start update to es", TimeTask.Time, now)
+	rows, err := Mysql.Raw(`select docId,downTimes,viewTimes from doc_statistics where updateDate>=? and updateDate<?`, TimeTask.Time, now).Rows()
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	if rows != nil {
+		defer rows.Close()
+	}
+	array := [][]string{}
+	index := 0
+	for rows.Next() {
+		var docId string
+		var downTimes int
+		var viewTimes int
+		err = rows.Scan(&docId, &downTimes, &viewTimes)
+		if err != nil {
+			log.Println(err)
+			break
+		}
+		log.Println("need update to es", "docId", docId, "downTimes", downTimes, "viewTimes", viewTimes)
+		index++
+		array = append(array, []string{docId, fmt.Sprintf("ctx._source.downTimes=%d;ctx._source.viewTimes=%d", downTimes, viewTimes)})
+		if len(array) == Config.BlukSize {
+			log.Println("update es index", index, elastic.NewBulkUpdate(Es_Jydoc, array...))
+			array = [][]string{}
+		}
+	}
+	if len(array) > 0 {
+		log.Println("update es index", index, elastic.NewBulkUpdate(Es_Jydoc, array...))
+		array = [][]string{}
+	}
+	TimeTask.Time = now
+	common.WriteSysConfig("./timetask.json", &TimeTask)
+	log.Println("update to es over", index)
+	time.AfterFunc(time.Duration(Config.DurationMinute)*time.Minute, run)
+}

timetask/timetask.json

@@ -0,0 +1 @@
+{"time":"2021-03-19 11:42:02"}