|
|
@@ -7,6 +7,7 @@ import (
|
|
|
"fmt"
|
|
|
"io/ioutil"
|
|
|
"log"
|
|
|
+ "os"
|
|
|
"strconv"
|
|
|
"strings"
|
|
|
"sync"
|
|
|
@@ -42,6 +43,7 @@ type (
|
|
|
Sync string
|
|
|
Index string
|
|
|
Stype string
|
|
|
+ Boolsql string
|
|
|
Fields []string
|
|
|
User string
|
|
|
Pwd string
|
|
|
@@ -102,6 +104,17 @@ func main() {
|
|
|
// v := (int64Value + int64Value2) / 2
|
|
|
|
|
|
// log.Println(int64Value, int64Value2, fmt.Sprintf("%x", v))
|
|
|
+ if len(Cfg.Ses.Boolsql) > 0 {
|
|
|
+ sql := esv7.NewBoolQuery().Filter(esv7.NewRawStringQuery(Cfg.Ses.Boolsql))
|
|
|
+ sqls, err := sql.Source()
|
|
|
+ by, _ := json.Marshal(sqls)
|
|
|
+ log.Println("限定查询范围:", string(by), err)
|
|
|
+ }
|
|
|
+ if len(Cfg.Des.Fields) > 0 {
|
|
|
+ log.Println("限定同步字段", Cfg.Des.Fields)
|
|
|
+ }
|
|
|
+ log.Println("准备执行任务...")
|
|
|
+ time.Sleep(1 * time.Second)
|
|
|
|
|
|
switch Cfg.Mode {
|
|
|
case 1, 2: //单向取时间段
|
|
|
@@ -151,10 +164,30 @@ func main() {
|
|
|
time.Sleep(99999 * time.Hour)
|
|
|
}
|
|
|
|
|
|
+// 获取统计sql
|
|
|
+func GetCountSql(source *els, target *els, sid, eid string) any {
|
|
|
+ if len(source.Boolsql) > 0 {
|
|
|
+ rawsql := esv7.NewRawStringQuery(source.Boolsql)
|
|
|
+ _, err := rawsql.Source()
|
|
|
+ if err == nil {
|
|
|
+ count := esv7.NewBoolQuery()
|
|
|
+ count.Filter(esv7.NewRangeQuery("id").Gt(sid).Lte(eid), rawsql)
|
|
|
+ return count
|
|
|
+ } else {
|
|
|
+ log.Println("sql转换出错", err, source.Boolsql)
|
|
|
+ os.Exit(1)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return fmt.Sprintf(sql_count, sid, eid)
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
// 二分法查找执行任务
|
|
|
func BinarySearch(source *els, target *els, sid, eid, key string) {
|
|
|
- scount := source.ES.Count(source.Index, source.Stype, fmt.Sprintf(sql_count, sid, eid))
|
|
|
- dcount := target.ES.Count(target.Index, target.Stype, fmt.Sprintf(sql_count, sid, eid))
|
|
|
+ // scount := source.ES.Count(source.Index, source.Stype, fmt.Sprintf(sql_count, sid, eid))
|
|
|
+ // dcount := target.ES.Count(target.Index, target.Stype, fmt.Sprintf(sql_count, sid, eid))
|
|
|
+ scount := source.ES.Count(source.Index, source.Stype, GetCountSql(source, target, sid, eid))
|
|
|
+ dcount := target.ES.Count(target.Index, target.Stype, GetCountSql(source, target, sid, eid))
|
|
|
log.Println("compare:", key, sid, eid, scount, dcount)
|
|
|
if scount > 0 && scount != dcount && !(scount < dcount && dcount < 10000) {
|
|
|
if dcount-scount == 1 { //目标集群比源集群量多1条，不继续
|
|
|
@@ -274,8 +307,10 @@ func Reindex(source *els, target *els, sid, eid, key string) {
|
|
|
}
|
|
|
conn := target.ES.GetEsConn()
|
|
|
defer target.ES.DestoryEsConn(conn)
|
|
|
+ //创建reindex对象
|
|
|
rs := esv7.NewReindexSource()
|
|
|
rs.Index(source.Index)
|
|
|
+ //不在同一集群时，要增加remote，同时确保目标端能访问这个配置
|
|
|
if target.Addr != source.Addr {
|
|
|
ri := esv7.NewReindexRemoteInfo()
|
|
|
addr := source.Addr
|
|
|
@@ -285,11 +320,29 @@ func Reindex(source *els, target *els, sid, eid, key string) {
|
|
|
ri.Host(addr).Username(source.User).Password(source.Pwd)
|
|
|
rs.RemoteInfo(ri)
|
|
|
}
|
|
|
-
|
|
|
- rs.Query(esv7.NewBoolQuery().Filter(esv7.NewRangeQuery("id").Gt(sid).Lte(eid)))
|
|
|
+ //生成目标端查询sql
|
|
|
+ find := esv7.NewBoolQuery()
|
|
|
+ querys := []esv7.Query{esv7.NewRangeQuery("id").Gt(sid).Lte(eid)}
|
|
|
+ //自定义了查询，增加
|
|
|
+ if len(source.Boolsql) > 0 {
|
|
|
+ rawsql := esv7.NewRawStringQuery(source.Boolsql)
|
|
|
+ _, err := rawsql.Source()
|
|
|
+ if err == nil {
|
|
|
+ querys = append(querys, rawsql)
|
|
|
+ } else {
|
|
|
+ log.Println("sql转换出错", err, source.Boolsql)
|
|
|
+ os.Exit(1)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ find.Filter(querys...)
|
|
|
+ rs.Query(find)
|
|
|
+ // s, _ := rs.Source()
|
|
|
+ // log.Println(s)
|
|
|
+ //限定了查询字段
|
|
|
if len(target.Fields) > 0 {
|
|
|
rs = rs.FetchSourceIncludeExclude(target.Fields, []string{})
|
|
|
}
|
|
|
+
|
|
|
ds := esv7.NewReindexDestination()
|
|
|
ds.Index(target.Index)
|
|
|
reindex := conn.Reindex().Source(rs).Destination(ds).Conflicts("proceed")
|
