增加模式

config.yaml

@@ -13,23 +13,23 @@ ses:
 #目标端配置
 des:
   addr: "http://127.0.0.1:19902"
-  index: "bidding_temp"
+  index: "bidding_v1"
   stype: "bidding"
   #当需要指定同步字段时要使用，比如同步到未登录用户只需要个别字段，同步全部字段请注释此行
-  #fields: ["are","autoid","bidamount","bidstatus","budget","buyer","buyerclass","city","comeintime","createtime","dataweight","detail","district","entidlist","href","id","multipackage","pici","projectcode","publishtime","s_subscopeclass","s_topscopeclass","site","subtype","title","topscopeclass","toptype"]
+  #fields: ["area","autoid","bidamount","bidstatus","budget","buyer","buyerclass","city","comeintime","createtime","dataweight","detail","district","entidlist","href","id","multipackage","pici","projectcode","publishtime","s_subscopeclass","s_topscopeclass","site","subtype","title","topscopeclass","toptype"]
   user: "jybid"
   pwd: "Top2023_JEB01i@31"
   size: 3
 
 # id支持时间19位格式和id24位格式---------id支持两种模式，不能混合使用
 #gtid: "650ea35ce17a7c80fbe231ed"
-gtid: "2023-09-25 20:10:00"
-lteid: "2023-09-26 08:20:00"
+gtid: "2023-09-26 20:00:00"
+lteid: "2023-09-28 08:00:00"
 #lteid: "650eb13de2d7d34fa0415373"
 
 #同步模式mode  1是初始模式源到目标  2双向同步   3是定时源到目标模式  4双向定时
 mode: 1
-# 数据连续模式 0默认 1最大id  2比较
+# 数据连续模式 0默认传递的id/时间段 1目标端最大id  2是比较前两者取最小的  3最大pici  4指定pici段
 lastmode: 0
 # 空跑输出差异结果，不同步数据，false时表示不空跑且同步数据
 synctest: true 
@@ -44,6 +44,8 @@ sync:
     lastmode: 1
     # 数据范围为数据截止时间的前300秒
     scope: 300
+    # 强制同步
+    force: true
   -
     freq: 3600
     before: 600

config_n.yaml

@@ -1,33 +1,30 @@
 ses:
   addr: "http://127.0.0.1:19902"
-  index: "bidding_v1"
+  index: "bidding"
   stype: "bidding"
   user: "jybid"
   pwd: "Top2023_JEB01i@31"
   size: 2
 des:
   addr: "http://127.0.0.1:19902"
-  index: "bidding_y1"
+  index: "bidding_year"
   stype: "bidding"
   user: "jybid"
   pwd: "Top2023_JEB01i@31"
-  fields: ["are","autoid","bidamount","bidstatus","budget","buyer","buyerclass","city","comeintime","createtime","dataweight","detail","district","entidlist","href","id","multipackage","pici","projectcode","publishtime","s_subscopeclass","s_topscopeclass","site","subtype","title","topscopeclass","toptype"]
+  #fields: ["area","autoid","bidamount","bidstatus","budget","buyer","buyerclass","city","comeintime","createtime","dataweight","detail","district","entidlist","href","id","multipackage","pici","projectcode","publishtime","s_subscopeclass","s_topscopeclass","site","subtype","title","topscopeclass","toptype"]
   size: 2
 # id支持时间19位格式和id24位格式
-gtid: "2023-01-01 07:00:00"
-lteid: "2023-09-25 21:00:00"
+gtid: "2023-10-06 22:00:00"
+lteid: "2023-10-08 08:00:00"
 # 同步模式mode 1,2是初始模式  3,4定时
-mode: 1
-# 数据连续模式 0默认 1最大id  2比较
+mode: 3
+# 数据连续模式 0默认 1最大id  2比较 3pici 4pici段
 lastmode: 0
-synctest: false 
+synctest: true 
 sync: 
   -
     freq: 60
-    before: 600
-    lastmode: 1
-    scope: 600
-  -
-    freq: 3600
-    before: 600
+    before: 2400
+    lastmode: 4
     scope: 7200
+

config_o.yaml

@@ -1,24 +1,28 @@
-des:
+ses:
   addr: "http://127.0.0.1:9902"
-  index: "bidding_temporary"
+  index: "bidding_v1"
   stype: "bidding"
   user: "es_all"
   pwd: "TopJkO2E_d1x"
   size: 2
-ses:
+des:
   addr: "http://127.0.0.1:9902"
-  index: "bidding_v1"
+  index: "bidding_temporary"
   stype: "bidding"
   user: "es_all"
   pwd: "TopJkO2E_d1x"
   size: 2
 # id支持时间19位格式和id24位格式
 #gtid: "61a84bd30000000000000000"
-gtid: "2023-09-22 10:00:00"
-lteid: "2023-09-22 12:00:00"
+gtid: "2023-09-25 20:00:00"
+lteid: "2023-09-26 18:00:00"
 #lteid: "61a84bd40000000000000000"
-# 模式mode 1是初始模式 2是定时模式
-mode: 3
+# 模式mode 1是初始模式 2是初始双向 3定时单向 4是定时双向 
+mode: 1
+# 数据连续模式
+lastmode: 0
+# 是否空跑不同步数据
+synctest: true 
 sync: 
   -
     freq: 60

config_on.yaml

@@ -13,10 +13,10 @@ ses:
 #目标端配置
 des:
   addr: "http://127.0.0.1:19902"
-  index: "bidding_temp"
+  index: "bidding_year"
   stype: "bidding"
   #当需要指定同步字段时要使用，比如同步到未登录用户只需要个别字段，同步全部字段请注释此行
-  #fields: ["are","autoid","bidamount","bidstatus","budget","buyer","buyerclass","city","comeintime","createtime","dataweight","detail","district","entidlist","href","id","multipackage","pici","projectcode","publishtime","s_subscopeclass","s_topscopeclass","site","subtype","title","topscopeclass","toptype"]
+  #fields: ["area","autoid","bidamount","bidstatus","budget","buyer","buyerclass","city","comeintime","createtime","dataweight","detail","district","entidlist","href","id","multipackage","pici","projectcode","publishtime","s_subscopeclass","s_topscopeclass","site","subtype","title","topscopeclass","toptype"]
   user: "jybid"
   pwd: "Top2023_JEB01i@31"
   size: 3
@@ -28,24 +28,19 @@ lteid: "2023-09-25 20:00:00"
 #lteid: "650eb13de2d7d34fa0415373"
 
 #同步模式mode  1是初始模式源到目标  2双向同步   3是定时源到目标模式  4双向定时
-mode: 1
+mode: 3
 # 数据连续模式 0默认 1最大id  2比较
 lastmode: 0
 # 空跑输出差异结果，不同步数据，false时表示不空跑且同步数据
-synctest: false 
+synctest: true 
 # 当为定时模式时，使用sync下的配置，支持多套定时方案，且lastmode使用下面的子配置
 sync: 
   -
     # 定时频率，每60秒执行一次
-    freq: 60
+    freq: 600
     # 执行时，选定数据截止到10分钟之前
-    before: 600
+    before: 3600
     # 数据同步模式，1为最目标端最大id
-    lastmode: 1
+    lastmode: 0
     # 数据范围为数据截止时间的前300秒
-    scope: 300
-  -
-    freq: 3600
-    before: 600
-    lastmode: 2
-    scope: 7200
+    scope: 86400

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	app.yhyue.com/moapp/jybase v0.0.0-20230901064756-2fc66b18db40
 	github.com/dgraph-io/badger/v4 v4.2.0
+	github.com/gogf/gf/v2 v2.5.4
 	github.com/olivere/elastic/v7 v7.0.22
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -26,7 +27,9 @@ require (
 	github.com/olivere/elastic v6.2.37+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	go.opencensus.io v0.22.5 // indirect
-	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
+	go.opentelemetry.io/otel v1.14.0 // indirect
+	go.opentelemetry.io/otel/trace v1.14.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 )

go.sum

@@ -43,6 +43,7 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
 github.com/ClickHouse/clickhouse-go/v2 v2.2.0/go.mod h1:8f2XZUi7XoeU+uPIytSi1cvx8fmJxi7vIgqpvYTF1+o=
@@ -78,6 +79,7 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -121,6 +123,7 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
@@ -128,6 +131,7 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
@@ -145,6 +149,8 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
@@ -158,6 +164,8 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogf/gf/v2 v2.5.4 h1:UBCSw8mInkHmEqL0E1LYc6QhSpaNFY/wHcFrTI/rzTk=
+github.com/gogf/gf/v2 v2.5.4/go.mod h1:7yf5qp0BznfsYx7Sw49m3mQvBsHpwAjJk3Q9ZnKoUEc=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -216,8 +224,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -245,7 +253,9 @@ github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+
 github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
@@ -297,16 +307,20 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
@@ -328,6 +342,7 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uY
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/olivere/elastic v6.2.37+incompatible h1:UfSGJem5czY+x/LqxgeCBgjDn6St+z8OnsCuxwD3L0U=
 github.com/olivere/elastic v6.2.37+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
@@ -384,6 +399,7 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -416,8 +432,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
 github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
@@ -448,11 +464,16 @@ go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk=
 go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM=
+go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
+go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
 go.opentelemetry.io/otel/exporters/jaeger v1.8.0/go.mod h1:GbWg+ng88rDtx+id26C34QLqw2erqJeAjsCx9AFeHfE=
 go.opentelemetry.io/otel/exporters/zipkin v1.8.0/go.mod h1:0uYAyCuGT67MFV9Z/Mmx93wGuugHw0FbxMc74fs3LNo=
 go.opentelemetry.io/otel/sdk v1.8.0/go.mod h1:uPSfc+yfDH2StDM/Rm35WE8gXSNdvCg023J6HeGNO0c=
+go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY=
 go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU=
 go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4=
+go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
+go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@@ -545,8 +566,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220531201128-c960675eff93/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
+golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -625,8 +646,8 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -639,6 +660,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

main.go

@@ -15,6 +15,7 @@ import (
 
 	"app.yhyue.com/moapp/jybase/es"
 	"github.com/dgraph-io/badger/v4"
+	"github.com/gogf/gf/v2/util/gconv"
 	esv7 "github.com/olivere/elastic/v7"
 	"gopkg.in/yaml.v3"
 )
@@ -36,7 +37,12 @@ type (
 		Freq     int64
 		Before   int64
 		Scope    int64
-		Lastmode int //0默认取时间  1取最大id  2取最大id和时间的最小值
+		Lastmode int  //0默认取时间  1取最大id  2取最大id和时间的最小值
+		Force    bool //强制同步，不对比
+	}
+	sypici struct { //pici模式时需要传递时间对象
+		Gt  int64
+		Lte int64
 	}
 	els struct {
 		Addr        string
@@ -59,8 +65,10 @@ var (
 	//统计sql
 	sql_count = `{"query":{"range":{"id":{"gt":"%s","lte":"%s"}}}}`
 	//取最大id sql
-	sql_last = `{"query":{"match_all":{}},"sort":[{"id":"desc"}],"_source":["id"],"size":1}`
-	layout   = "2006-01-02 15:04:05"
+	sql_last     = `{"query":{"match_all":{}},"sort":[{"id":"desc"}],"_source":["id"],"size":1}`
+	sql_picilast = `{"query":{"match_all":{}},"sort":[{"pici":"desc"}],"_source":["pici"],"size":1}`
+	sql_piciid   = `{"query": {"range": {"pici":{"gt":%d,"lte":%d}}},"sort": [{"id": "asc"}],"_source": ["id"],"size": 1}`
+	layout       = "2006-01-02 15:04:05"
 	//本在数据库
 	db   = &bdb{DB()}
 	conf = "./config.yaml"
@@ -120,15 +128,15 @@ func main() {
 	case 1, 2: //单向取时间段
 		func() {
 			go GetTask(Cfg.Des, "atask")
-			gid, eid := GetIds(Cfg.Gtid, Cfg.Lteid, Cfg.Des, Cfg.Lastmode)
-			BinarySearch(Cfg.Ses, Cfg.Des, gid, eid, "atask")
+			gid, eid, piciobj := GetIds(Cfg.Gtid, Cfg.Lteid, Cfg.Des, Cfg.Ses, Cfg.Lastmode)
+			BinarySearch(Cfg.Ses, Cfg.Des, gid, eid, "atask", Cfg.Lastmode, piciobj)
 			log.Println("atask", "over...")
 		}()
 		if Cfg.Mode == 2 {
 			func() {
 				go GetTask(Cfg.Ses, "btask")
-				gid, eid := GetIds(Cfg.Gtid, Cfg.Lteid, Cfg.Ses, Cfg.Lastmode)
-				BinarySearch(Cfg.Des, Cfg.Ses, gid, eid, "btask")
+				gid, eid, piciobj := GetIds(Cfg.Gtid, Cfg.Lteid, Cfg.Ses, Cfg.Des, Cfg.Lastmode)
+				BinarySearch(Cfg.Des, Cfg.Ses, gid, eid, "btask", Cfg.Lastmode, piciobj)
 				log.Println("btask", "over...")
 			}()
 		}
@@ -146,13 +154,25 @@ func main() {
 					no := now - obj.Before //从几分钟前开始
 					gtid := TimestampToId(no - obj.Scope)
 					lteid := TimestampToId(no)
-					gid, eid := GetIds(gtid, lteid, Cfg.Des, obj.Lastmode)
-					BinarySearch(Cfg.Ses, Cfg.Des, gid, eid, "atask")
+					gid, eid, piciobj := GetIds(gtid, lteid, Cfg.Des, Cfg.Ses, obj.Lastmode)
+					if obj.Force {
+						if !Cfg.Synctest {
+							Reindex(Cfg.Ses, Cfg.Des, gid, eid, "atask", obj.Lastmode, piciobj)
+						}
+					} else {
+						BinarySearch(Cfg.Ses, Cfg.Des, gid, eid, "atask", obj.Lastmode, piciobj)
+					}
 					log.Println("atask", "over...", "freq", obj.Freq)
 					//-----------------
 					if Cfg.Mode == 4 {
-						gid1, eid1 := GetIds(gtid, lteid, Cfg.Ses, obj.Lastmode)
-						BinarySearch(Cfg.Des, Cfg.Ses, gid1, eid1, "atask")
+						gid1, eid1, piciobj := GetIds(gtid, lteid, Cfg.Ses, Cfg.Des, obj.Lastmode)
+						if obj.Force {
+							if !Cfg.Synctest {
+								Reindex(Cfg.Des, Cfg.Ses, gid1, eid1, "atask", obj.Lastmode, piciobj)
+							}
+						} else {
+							BinarySearch(Cfg.Des, Cfg.Ses, gid1, eid1, "atask", obj.Lastmode, piciobj)
+						}
 						log.Println("btask", "over...", "freq", obj.Freq)
 					}
 					last := nt - time.Now().Unix()
@@ -161,63 +181,77 @@ func main() {
 					}
 				}
 			}(obj)
+			time.Sleep(25 * time.Second)
 		}
 	}
 	time.Sleep(99999 * time.Hour)
 }
 
 // 获取统计sql
-func GetCountSql(source *els, target *els, sid, eid string) any {
+func GetCountSql(source *els, target *els, sid, eid string, lastmode int, piciobj *sypici) any {
+	count := esv7.NewBoolQuery()
+	if lastmode >= 3 { //使用pici
+		count.Filter(esv7.NewRangeQuery("pici").Gt(piciobj.Gt).Lte(piciobj.Lte))
+	} else {
+		count.Filter(esv7.NewRangeQuery("id").Gt(sid).Lte(eid))
+	}
 	if len(source.Boolsql) > 0 {
-		rawsql := esv7.NewRawStringQuery(source.Boolsql)
-		_, err := rawsql.Source()
-		if err == nil {
-			count := esv7.NewBoolQuery()
-			count.Filter(esv7.NewRangeQuery("id").Gt(sid).Lte(eid), rawsql)
-			return count
-		} else {
-			log.Println("sql转换出错", err, source.Boolsql)
-			os.Exit(1)
-		}
+		count.Filter(esv7.NewRawStringQuery(source.Boolsql))
 	}
-	return fmt.Sprintf(sql_count, sid, eid)
+	PrintSql(count, "count")
+	return count
+}
 
+func PrintSql(query esv7.Query, note string) {
+	v, _ := query.Source()
+	bs, _ := json.Marshal(v)
+	log.Println("query:", note, string(bs))
 }
 
 // 二分法查找执行任务
-func BinarySearch(source *els, target *els, sid, eid, key string) {
-	// scount := source.ES.Count(source.Index, source.Stype, fmt.Sprintf(sql_count, sid, eid))
-	// dcount := target.ES.Count(target.Index, target.Stype, fmt.Sprintf(sql_count, sid, eid))
-	scount := source.ES.Count(source.Index, source.Stype, GetCountSql(source, target, sid, eid))
-	dcount := target.ES.Count(target.Index, target.Stype, GetCountSql(source, target, sid, eid))
-	log.Println("compare:", key, sid, eid, scount, dcount)
+func BinarySearch(source *els, target *els, sid, eid, key string, lastmode int, piciobj *sypici) {
+	countSql := GetCountSql(source, target, sid, eid, lastmode, piciobj)
+	scount := source.ES.Count(source.Index, source.Stype, countSql)
+	dcount := target.ES.Count(target.Index, target.Stype, countSql)
+	log.Println("compare:", lastmode, key, sid, eid, scount, dcount)
 	if scount > 0 && scount != dcount && !(scount < dcount && dcount < 10000) {
-		if dcount-scount == 1 { //目标集群比源集群量多1条，不继续
-			return
-		}
-		//满足条件则开启同步
-		isid, _ := strconv.ParseInt(sid[:8], 16, 64)
-		ieid, _ := strconv.ParseInt(eid[:8], 16, 64)
-		mid := TimestampToId((isid + ieid) / 2)
-		//6509dbd90000000000000000 6509dbda0000000000000000
-		if dcount == 0 || isid == ieid || sid == mid || (dcount < scount && (float64(scount-dcount)/float64(scount)) > 0.98) {
-			log.Println("sync", !Cfg.Synctest, key, sid, eid, scount, dcount)
-			if !Cfg.Synctest {
-				Reindex(source, target, sid, eid, key)
+		if lastmode > 2 { //pici模式
+			if dcount > scount { //目标集群比源集群量多，不继续
+				return
+			}
+			mid := (piciobj.Gt + piciobj.Lte) / 2
+			if dcount == 0 || piciobj.Gt == piciobj.Lte || piciobj.Gt == mid || (dcount < scount && (float64(scount-dcount)/float64(scount)) > 0.98) {
+				log.Println("sync-pici", !Cfg.Synctest, key, piciobj.Gt, piciobj.Lte, mid, scount, dcount)
+				if !Cfg.Synctest {
+					Reindex(source, target, sid, eid, key, lastmode, piciobj)
+				}
+			} else {
+				BinarySearch(source, target, sid, "", key, lastmode, &sypici{piciobj.Gt, mid})
+				BinarySearch(source, target, "", eid, key, lastmode, &sypici{mid, piciobj.Lte})
 			}
+
 		} else {
-			BinarySearch(source, target, sid, mid, key)
-			BinarySearch(source, target, mid, eid, key)
+			if dcount-scount == 1 { //目标集群比源集群量多1条，不继续
+				return
+			}
+			//满足条件则开启同步
+			isid, _ := strconv.ParseInt(sid[:8], 16, 64)
+			ieid, _ := strconv.ParseInt(eid[:8], 16, 64)
+			mid := TimestampToId((isid + ieid) / 2)
+			//6509dbd90000000000000000 6509dbda0000000000000000
+			if dcount == 0 || isid == ieid || sid == mid || (dcount < scount && (float64(scount-dcount)/float64(scount)) > 0.98) {
+				log.Println("sync", !Cfg.Synctest, key, sid, eid, scount, dcount)
+				if !Cfg.Synctest {
+					Reindex(source, target, sid, eid, key, lastmode, piciobj)
+				}
+			} else {
+				BinarySearch(source, target, sid, mid, key, lastmode, piciobj)
+				BinarySearch(source, target, mid, eid, key, lastmode, piciobj)
+			}
 		}
 	}
 }
 
-// var (
-// 	taskLock = &sync.Mutex{}
-// 	//key         = "tasks"
-// 	currentTask = 0
-// )
-
 // 定时获取任务列表并清除队列，只为修改currentTask值
 func GetTask(target *els, key string) {
 	conn := target.ES.GetEsConn()
@@ -259,8 +293,8 @@ func GetTask(target *els, key string) {
 }
 
 // 获取同步的id范围
-func GetIds(Gtid, Lteid string, target *els, mode int) (gid, eid string) {
-	switch mode {
+func GetIds(Gtid, Lteid string, target *els, source *els, lastmode int) (gid, eid string, piciobj *sypici) {
+	switch lastmode {
 	case 0:
 		gid = Gtid
 		eid = Lteid
@@ -281,10 +315,60 @@ func GetIds(Gtid, Lteid string, target *els, mode int) (gid, eid string) {
 		if eid < gid {
 			eid = gid
 		}
+	//pici模式，只需要pici即可，只有定时模式适用！！！---删除---取源端的最大id和最小id
+	case 3, 4:
+		// lastmode > 2 {
+		piciMin := int64(0)
+		piciMax, _ := strconv.ParseInt(Lteid[0:8], 16, 64)
+		if lastmode == 3 { //取target的最大pici
+			piciMin = GetPiciLast(target)
+		} else if lastmode == 4 { //取传递的最小pici
+			piciMin, _ = strconv.ParseInt(Gtid[0:8], 16, 64)
+		}
+		piciobj = &sypici{piciMin, piciMax}
+		// } else {
+		// 	gid = Gtid
+		// 	// //到源端去查找最小id,这个id有可能也没同步，所有用gt会漏掉这一条数据
+		// 	res := source.ES.Get(source.Index, source.Stype, fmt.Sprintf(sql_piciid, piciMin, piciMax))
+		// 	if res != nil && len(*res) == 1 {
+		// 		gid, _ = (*res)[0]["id"].(string)
+		// 		//id减1
+		// 		gid = IdSub1(gid)
+		// 	}
+		// 	//最近10天内
+		// 	//gid = TimestampToId(piciMin - 864000)
+		// 	//此处如果还用原来的lteid，在同一个时间戳内有数据会被漏掉，所以要用大的id段，最当前时间的id段
+		// 	// eid = Lteid
+		// 	eid = TimestampToId(time.Now().Unix())
+		// }
 	}
 	return
 }
 
+// id减1
+func IdSub1(id1 string) string {
+	str := ""
+	bf := false
+	for k := len(id1) - 1; k >= 0; k-- {
+		v := id1[k]
+		if !bf {
+			n, _ := strconv.ParseInt(string(v), 16, 64)
+			if n > 0 {
+				n--
+				str = fmt.Sprintf("%x", n) + str
+				bf = true
+			} else {
+				str = "9" + str
+			}
+		} else {
+			//str = string(v) + str
+			str = string(id1[:k+1]) + str
+			break
+		}
+	}
+	return str
+}
+
 // 获取最大id,无数据时返回默认的一个id
 func GetLastId(target *els) (id string) {
 	res := target.ES.Get(target.Index, target.Stype, sql_last)
@@ -296,11 +380,23 @@ func GetLastId(target *els) (id string) {
 	}
 	log.Println("----id---", id)
 	return
+}
 
+// 默认取3天前
+func GetPiciLast(target *els) (pici int64) {
+	res := target.ES.Get(target.Index, target.Stype, sql_picilast)
+	if res != nil && len(*res) == 1 {
+		pici = gconv.Int64((*res)[0]["pici"])
+	}
+	if pici == 0 {
+		pici = time.Now().Unix() - 3*86400
+	}
+	log.Println("----pici---", pici)
+	return
 }
 
 // 使用redindex模式
-func Reindex(source *els, target *els, sid, eid, key string) {
+func Reindex(source *els, target *els, sid, eid, key string, lastmode int, piciobj *sypici) {
 	for {
 		if target.currentTask < 5 {
 			break
@@ -324,7 +420,10 @@ func Reindex(source *els, target *els, sid, eid, key string) {
 	}
 	//生成目标端查询sql
 	find := esv7.NewBoolQuery()
-	querys := []esv7.Query{esv7.NewRangeQuery("id").Gt(sid).Lte(eid)}
+	querys := []esv7.Query{}
+	if !(lastmode > 2 && sid == "" && eid == "") { //当是pici模式时，不再传递id段，只适用于定时模式
+		querys = append(querys, esv7.NewRangeQuery("id").Gt(sid).Lte(eid))
+	}
 	//自定义了查询，增加
 	if len(source.Boolsql) > 0 {
 		rawsql := esv7.NewRawStringQuery(source.Boolsql)
@@ -336,10 +435,12 @@ func Reindex(source *els, target *els, sid, eid, key string) {
 			os.Exit(1)
 		}
 	}
+	if lastmode >= 3 && piciobj != nil {
+		querys = append(querys, esv7.NewRangeQuery("pici").Gt(piciobj.Gt).Lte(piciobj.Lte))
+	}
 	find.Filter(querys...)
 	rs.Query(find)
-	// s, _ := rs.Source()
-	// log.Println(s)
+	PrintSql(rs, "reindex")
 	//限定了查询字段
 	if len(target.Fields) > 0 {
 		rs = rs.FetchSourceIncludeExclude(target.Fields, []string{})