|
@@ -55,6 +55,7 @@ func (f *FinalJob) LoadDatas(toTable string) {
|
|
f.All = map[int64]string{}
|
|
f.All = map[int64]string{}
|
|
reTry := 0
|
|
reTry := 0
|
|
for {
|
|
for {
|
|
|
|
+ Mysql_Main.ExecBySql(`SET session max_execution_time=86400`)
|
|
Mysql_Main.SelectByBath(Config.SelectBathSize, func(l *[]map[string]interface{}) {
|
|
Mysql_Main.SelectByBath(Config.SelectBathSize, func(l *[]map[string]interface{}) {
|
|
for _, v := range *l {
|
|
for _, v := range *l {
|
|
index++
|
|
index++
|
|
@@ -109,63 +110,76 @@ func (f *FinalJob) Run(start_layout, end_layout string) {
|
|
if f.All == nil || len(f.All) == 0 {
|
|
if f.All == nil || len(f.All) == 0 {
|
|
f.LoadDatas(toTable)
|
|
f.LoadDatas(toTable)
|
|
}
|
|
}
|
|
- index := 0
|
|
|
|
- var addIndex int64
|
|
|
|
- var updateIndex int64
|
|
|
|
- pool := make(chan bool, Config.UpdatePool)
|
|
|
|
- wait := &sync.WaitGroup{}
|
|
|
|
- lock := &sync.Mutex{}
|
|
|
|
query := `select ` + strings.Join(f.SaveFields(), ",") + ` from ` + f.TableName()
|
|
query := `select ` + strings.Join(f.SaveFields(), ",") + ` from ` + f.TableName()
|
|
if start_layout != "" && end_layout != "" && f.IncField != "" {
|
|
if start_layout != "" && end_layout != "" && f.IncField != "" {
|
|
query += ` where ` + f.IncField + `>='` + start_layout + `' and ` + f.IncField + `<'` + end_layout + `'`
|
|
query += ` where ` + f.IncField + `>='` + start_layout + `' and ` + f.IncField + `<'` + end_layout + `'`
|
|
}
|
|
}
|
|
- Mysql_Main.SelectByBath(Config.SelectBathSize, func(l *[]map[string]interface{}) {
|
|
|
|
- for _, v := range *l {
|
|
|
|
- index++
|
|
|
|
- var result strings.Builder
|
|
|
|
- array := []interface{}{}
|
|
|
|
- for _, field := range f.SaveFields() {
|
|
|
|
- result.WriteString(fmt.Sprintf("%v", v[field]))
|
|
|
|
- array = append(array, v[field])
|
|
|
|
- }
|
|
|
|
- resultMd5 := GetMd5String(result.String())
|
|
|
|
- id := Int64All(v["id"])
|
|
|
|
- pool <- true
|
|
|
|
- wait.Add(1)
|
|
|
|
- go func(delInsertId int64, delInsertArr []interface{}) {
|
|
|
|
- defer func() {
|
|
|
|
- <-pool
|
|
|
|
- wait.Done()
|
|
|
|
- }()
|
|
|
|
- lock.Lock()
|
|
|
|
- allVal := f.All[id]
|
|
|
|
- lock.Unlock()
|
|
|
|
- if allVal == "" {
|
|
|
|
- r1, r2 := Mysql_Main.InsertBatch(toTable, f.SaveFields(), delInsertArr)
|
|
|
|
- if r1 > 0 && r2 > 0 {
|
|
|
|
- lock.Lock()
|
|
|
|
- f.All[id] = resultMd5
|
|
|
|
- lock.Unlock()
|
|
|
|
- atomic.AddInt64(&addIndex, 1)
|
|
|
|
- }
|
|
|
|
- } else if allVal != resultMd5 {
|
|
|
|
- if Mysql_Main.ExecTx("", func(tx *sql.Tx) bool {
|
|
|
|
- r3 := Mysql_Main.UpdateOrDeleteBySqlByTx(tx, `delete from `+toTable+" where id=?", delInsertId)
|
|
|
|
- r1, r2 := Mysql_Main.InsertBatchByTx(tx, toTable, f.SaveFields(), delInsertArr)
|
|
|
|
- return r1 > 0 && r2 > 0 && r3 > 0
|
|
|
|
- }) {
|
|
|
|
- lock.Lock()
|
|
|
|
- f.All[id] = resultMd5
|
|
|
|
- lock.Unlock()
|
|
|
|
- atomic.AddInt64(&updateIndex, 1)
|
|
|
|
|
|
+ thisIndex := 0
|
|
|
|
+ for {
|
|
|
|
+ thisIndex++
|
|
|
|
+ index := 0
|
|
|
|
+ var addSuccessIndex, addFailIndex, updateSuccessIndex, updateFailIndex int64
|
|
|
|
+ pool := make(chan bool, Config.UpdatePool)
|
|
|
|
+ wait := &sync.WaitGroup{}
|
|
|
|
+ lock := &sync.Mutex{}
|
|
|
|
+ Mysql_Main.ExecBySql(`SET session max_execution_time=86400`)
|
|
|
|
+ Mysql_Main.SelectByBath(Config.SelectBathSize, func(l *[]map[string]interface{}) {
|
|
|
|
+ for _, v := range *l {
|
|
|
|
+ index++
|
|
|
|
+ var result strings.Builder
|
|
|
|
+ array := []interface{}{}
|
|
|
|
+ for _, field := range f.SaveFields() {
|
|
|
|
+ result.WriteString(fmt.Sprintf("%v", v[field]))
|
|
|
|
+ array = append(array, v[field])
|
|
|
|
+ }
|
|
|
|
+ resultMd5 := GetMd5String(result.String())
|
|
|
|
+ id := Int64All(v["id"])
|
|
|
|
+ pool <- true
|
|
|
|
+ wait.Add(1)
|
|
|
|
+ go func(delInsertId int64, delInsertArr []interface{}) {
|
|
|
|
+ defer func() {
|
|
|
|
+ <-pool
|
|
|
|
+ wait.Done()
|
|
|
|
+ }()
|
|
|
|
+ lock.Lock()
|
|
|
|
+ allVal := f.All[id]
|
|
|
|
+ lock.Unlock()
|
|
|
|
+ if allVal == "" {
|
|
|
|
+ r1, r2 := Mysql_Main.InsertBatch(toTable, f.SaveFields(), delInsertArr)
|
|
|
|
+ if r1 > 0 && r2 > 0 {
|
|
|
|
+ lock.Lock()
|
|
|
|
+ f.All[id] = resultMd5
|
|
|
|
+ lock.Unlock()
|
|
|
|
+ atomic.AddInt64(&addSuccessIndex, 1)
|
|
|
|
+ } else {
|
|
|
|
+ atomic.AddInt64(&addFailIndex, 1)
|
|
|
|
+ }
|
|
|
|
+ } else if allVal != resultMd5 {
|
|
|
|
+ if Mysql_Main.ExecTx("", func(tx *sql.Tx) bool {
|
|
|
|
+ r3 := Mysql_Main.UpdateOrDeleteBySqlByTx(tx, `delete from `+toTable+" where id=?", delInsertId)
|
|
|
|
+ r1, r2 := Mysql_Main.InsertBatchByTx(tx, toTable, f.SaveFields(), delInsertArr)
|
|
|
|
+ return r1 > 0 && r2 > 0 && r3 > 0
|
|
|
|
+ }) {
|
|
|
|
+ lock.Lock()
|
|
|
|
+ f.All[id] = resultMd5
|
|
|
|
+ lock.Unlock()
|
|
|
|
+ atomic.AddInt64(&updateSuccessIndex, 1)
|
|
|
|
+ } else {
|
|
|
|
+ atomic.AddInt64(&updateFailIndex, 1)
|
|
|
|
+ }
|
|
}
|
|
}
|
|
|
|
+ }(id, array)
|
|
|
|
+ if index%500 == 0 {
|
|
|
|
+ log.Println("对比公网", fromTableName, "数据结束", index)
|
|
}
|
|
}
|
|
- }(id, array)
|
|
|
|
- if index%500 == 0 {
|
|
|
|
- log.Println("对比公网", fromTableName, "数据结束", index)
|
|
|
|
}
|
|
}
|
|
|
|
+ }, query)
|
|
|
|
+ wait.Wait()
|
|
|
|
+ log.Println("开始对比公网", fromTableName, "数据结束。。。", index, "新增成功", addSuccessIndex, "新增失败", addFailIndex, "更新成功", updateSuccessIndex, "更新失败", updateFailIndex)
|
|
|
|
+ notEqCount := Mysql_Main.CountBySql(`SELECT COUNT(1) FROM `+fromTableName+` a INNER JOIN `+toTable+` b ON (a.`+f.IncField+`>=? and a.`+f.IncField+`<? and a.id=b.id AND a.autoUpdate!=b.autoUpdate)`, start_layout, end_layout)
|
|
|
|
+ if notEqCount == 0 || thisIndex == 10 {
|
|
|
|
+ break
|
|
}
|
|
}
|
|
- }, query)
|
|
|
|
- wait.Wait()
|
|
|
|
- log.Println("开始对比公网", fromTableName, "数据结束。。。", index, "新增", addIndex, "更新", updateIndex)
|
|
|
|
|
|
+ SendMail(fmt.Sprintf("%s表第%d次同步结束后还有%d条数据未同步完,休眠10s后重试", toTable, thisIndex, notEqCount))
|
|
|
|
+ }
|
|
}
|
|
}
|