From 7cf307c12b1f81baa3bc476d1c40f7e88ec9f360 Mon Sep 17 00:00:00 2001 From: 5kbpers <20279863+5kbpers@users.noreply.github.com> Date: Fri, 10 Jan 2020 19:10:52 +0800 Subject: [PATCH 01/10] restore: fix alter auto increment id for no-primary-key table (#139) * restore: fix alter auto increment id for no-primary-key table Signed-off-by: 5kbpers * add integration test Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers --- pkg/restore/db.go | 13 +++++ pkg/restore/db_test.go | 14 ++--- pkg/restore/util.go | 9 +++ tests/br_insert_after_restore/run.sh | 82 ++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 7 deletions(-) create mode 100755 tests/br_insert_after_restore/run.sh diff --git a/pkg/restore/db.go b/pkg/restore/db.go index 71ffe059e..b114b7629 100644 --- a/pkg/restore/db.go +++ b/pkg/restore/db.go @@ -92,6 +92,19 @@ func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error { zap.Error(err)) return errors.Trace(err) } + alterAutoIncIDSQL := fmt.Sprintf( + "alter table %s auto_increment = %d", + escapeTableName(schema.Name), + schema.AutoIncID) + _, err = db.se.Execute(ctx, alterAutoIncIDSQL) + if err != nil { + log.Error("alter AutoIncID failed", + zap.String("SQL", alterAutoIncIDSQL), + zap.Stringer("db", table.Db.Name), + zap.Stringer("table", table.Schema.Name), + zap.Error(err)) + return errors.Trace(err) + } return nil } diff --git a/pkg/restore/db_test.go b/pkg/restore/db_test.go index aa015d640..9583f7f8c 100644 --- a/pkg/restore/db_test.go +++ b/pkg/restore/db_test.go @@ -41,23 +41,23 @@ func (s *testRestoreSchemaSuite) TestRestoreAutoIncID(c *C) { tk := testkit.NewTestKit(c, s.mock.Storage) tk.MustExec("use test") tk.MustExec("set @@sql_mode=''") - tk.MustExec("drop table if exists t;") + tk.MustExec("drop table if exists `\"t\"`;") // Test SQL Mode - tk.MustExec("create table t (" + - "a int not null auto_increment," + + tk.MustExec("create table `\"t\"` (" + + "a int not null," + "time timestamp not null default '0000-00-00 00:00:00'," + "primary key (a));", ) - tk.MustExec("insert into t values (10, '0000-00-00 00:00:00');") + tk.MustExec("insert into `\"t\"` values (10, '0000-00-00 00:00:00');") // Query the current AutoIncID - autoIncID, err := strconv.ParseUint(tk.MustQuery("admin show t next_row_id").Rows()[0][3].(string), 10, 64) + autoIncID, err := strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64) c.Assert(err, IsNil, Commentf("Error query auto inc id: %s", err)) // Get schemas of db and table info, err := s.mock.Domain.GetSnapshotInfoSchema(math.MaxUint64) c.Assert(err, IsNil, Commentf("Error get snapshot info schema: %s", err)) dbInfo, exists := info.SchemaByName(model.NewCIStr("test")) c.Assert(exists, IsTrue, Commentf("Error get db info")) - tableInfo, err := info.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + tableInfo, err := info.TableByName(model.NewCIStr("test"), model.NewCIStr("\"t\"")) c.Assert(err, IsNil, Commentf("Error get table info: %s", err)) table := utils.Table{ Schema: tableInfo.Meta(), @@ -88,7 +88,7 @@ func (s *testRestoreSchemaSuite) TestRestoreAutoIncID(c *C) { c.Assert(err, IsNil, Commentf("Error create table: %s %s", err, s.mock.DSN)) tk.MustExec("use test") // Check if AutoIncID is altered successfully - autoIncID, err = strconv.ParseUint(tk.MustQuery("admin show t next_row_id").Rows()[0][3].(string), 10, 64) + autoIncID, err = strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64) c.Assert(err, IsNil, Commentf("Error query auto inc id: %s", err)) c.Assert(autoIncID, Equals, uint64(globalAutoID+100)) } diff --git a/pkg/restore/util.go b/pkg/restore/util.go index 6dddb5aa5..126e864fd 100644 --- a/pkg/restore/util.go +++ b/pkg/restore/util.go @@ -368,3 +368,12 @@ func encodeKeyPrefix(key []byte) []byte { encodedPrefix = append(encodedPrefix, codec.EncodeBytes([]byte{}, key[:len(key)-ungroupedLen])...) return append(encodedPrefix[:len(encodedPrefix)-9], key[len(key)-ungroupedLen:]...) } + +// escape the identifier for pretty-printing. +// For instance, the identifier "foo `bar`" will become "`foo ``bar```". +// The sqlMode controls whether to escape with backquotes (`) or double quotes +// (`"`) depending on whether mysql.ModeANSIQuotes is enabled. +func escapeTableName(cis model.CIStr) string { + quote := "`" + return quote + strings.Replace(cis.O, quote, quote+quote, -1) + quote +} diff --git a/tests/br_insert_after_restore/run.sh b/tests/br_insert_after_restore/run.sh new file mode 100755 index 000000000..1c72db9ee --- /dev/null +++ b/tests/br_insert_after_restore/run.sh @@ -0,0 +1,82 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu +DB="$TEST_NAME" +TABLE="usertable" +ROW_COUNT=10 +PATH="tests/$TEST_NAME:bin:$PATH" + +insertRecords() { + for i in $(seq $1); do + run_sql "INSERT INTO $DB.$TABLE VALUES ('$i');" + done +} + +createTable() { + run_sql "CREATE TABLE IF NOT EXISTS $DB.$TABLE (c1 CHAR(255));" +} + +echo "load data..." +echo "create database" +run_sql "CREATE DATABASE IF NOT EXISTS $DB;" +echo "create table" +createTable +echo "insert records" +insertRecords $ROW_COUNT + +row_count_ori=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') + +# backup full +echo "backup start..." +run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 + +run_sql "DROP DATABASE $DB;" + +# restore full +echo "restore start..." +run_br restore full -s "local://$TEST_DIR/$DB" --pd $PD_ADDR + +row_count_new=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') + +fail=false +if [ "${row_count_ori}" != "${row_count_new}" ];then + fail=true + echo "TEST: [$TEST_NAME] fail on database $DB" +fi +echo "database $DB [original] row count: ${row_count_ori}, [after br] row count: ${row_count_new}" + +if $fail; then + echo "TEST: [$TEST_NAME] failed!" + exit 1 +fi + +# insert records +insertRecords $ROW_COUNT +row_count_insert=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') +fail=false +if [ "${row_count_insert}" != "$(expr $row_count_new \* 2)" ];then + fail=true + echo "TEST: [$TEST_NAME] fail on inserting records to database $DB after restore: ${row_count_insert}" +fi + +if $fail; then + echo "TEST: [$TEST_NAME] failed!" + exit 1 +else + echo "TEST: [$TEST_NAME] successed!" +fi + +run_sql "DROP DATABASE $DB;" From 191fdce13791c7b19fe5b3141b35e43363b71a44 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Wed, 15 Jan 2020 19:34:58 +0800 Subject: [PATCH 02/10] *: use oracle package to manipulate ts and test gc safe point (#121) Signed-off-by: Neil Shen --- Makefile | 2 + cmd/backup.go | 21 ++++++++--- pkg/backup/client.go | 39 ++++++++++--------- pkg/backup/client_test.go | 31 +++++++++------- pkg/backup/safe_point.go | 17 ++++----- pkg/restore/client.go | 7 +--- pkg/utils/tso.go | 24 ------------ pkg/utils/tso_test.go | 22 ----------- pkg/utils/unit.go | 2 +- pkg/utils/unit_test.go | 17 +++++++++ tests/br_z_gc_safepoint/gc.go | 64 ++++++++++++++++++++++++++++++++ tests/br_z_gc_safepoint/run.sh | 46 +++++++++++++++++++++++ tests/br_z_gc_safepoint/workload | 12 ++++++ 13 files changed, 203 insertions(+), 101 deletions(-) delete mode 100644 pkg/utils/tso_test.go create mode 100644 pkg/utils/unit_test.go create mode 100644 tests/br_z_gc_safepoint/gc.go create mode 100755 tests/br_z_gc_safepoint/run.sh create mode 100644 tests/br_z_gc_safepoint/workload diff --git a/Makefile b/Makefile index a03cedc54..839a27b9e 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,8 @@ build_for_integration_test: -o bin/br.test # build key locker GO111MODULE=on go build -race -o bin/locker tests/br_key_locked/*.go + # build gc + GO111MODULE=on go build -race -o bin/gc tests/br_z_gc_safepoint/*.go test: GO111MODULE=on go test -race -tags leak ./... diff --git a/cmd/backup.go b/cmd/backup.go index 34e1a8970..73ae6106f 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -17,11 +17,12 @@ import ( ) const ( - flagBackupTimeago = "timeago" - flagBackupRateLimit = "ratelimit" - flagBackupConcurrency = "concurrency" - flagBackupChecksum = "checksum" - flagLastBackupTS = "lastbackupts" + flagBackupTimeago = "timeago" + flagBackupRateLimit = "ratelimit" + flagBackupRateLimitUnit = "ratelimit-unit" + flagBackupConcurrency = "concurrency" + flagBackupChecksum = "checksum" + flagLastBackupTS = "lastbackupts" ) func defineBackupFlags(flagSet *pflag.FlagSet) { @@ -36,6 +37,11 @@ func defineBackupFlags(flagSet *pflag.FlagSet) { "Run checksum after backup") flagSet.Uint64P(flagLastBackupTS, "", 0, "the last time backup ts") _ = flagSet.MarkHidden(flagLastBackupTS) + + // Test only flag. + flagSet.Uint64P( + flagBackupRateLimitUnit, "", utils.MB, "The unit of rate limit of the backup task") + _ = flagSet.MarkHidden(flagBackupRateLimitUnit) } func runBackup(flagSet *pflag.FlagSet, cmdName, db, table string) error { @@ -57,6 +63,11 @@ func runBackup(flagSet *pflag.FlagSet, cmdName, db, table string) error { if err != nil { return err } + ratelimitUnit, err := flagSet.GetUint64(flagBackupRateLimitUnit) + if err != nil { + return err + } + ratelimit *= ratelimitUnit concurrency, err := flagSet.GetUint32(flagBackupConcurrency) if err != nil { diff --git a/pkg/backup/client.go b/pkg/backup/client.go index 0d89ac4e1..5cba2d9bf 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/ranger" "go.uber.org/zap" @@ -71,31 +72,31 @@ func (bc *Client) GetTS(ctx context.Context, timeAgo string) (uint64, error) { if err != nil { return 0, errors.Trace(err) } + backupTS := oracle.ComposeTS(p, l) if timeAgo != "" { duration, err := time.ParseDuration(timeAgo) if err != nil { return 0, errors.Trace(err) } - t := duration.Nanoseconds() / int64(time.Millisecond) - log.Info("backup time ago", zap.Int64("MillisecondsAgo", t)) - - // check backup time do not exceed GCSafePoint - safePoint, err := GetGCSafePoint(ctx, bc.mgr.GetPDClient()) - if err != nil { - return 0, errors.Trace(err) + if duration <= 0 { + return 0, errors.New("negative timeago is not allowed") } - if p-t < safePoint.Physical { - return 0, errors.New("given backup time exceed GCSafePoint") + log.Info("backup time ago", zap.Duration("timeago", duration)) + + backupTime := oracle.GetTimeFromTS(backupTS) + backupAgo := backupTime.Add(-duration) + if backupTS < oracle.ComposeTS(oracle.GetPhysical(backupAgo), l) { + return 0, errors.New("backup ts overflow please choose a smaller timeago") } - p -= t + backupTS = oracle.ComposeTS(oracle.GetPhysical(backupAgo), l) } - ts := utils.Timestamp{ - Physical: p, - Logical: l, + // check backup time do not exceed GCSafePoint + err = CheckGCSafepoint(ctx, bc.mgr.GetPDClient(), backupTS) + if err != nil { + return 0, errors.Trace(err) } - backupTS := utils.EncodeTs(ts) log.Info("backup encode timestamp", zap.Uint64("BackupTS", backupTS)) return backupTS, nil } @@ -281,7 +282,7 @@ func (bc *Client) BackupRanges( ranges []Range, lastBackupTS uint64, backupTS uint64, - rate uint64, + rateLimit uint64, concurrency uint32, updateCh chan<- struct{}, ) error { @@ -297,7 +298,7 @@ func (bc *Client) BackupRanges( go func() { for _, r := range ranges { err := bc.backupRange( - ctx, r.StartKey, r.EndKey, lastBackupTS, backupTS, rate, concurrency, updateCh) + ctx, r.StartKey, r.EndKey, lastBackupTS, backupTS, rateLimit, concurrency, updateCh) if err != nil { errCh <- err return @@ -342,7 +343,7 @@ func (bc *Client) backupRange( startKey, endKey []byte, lastBackupTS uint64, backupTS uint64, - rateMBs uint64, + rateLimit uint64, concurrency uint32, updateCh chan<- struct{}, ) (err error) { @@ -357,12 +358,10 @@ func (bc *Client) backupRange( summary.CollectSuccessUnit(key, elapsed) } }() - // The unit of rate limit in protocol is bytes per second. - rateLimit := rateMBs * 1024 * 1024 log.Info("backup started", zap.Binary("StartKey", startKey), zap.Binary("EndKey", endKey), - zap.Uint64("RateLimit", rateMBs), + zap.Uint64("RateLimit", rateLimit), zap.Uint32("Concurrency", concurrency)) ctx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/pkg/backup/client_test.go b/pkg/backup/client_test.go index 6971026d5..44ca1ad5a 100644 --- a/pkg/backup/client_test.go +++ b/pkg/backup/client_test.go @@ -10,11 +10,11 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/mockstore/mocktikv" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/br/pkg/conn" - "github.com/pingcap/br/pkg/utils" ) type testBackup struct { @@ -61,7 +61,7 @@ func (r *testBackup) TestGetTS(c *C) { currentTs := time.Now().UnixNano() / int64(time.Millisecond) ts, err := r.backupClient.GetTS(r.ctx, timeAgo) c.Assert(err, IsNil) - pdTs := utils.DecodeTs(ts).Physical + pdTs := oracle.ExtractPhysical(ts) duration := int(currentTs - pdTs) c.Assert(duration, Greater, expectedDuration-deviation) c.Assert(duration, Less, expectedDuration+deviation) @@ -72,27 +72,30 @@ func (r *testBackup) TestGetTS(c *C) { currentTs = time.Now().UnixNano() / int64(time.Millisecond) ts, err = r.backupClient.GetTS(r.ctx, timeAgo) c.Assert(err, IsNil) - pdTs = utils.DecodeTs(ts).Physical + pdTs = oracle.ExtractPhysical(ts) duration = int(currentTs - pdTs) c.Assert(duration, Greater, expectedDuration-deviation) c.Assert(duration, Less, expectedDuration+deviation) // timeago = "-1m" timeAgo = "-1m" - expectedDuration = -60000 - currentTs = time.Now().UnixNano() / int64(time.Millisecond) - ts, err = r.backupClient.GetTS(r.ctx, timeAgo) - c.Assert(err, IsNil) - pdTs = utils.DecodeTs(ts).Physical - duration = int(currentTs - pdTs) - c.Assert(duration, Greater, expectedDuration-deviation) - c.Assert(duration, Less, expectedDuration+deviation) + _, err = r.backupClient.GetTS(r.ctx, timeAgo) + c.Assert(err, ErrorMatches, "negative timeago is not allowed") - // timeago = "1000000h" exceed GCSafePoint - // because GCSafePoint in mockPDClient is 0 + // timeago = "1000000h" overflows timeAgo = "1000000h" _, err = r.backupClient.GetTS(r.ctx, timeAgo) - c.Assert(err, ErrorMatches, "given backup time exceed GCSafePoint") + c.Assert(err, ErrorMatches, "backup ts overflow.*") + + // timeago = "10h" exceed GCSafePoint + p, l, err := r.backupClient.mgr.GetPDClient().GetTS(r.ctx) + c.Assert(err, IsNil) + now := oracle.ComposeTS(p, l) + _, err = r.backupClient.mgr.GetPDClient().UpdateGCSafePoint(r.ctx, now) + c.Assert(err, IsNil) + timeAgo = "10h" + _, err = r.backupClient.GetTS(r.ctx, timeAgo) + c.Assert(err, ErrorMatches, "GC safepoint [0-9]+ exceed TS [0-9]+") } func (r *testBackup) TestBuildTableRange(c *C) { diff --git a/pkg/backup/safe_point.go b/pkg/backup/safe_point.go index bc24a01ba..bb73bc7d9 100644 --- a/pkg/backup/safe_point.go +++ b/pkg/backup/safe_point.go @@ -7,32 +7,29 @@ import ( "github.com/pingcap/log" pd "github.com/pingcap/pd/client" "go.uber.org/zap" - - "github.com/pingcap/br/pkg/utils" ) -// GetGCSafePoint returns the current gc safe point. +// getGCSafePoint returns the current gc safe point. // TODO: Some cluster may not enable distributed GC. -func GetGCSafePoint(ctx context.Context, pdClient pd.Client) (utils.Timestamp, error) { +func getGCSafePoint(ctx context.Context, pdClient pd.Client) (uint64, error) { safePoint, err := pdClient.UpdateGCSafePoint(ctx, 0) if err != nil { - return utils.Timestamp{}, err + return 0, err } - return utils.DecodeTs(safePoint), nil + return safePoint, nil } // CheckGCSafepoint checks whether the ts is older than GC safepoint. // Note: It ignores errors other than exceed GC safepoint. func CheckGCSafepoint(ctx context.Context, pdClient pd.Client, ts uint64) error { // TODO: use PDClient.GetGCSafePoint instead once PD client exports it. - safePoint, err := GetGCSafePoint(ctx, pdClient) + safePoint, err := getGCSafePoint(ctx, pdClient) if err != nil { log.Warn("fail to get GC safe point", zap.Error(err)) return nil } - safePointTS := utils.EncodeTs(safePoint) - if ts <= safePointTS { - return errors.Errorf("GC safepoint %d exceed TS %d", safePointTS, ts) + if ts <= safePoint { + return errors.Errorf("GC safepoint %d exceed TS %d", safePoint, ts) } return nil } diff --git a/pkg/restore/client.go b/pkg/restore/client.go index 02875cdb2..9714edc2a 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -16,6 +16,7 @@ import ( restore_util "github.com/pingcap/tidb-tools/pkg/restore-util" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/store/tikv/oracle" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/backoff" @@ -129,11 +130,7 @@ func (rc *Client) GetTS(ctx context.Context) (uint64, error) { if err != nil { return 0, errors.Trace(err) } - ts := utils.Timestamp{ - Physical: p, - Logical: l, - } - restoreTS := utils.EncodeTs(ts) + restoreTS := oracle.ComposeTS(p, l) return restoreTS, nil } diff --git a/pkg/utils/tso.go b/pkg/utils/tso.go index 44c23fc48..a4ca5f5b5 100644 --- a/pkg/utils/tso.go +++ b/pkg/utils/tso.go @@ -8,36 +8,12 @@ import ( "strings" "github.com/pingcap/errors" - "github.com/pingcap/tidb/store/tikv/oracle" ) const ( resetTSURL = "/pd/api/v1/admin/reset-ts" ) -// Timestamp is composed by a physical unix timestamp and a logical timestamp. -type Timestamp struct { - Physical int64 - Logical int64 -} - -const physicalShiftBits = 18 - -// DecodeTs decodes Timestamp from a uint64 -func DecodeTs(ts uint64) Timestamp { - physical := oracle.ExtractPhysical(ts) - logical := ts - (uint64(physical) << physicalShiftBits) - return Timestamp{ - Physical: physical, - Logical: int64(logical), - } -} - -// EncodeTs encodes Timestamp into a uint64 -func EncodeTs(tp Timestamp) uint64 { - return uint64((tp.Physical << physicalShiftBits) + tp.Logical) -} - // ResetTS resets the timestamp of PD to a bigger value func ResetTS(pdAddr string, ts uint64) error { req, err := json.Marshal(struct { diff --git a/pkg/utils/tso_test.go b/pkg/utils/tso_test.go deleted file mode 100644 index 3e6ecd9e5..000000000 --- a/pkg/utils/tso_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package utils - -import ( - "math/rand" - "time" - - . "github.com/pingcap/check" -) - -type testTsoSuite struct{} - -var _ = Suite(&testTsoSuite{}) - -func (r *testTsoSuite) TestTimestampEncodeDecode(c *C) { - rand.Seed(time.Now().UnixNano()) - for i := 0; i < 10; i++ { - ts := rand.Uint64() - tp := DecodeTs(ts) - ts1 := EncodeTs(tp) - c.Assert(ts, DeepEquals, ts1) - } -} diff --git a/pkg/utils/unit.go b/pkg/utils/unit.go index 5f8009878..a12dcb6c2 100644 --- a/pkg/utils/unit.go +++ b/pkg/utils/unit.go @@ -2,7 +2,7 @@ package utils // unit of storage const ( - B = 1 << (iota * 10) + B = uint64(1) << (iota * 10) KB MB GB diff --git a/pkg/utils/unit_test.go b/pkg/utils/unit_test.go new file mode 100644 index 000000000..5b3c00530 --- /dev/null +++ b/pkg/utils/unit_test.go @@ -0,0 +1,17 @@ +package utils + +import ( + . "github.com/pingcap/check" +) + +type testUnitSuite struct{} + +var _ = Suite(&testUnitSuite{}) + +func (r *testUnitSuite) TestLoadBackupMeta(c *C) { + c.Assert(B, Equals, uint64(1)) + c.Assert(KB, Equals, uint64(1024)) + c.Assert(MB, Equals, uint64(1024*1024)) + c.Assert(GB, Equals, uint64(1024*1024*1024)) + c.Assert(TB, Equals, uint64(1024*1024*1024*1024)) +} diff --git a/tests/br_z_gc_safepoint/gc.go b/tests/br_z_gc_safepoint/gc.go new file mode 100644 index 000000000..a18367259 --- /dev/null +++ b/tests/br_z_gc_safepoint/gc.go @@ -0,0 +1,64 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +// Test backup with exceeding GC safe point. + +package main + +import ( + "context" + "flag" + "time" + + "github.com/pingcap/log" + pd "github.com/pingcap/pd/client" + "github.com/pingcap/tidb/store/tikv/oracle" + "go.uber.org/zap" +) + +var ( + pdAddr = flag.String("pd", "", "PD address") + gcOffset = flag.Duration("gc-offset", time.Second*10, + "Set GC safe point to current time - gc-offset, default: 10s") +) + +func main() { + flag.Parse() + if *pdAddr == "" { + log.Fatal("pd address is empty") + } + if *gcOffset == time.Duration(0) { + log.Fatal("zero gc-offset is not allowed") + } + + timeout := time.Second * 10 + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + pdclient, err := pd.NewClientWithContext(ctx, []string{*pdAddr}, pd.SecurityOption{}) + if err != nil { + log.Fatal("create pd client failed", zap.Error(err)) + } + p, l, err := pdclient.GetTS(ctx) + if err != nil { + log.Fatal("get ts failed", zap.Error(err)) + } + now := oracle.ComposeTS(p, l) + nowMinusOffset := oracle.GetTimeFromTS(now).Add(-*gcOffset) + newSP := oracle.ComposeTS(oracle.GetPhysical(nowMinusOffset), 0) + _, err = pdclient.UpdateGCSafePoint(ctx, newSP) + if err != nil { + log.Fatal("create pd client failed", zap.Error(err)) + } + + log.Info("update GC safe point", zap.Uint64("SP", newSP), zap.Uint64("now", now)) +} diff --git a/tests/br_z_gc_safepoint/run.sh b/tests/br_z_gc_safepoint/run.sh new file mode 100755 index 000000000..916ca1fa8 --- /dev/null +++ b/tests/br_z_gc_safepoint/run.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test whether BR fails fast when backup ts exceeds GC safe point. +# It is call br_*z*_gc_safepoint, because it brings lots of write and +# slows down other tests to changing GC safe point. Adding a z prefix to run +# the test last. + +set -eu + +DB="$TEST_NAME" +TABLE="usertable" + +run_sql "CREATE DATABASE $DB;" + +go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB + +row_count_ori=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') + +# Update GC safepoint to now + 5s after 10s seconds. +sleep 10 && bin/gc -pd $PD_ADDR -gc-offset "5s" & + +# Set ratelimit to 1 bytes/second, we assume it can not finish within 10s, +# so it will trigger exceed GC safe point error. +backup_gc_fail=0 +echo "backup start (expect fail)..." +run_br --pd $PD_ADDR backup table -s "local://$TEST_DIR/$DB" --db $DB -t $TABLE --ratelimit 1 --ratelimit-unit 1 || backup_gc_fail=1 + +if [ "$backup_gc_fail" -ne "1" ];then + echo "TEST: [$TEST_NAME] failed!" + exit 1 +fi + +run_sql "DROP TABLE $DB.$TABLE;" diff --git a/tests/br_z_gc_safepoint/workload b/tests/br_z_gc_safepoint/workload new file mode 100644 index 000000000..448ca3c1a --- /dev/null +++ b/tests/br_z_gc_safepoint/workload @@ -0,0 +1,12 @@ +recordcount=1000 +operationcount=0 +workload=core + +readallfields=true + +readproportion=0 +updateproportion=0 +scanproportion=0 +insertproportion=0 + +requestdistribution=uniform \ No newline at end of file From 2aa13222a70f7baa93b09bb05ca171291a23f71d Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Thu, 16 Jan 2020 17:17:07 +0800 Subject: [PATCH 03/10] cmd: convert version command to flags (#144) * cmd: convert version command to flags Signed-off-by: Neil Shen * address comments Signed-off-by: Neil Shen --- cmd/cmd.go | 9 ++++++++- cmd/version.go | 20 -------------------- main.go | 1 - pkg/utils/version.go | 23 +++++++++++++++-------- tests/br_debug_meta/run.sh | 3 --- tests/br_other/run.sh | 3 ++- 6 files changed, 25 insertions(+), 34 deletions(-) delete mode 100644 cmd/version.go diff --git a/cmd/cmd.go b/cmd/cmd.go index e97adec43..468c35232 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/br/pkg/conn" "github.com/pingcap/br/pkg/storage" + "github.com/pingcap/br/pkg/utils" ) var ( @@ -52,12 +53,18 @@ const ( FlagSlowLogFile = "slow-log-file" flagDatabase = "db" + flagTable = "table" - flagTable = "table" + flagVersion = "version" + flagVersionShort = "V" ) // AddFlags adds flags to the given cmd. func AddFlags(cmd *cobra.Command) { + cmd.Version = utils.BRInfo() + cmd.Flags().BoolP(flagVersion, flagVersionShort, false, "Display version information about BR") + cmd.SetVersionTemplate("{{printf \"%s\" .Version}}\n") + cmd.PersistentFlags().StringP(FlagPD, "u", "127.0.0.1:2379", "PD address") cmd.PersistentFlags().String(FlagCA, "", "CA certificate path for TLS connection") cmd.PersistentFlags().String(FlagCert, "", "Certificate path for TLS connection") diff --git a/cmd/version.go b/cmd/version.go deleted file mode 100644 index af4f7d386..000000000 --- a/cmd/version.go +++ /dev/null @@ -1,20 +0,0 @@ -package cmd - -import ( - "github.com/spf13/cobra" - - "github.com/pingcap/br/pkg/utils" -) - -// NewVersionCommand returns a restore subcommand -func NewVersionCommand() *cobra.Command { - bp := &cobra.Command{ - Use: "version", - Short: "output version information", - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - utils.PrintBRInfo() - }, - } - return bp -} diff --git a/main.go b/main.go index 8ae652f86..103699614 100644 --- a/main.go +++ b/main.go @@ -47,7 +47,6 @@ func main() { cmd.AddFlags(rootCmd) cmd.SetDefaultContext(ctx) rootCmd.AddCommand( - cmd.NewVersionCommand(), cmd.NewValidateCommand(), cmd.NewBackupCommand(), cmd.NewRestoreCommand(), diff --git a/pkg/utils/version.go b/pkg/utils/version.go index bed19ffa9..13a3c7a92 100644 --- a/pkg/utils/version.go +++ b/pkg/utils/version.go @@ -1,7 +1,9 @@ package utils import ( + "bytes" "fmt" + "runtime" "github.com/pingcap/log" "github.com/pingcap/tidb/util/israce" @@ -16,25 +18,30 @@ var ( BRBuildTS = "None" BRGitHash = "None" BRGitBranch = "None" + goVersion = runtime.Version() ) -// LogBRInfo prints the BR version information. +// LogBRInfo logs version information about BR. func LogBRInfo() { log.Info("Welcome to Backup & Restore (BR)") log.Info("BR", zap.String("release-version", BRReleaseVersion)) log.Info("BR", zap.String("git-hash", BRGitHash)) log.Info("BR", zap.String("git-branch", BRGitBranch)) + log.Info("BR", zap.String("go-version", goVersion)) log.Info("BR", zap.String("utc-build-time", BRBuildTS)) log.Info("BR", zap.Bool("race-enabled", israce.RaceEnabled)) } -// PrintBRInfo prints the BR version information without log info. -func PrintBRInfo() { - fmt.Println("Release Version:", BRReleaseVersion) - fmt.Println("Git Commit Hash:", BRGitHash) - fmt.Println("Git Branch:", BRGitBranch) - fmt.Println("UTC Build Time: ", BRBuildTS) - fmt.Println("Race Enabled: ", israce.RaceEnabled) +// BRInfo returns version information about BR. +func BRInfo() string { + buf := bytes.Buffer{} + fmt.Fprintf(&buf, "Release Version: %s\n", BRReleaseVersion) + fmt.Fprintf(&buf, "Git Commit Hash: %s\n", BRGitHash) + fmt.Fprintf(&buf, "Git Branch: %s\n", BRGitBranch) + fmt.Fprintf(&buf, "Go Version: %s\n", goVersion) + fmt.Fprintf(&buf, "UTC Build Time: %s\n", BRBuildTS) + fmt.Fprintf(&buf, "Race Enabled: %t", israce.RaceEnabled) + return buf.String() } // LogArguments prints origin command arguments diff --git a/tests/br_debug_meta/run.sh b/tests/br_debug_meta/run.sh index 679602deb..1dcfccefe 100644 --- a/tests/br_debug_meta/run.sh +++ b/tests/br_debug_meta/run.sh @@ -57,6 +57,3 @@ then fi run_sql "DROP DATABASE $DB;" - -# Test version -run_br version \ No newline at end of file diff --git a/tests/br_other/run.sh b/tests/br_other/run.sh index 579c630df..e25dd2eae 100644 --- a/tests/br_other/run.sh +++ b/tests/br_other/run.sh @@ -55,4 +55,5 @@ fi run_sql "DROP DATABASE $DB;" # Test version -run_br version +run_br --version +run_br -V From 0c1dea4d905e7b115019698574b958c9cdf9648e Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Thu, 16 Jan 2020 19:00:24 +0800 Subject: [PATCH 04/10] *: add changelog and github templates (#143) Signed-off-by: Neil Shen --- .github/ISSUE_TEMPLATE/bug-report.md | 29 ++++++++++++++++++ .github/ISSUE_TEMPLATE/feature-request.md | 19 ++++++++++++ .github/pull_request_template.md | 37 +++++++++++++++++++++++ CHANGELOG.md | 18 +++++++++++ 4 files changed, 103 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/feature-request.md create mode 100644 .github/pull_request_template.md create mode 100644 CHANGELOG.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 000000000..872699919 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,29 @@ +--- +name: "🐛 Bug Report" +about: Something isn't working as expected +title: '' +labels: 'bug' +--- + +Please answer these questions before submitting your issue. Thanks! + +1. What did you do? +If possible, provide a recipe for reproducing the error. + + +2. What did you expect to see? + + + +3. What did you see instead? + + + +4. What version of BR and TiDB/TiKV/PD are you using? + + diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 000000000..e895af84d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,19 @@ +--- +name: "🚀 Feature Request" +about: I have a suggestion +labels: enhancement +--- + +## Feature Request + +### Describe your feature request related problem: + + +### Describe the feature you'd like: + + +### Describe alternatives you've considered: + + +### Teachability, Documentation, Adoption, Migration Strategy: + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..0f6ae8de0 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,37 @@ + + +### What problem does this PR solve? + + +### What is changed and how it works? + + +### Check List + +Tests + + - Unit test + - Integration test + - Manual test (add detailed scripts or steps below) + - No code + +Code changes + + - Has exported function/method change + - Has exported variable/fields change + - Has interface methods change + - Has persistent data change + +Side effects + + - Possible performance regression + - Increased code complexity + - Breaking backward compatibility + +Related changes + + - Need to cherry-pick to the release branch + - Need to update the documentation + - Need to be included in the release note diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..6db4cfd21 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +# BR (Backup and Restore) Change Log +All notable changes to this project are documented in this file. +See also, +- [TiDB Changelog](https://github.com/pingcap/tidb/blob/master/CHANGELOG.md), +- [TiKV Changelog](https://github.com/tikv/tikv/blob/master/CHANGELOG.md), +- [PD Changelog](https://github.com/pingcap/pd/blob/master/CHANGELOG.md). + +## [3.1.0-beta.1] - 2020-01-10 + +- Fix the inaccurate backup progress information [#127](https://github.com/pingcap/br/pull/127) +- Improve the performance of splitting Regions [#122](https://github.com/pingcap/br/pull/122) +- Add the backup and restore feature for partitioned tables [#137](https://github.com/pingcap/br/pull/137) +- Add the feature of automatically scheduling PD schedulers [#123](https://github.com/pingcap/br/pull/123) +- Fix the issue that data is overwritten after non `PKIsHandle` tables are restored [#139](https://github.com/pingcap/br/pull/139) + +## [3.1.0-beta] - 2019-12-20 + +Initial release of the distributed backup and restore tool From 0a43d2a28aa1a2fd0a4504192981a2a7c58652f0 Mon Sep 17 00:00:00 2001 From: Neil Shen Date: Wed, 22 Jan 2020 14:28:16 +0800 Subject: [PATCH 05/10] restore: merge tidb-tools/pkg/restore-util (#146) * restore-util: Implement split/scatter (#274) * implement split/scatter Signed-off-by: 5kbpers * init test Signed-off-by: 5kbpers * redesign output/input of the lib Signed-off-by: 5kbpers * update dependency Signed-off-by: 5kbpers * add commments and more tests Signed-off-by: 5kbpers * add ScanRegions interface to Client Signed-off-by: 5kbpers * fix potential data race Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * Apply suggestions from code review Co-Authored-By: kennytm * Update pkg/restore-util/client.go Co-Authored-By: kennytm * address comments Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * update dependency Signed-off-by: 5kbpers * resolve conflicts Signed-off-by: 5kbpers * fix prefix rewrite Signed-off-by: 5kbpers * add RewriteRule/skip failed scatter region/retry the SplitRegion Signed-off-by: 5kbpers * fix test Signed-off-by: 5kbpers * check if region has peer Signed-off-by: 5kbpers * more logs Signed-off-by: 5kbpers * restore-util: add split retry interval (#277) * reset dependencies to release-3.1 * add split retry interval Signed-off-by: 5kbpers * fix go.sum Signed-off-by: 5kbpers * restore-util: wait for scatter region sequentially (#279) * wait for scatter region sequentially Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * restore-util: add on split hook (#281) * restore-util: add on split hook Signed-off-by: Neil Shen * Nil check onSplit Co-Authored-By: kennytm * restore-util: fix returned new region is nil (#283) * restore-util: fix returned new region is nil Signed-off-by: 5kbpers * more logs Signed-off-by: 5kbpers * *: gofmt Signed-off-by: 5kbpers * Apply suggestions from code review Co-Authored-By: kennytm * fix log Signed-off-by: 5kbpers * restore-util: call onSplit on splitByRewriteRules (#285) Signed-off-by: Neil Shen * restore-util: fix overlapped error message (#293) * restore-util: fix overlapped error message Signed-off-by: 5kbpers * fix log message Signed-off-by: 5kbpers * reduce error trace Signed-off-by: 5kbpers * fix test Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * restore-util: log warning when cannot find matched rewrite rule (#299) * restore-util: add method to set placement rules and store labels (#301) * restore-util: add method to set placement rules and store labels Signed-off-by: disksing * minor fix Signed-off-by: disksing * address comment Signed-off-by: disksing * add GetPlacementRules Signed-off-by: disksing * fix test Signed-off-by: disksing * restore-util: support batch split (#300) * restore-util: support batch split Signed-off-by: 5kbpers * go fmt Signed-off-by: 5kbpers * Apply suggestions from code review Co-Authored-By: kennytm * address commits Signed-off-by: 5kbpers * Update pkg/restore-util/split.go Co-Authored-By: kennytm * add onSplit callback Signed-off-by: 5kbpers * fix test Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * restore-util: add upper bound time for waiting for scatter (#305) * restore: fix scatter regions failed Signed-off-by: 5kbpers * add log Signed-off-by: 5kbpers * stop waiting for scatter after 3min Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * restore-util: fix wrong url (#306) Signed-off-by: disksing * restore-util: add warning about unmatched table id (#313) * restore-util: support table partition Signed-off-by: 5kbpers * fix log Signed-off-by: 5kbpers * warn table id does not match Signed-off-by: 5kbpers * add unit tests Signed-off-by: 5kbpers * Apply suggestions from code review Co-Authored-By: Neil Shen * fix compile error Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * fix test Signed-off-by: 5kbpers Co-authored-by: Ian Co-authored-by: Neil Shen * *: prune tidb-tools Signed-off-by: Neil Shen * restore: address linters suggestions Signed-off-by: Neil Shen * restore: merge restoreutil into restore Signed-off-by: Neil Shen * address comment Signed-off-by: Neil Shen Co-authored-by: 5kbpers <20279863+5kbpers@users.noreply.github.com> Co-authored-by: kennytm Co-authored-by: disksing Co-authored-by: Ian --- .golangci.yml | 7 +- cmd/validate.go | 9 +- go.mod | 1 - go.sum | 3 +- pkg/restore/client.go | 13 +- pkg/restore/import.go | 33 ++-- pkg/restore/range.go | 148 +++++++++++++++ pkg/restore/range_test.go | 75 ++++++++ pkg/restore/split.go | 305 +++++++++++++++++++++++++++++++ pkg/restore/split_client.go | 353 ++++++++++++++++++++++++++++++++++++ pkg/restore/split_test.go | 301 ++++++++++++++++++++++++++++++ pkg/restore/util.go | 29 ++- pkg/restore/util_test.go | 3 +- 13 files changed, 1228 insertions(+), 52 deletions(-) create mode 100644 pkg/restore/range.go create mode 100644 pkg/restore/range_test.go create mode 100644 pkg/restore/split.go create mode 100644 pkg/restore/split_client.go create mode 100644 pkg/restore/split_test.go diff --git a/.golangci.yml b/.golangci.yml index 969cac759..1b025678e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -9,7 +9,8 @@ issues: text: "Potential HTTP request made with variable url" linters: - gosec - - path: .go - text: "Use of weak random number generator" + # TODO Remove it. + - path: split_client.go + text: "SA1019:" linters: - - gosec + - staticcheck diff --git a/cmd/validate.go b/cmd/validate.go index dd1e11fb0..8ba72b372 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -15,7 +15,6 @@ import ( "github.com/pingcap/log" "github.com/pingcap/parser/model" "github.com/pingcap/pd/pkg/mock/mockid" - restore_util "github.com/pingcap/tidb-tools/pkg/restore-util" "github.com/spf13/cobra" "go.uber.org/zap" @@ -187,15 +186,15 @@ func newBackupMetaCommand() *cobra.Command { tables = append(tables, db.Tables...) } // Check if the ranges of files overlapped - rangeTree := restore_util.NewRangeTree() + rangeTree := restore.NewRangeTree() for _, file := range files { - if out := rangeTree.InsertRange(restore_util.Range{ + if out := rangeTree.InsertRange(restore.Range{ StartKey: file.GetStartKey(), EndKey: file.GetEndKey(), }); out != nil { log.Error( "file ranges overlapped", - zap.Stringer("out", out.(*restore_util.Range)), + zap.Stringer("out", out.(*restore.Range)), zap.Stringer("file", file), ) } @@ -206,7 +205,7 @@ func newBackupMetaCommand() *cobra.Command { for offset := uint64(0); offset < tableIDOffset; offset++ { _, _ = tableIDAllocator.Alloc() // Ignore error } - rewriteRules := &restore_util.RewriteRules{ + rewriteRules := &restore.RewriteRules{ Table: make([]*import_sstpb.RewriteRule, 0), Data: make([]*import_sstpb.RewriteRule, 0), } diff --git a/go.mod b/go.mod index 8e50bbf35..9951c2922 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,6 @@ require ( github.com/pingcap/parser v0.0.0-20191210060830-bdf23a7ade01 github.com/pingcap/pd v1.1.0-beta.0.20191212045800-234784c7a9c5 github.com/pingcap/tidb v1.1.0-beta.0.20191213040028-9009da737834 - github.com/pingcap/tidb-tools v3.1.0-beta.0.20191223064326-e9c7a23a8dcb+incompatible github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33 github.com/prometheus/client_golang v1.0.0 github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum index 696ccee81..085e00355 100644 --- a/go.sum +++ b/go.sum @@ -283,9 +283,8 @@ github.com/pingcap/sysutil v0.0.0-20191126040022-986c5b3ed9a3 h1:HCNif3lukL83gNC github.com/pingcap/sysutil v0.0.0-20191126040022-986c5b3ed9a3/go.mod h1:Futrrmuw98pEsbEmoPsjw8aKLCmixwHEmT2rF+AsXGw= github.com/pingcap/tidb v1.1.0-beta.0.20191213040028-9009da737834 h1:eNf7bDY39moIzzcs5+PhLLW0BM2D2yrzFbjW/X42y0s= github.com/pingcap/tidb v1.1.0-beta.0.20191213040028-9009da737834/go.mod h1:VWx47QOXISBHHtZeWrDQlBOdbvth9TE9gei6QpoqJ4g= +github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tidb-tools v3.1.0-beta.0.20191223064326-e9c7a23a8dcb+incompatible h1:GxWxXVqA2aAZIgS+bEpasJkkspu9Jom1/oB2NmP7t/o= -github.com/pingcap/tidb-tools v3.1.0-beta.0.20191223064326-e9c7a23a8dcb+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33 h1:cTSaVv1hue17BCPqt+sURADTFSMpSD26ZuvKRyYIjJs= github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/pkg/restore/client.go b/pkg/restore/client.go index 9714edc2a..3030ba857 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -13,7 +13,6 @@ import ( "github.com/pingcap/log" "github.com/pingcap/parser/model" pd "github.com/pingcap/pd/client" - restore_util "github.com/pingcap/tidb-tools/pkg/restore-util" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/tikv/oracle" @@ -108,7 +107,7 @@ func (rc *Client) InitBackupMeta(backupMeta *backup.BackupMeta, backend *backup. rc.databases = databases rc.backupMeta = backupMeta - metaClient := restore_util.NewClient(rc.pdClient) + metaClient := NewSplitClient(rc.pdClient) importClient := NewImportClient(metaClient) rc.fileImporter = NewFileImporter(rc.ctx, metaClient, importClient, backend, rc.rateLimit) return nil @@ -189,8 +188,8 @@ func (rc *Client) CreateTables( dom *domain.Domain, tables []*utils.Table, newTS uint64, -) (*restore_util.RewriteRules, []*model.TableInfo, error) { - rewriteRules := &restore_util.RewriteRules{ +) (*RewriteRules, []*model.TableInfo, error) { + rewriteRules := &RewriteRules{ Table: make([]*import_sstpb.RewriteRule, 0), Data: make([]*import_sstpb.RewriteRule, 0), } @@ -232,7 +231,7 @@ func (rc *Client) setSpeedLimit() error { // RestoreTable tries to restore the data of a table. func (rc *Client) RestoreTable( table *utils.Table, - rewriteRules *restore_util.RewriteRules, + rewriteRules *RewriteRules, updateCh chan<- struct{}, ) (err error) { start := time.Now() @@ -300,7 +299,7 @@ func (rc *Client) RestoreTable( // RestoreDatabase tries to restore the data of a database func (rc *Client) RestoreDatabase( db *utils.Database, - rewriteRules *restore_util.RewriteRules, + rewriteRules *RewriteRules, updateCh chan<- struct{}, ) (err error) { start := time.Now() @@ -336,7 +335,7 @@ func (rc *Client) RestoreDatabase( // RestoreAll tries to restore all the data of backup files. func (rc *Client) RestoreAll( - rewriteRules *restore_util.RewriteRules, + rewriteRules *RewriteRules, updateCh chan<- struct{}, ) (err error) { start := time.Now() diff --git a/pkg/restore/import.go b/pkg/restore/import.go index fc09b7b16..77273ebab 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -12,7 +12,6 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/log" "github.com/pingcap/pd/pkg/codec" - restore_util "github.com/pingcap/tidb-tools/pkg/restore-util" "go.uber.org/zap" "google.golang.org/grpc" @@ -60,12 +59,12 @@ type ImporterClient interface { type importClient struct { mu sync.Mutex - metaClient restore_util.Client + metaClient SplitClient clients map[uint64]import_sstpb.ImportSSTClient } // NewImportClient returns a new ImporterClient -func NewImportClient(metaClient restore_util.Client) ImporterClient { +func NewImportClient(metaClient SplitClient) ImporterClient { return &importClient{ metaClient: metaClient, clients: make(map[uint64]import_sstpb.ImportSSTClient), @@ -133,7 +132,7 @@ func (ic *importClient) getImportClient( // FileImporter used to import a file to TiKV. type FileImporter struct { - metaClient restore_util.Client + metaClient SplitClient importClient ImporterClient backend *backup.StorageBackend rateLimit uint64 @@ -145,7 +144,7 @@ type FileImporter struct { // NewFileImporter returns a new file importClient. func NewFileImporter( ctx context.Context, - metaClient restore_util.Client, + metaClient SplitClient, importClient ImporterClient, backend *backup.StorageBackend, rateLimit uint64, @@ -163,7 +162,7 @@ func NewFileImporter( // Import tries to import a file. // All rules must contain encoded keys. -func (importer *FileImporter) Import(file *backup.File, rewriteRules *restore_util.RewriteRules) error { +func (importer *FileImporter) Import(file *backup.File, rewriteRules *RewriteRules) error { log.Debug("import file", zap.Stringer("file", file)) // Rewrite the start key and end key of file to scan regions startKey, endKey, err := rewriteFileKeys(file, rewriteRules) @@ -179,9 +178,9 @@ func (importer *FileImporter) Import(file *backup.File, rewriteRules *restore_ut ctx, cancel := context.WithTimeout(importer.ctx, importScanResgionTime) defer cancel() // Scan regions covered by the file range - regionInfos, err := importer.metaClient.ScanRegions(ctx, startKey, endKey, 0) - if err != nil { - return errors.Trace(err) + regionInfos, err1 := importer.metaClient.ScanRegions(ctx, startKey, endKey, 0) + if err1 != nil { + return errors.Trace(err1) } log.Debug("scan regions", zap.Stringer("file", file), zap.Int("count", len(regionInfos))) // Try to download and ingest the file in every region @@ -190,20 +189,20 @@ func (importer *FileImporter) Import(file *backup.File, rewriteRules *restore_ut info := regionInfo // Try to download file. err = withRetry(func() error { - var err error + var err2 error var isEmpty bool - downloadMeta, isEmpty, err = importer.downloadSST(info, file, rewriteRules) - if err != nil { + downloadMeta, isEmpty, err2 = importer.downloadSST(info, file, rewriteRules) + if err2 != nil { if err != errRewriteRuleNotFound { log.Warn("download file failed", zap.Stringer("file", file), zap.Stringer("region", info.Region), zap.Binary("startKey", startKey), zap.Binary("endKey", endKey), - zap.Error(err), + zap.Error(err2), ) } - return err + return err2 } if isEmpty { log.Info( @@ -255,9 +254,9 @@ func (importer *FileImporter) setDownloadSpeedLimit(storeID uint64) error { } func (importer *FileImporter) downloadSST( - regionInfo *restore_util.RegionInfo, + regionInfo *RegionInfo, file *backup.File, - rewriteRules *restore_util.RewriteRules, + rewriteRules *RewriteRules, ) (*import_sstpb.SSTMeta, bool, error) { id, err := uuid.New().MarshalBinary() if err != nil { @@ -312,7 +311,7 @@ func (importer *FileImporter) downloadSST( func (importer *FileImporter) ingestSST( sstMeta *import_sstpb.SSTMeta, - regionInfo *restore_util.RegionInfo, + regionInfo *RegionInfo, ) error { leader := regionInfo.Leader if leader == nil { diff --git a/pkg/restore/range.go b/pkg/restore/range.go new file mode 100644 index 000000000..f3914539e --- /dev/null +++ b/pkg/restore/range.go @@ -0,0 +1,148 @@ +package restore + +import ( + "bytes" + "fmt" + + "github.com/google/btree" + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/import_sstpb" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/log" + "github.com/pingcap/tidb/tablecodec" + "go.uber.org/zap" +) + +// Range represents a range of keys. +type Range struct { + StartKey []byte + EndKey []byte +} + +// String formats a range to a string +func (r *Range) String() string { + return fmt.Sprintf("[%x %x]", r.StartKey, r.EndKey) +} + +// Less compares a range with a btree.Item +func (r *Range) Less(than btree.Item) bool { + t := than.(*Range) + return len(r.EndKey) != 0 && bytes.Compare(r.EndKey, t.StartKey) <= 0 +} + +// contains returns if a key is included in the range. +func (r *Range) contains(key []byte) bool { + start, end := r.StartKey, r.EndKey + return bytes.Compare(key, start) >= 0 && + (len(end) == 0 || bytes.Compare(key, end) < 0) +} + +// sortRanges checks if the range overlapped and sort them +func sortRanges(ranges []Range, rewriteRules *RewriteRules) ([]Range, error) { + rangeTree := NewRangeTree() + for _, rg := range ranges { + if rewriteRules != nil { + startID := tablecodec.DecodeTableID(rg.StartKey) + endID := tablecodec.DecodeTableID(rg.EndKey) + var rule *import_sstpb.RewriteRule + if startID == endID { + rg.StartKey, rule = replacePrefix(rg.StartKey, rewriteRules) + if rule == nil { + log.Warn("cannot find rewrite rule", zap.Binary("key", rg.StartKey)) + } else { + log.Debug( + "rewrite start key", + zap.Binary("key", rg.StartKey), + zap.Stringer("rule", rule)) + } + rg.EndKey, rule = replacePrefix(rg.EndKey, rewriteRules) + if rule == nil { + log.Warn("cannot find rewrite rule", zap.Binary("key", rg.EndKey)) + } else { + log.Debug( + "rewrite end key", + zap.Binary("key", rg.EndKey), + zap.Stringer("rule", rule)) + } + } else { + log.Warn("table id does not match", + zap.Binary("startKey", rg.StartKey), + zap.Binary("endKey", rg.EndKey), + zap.Int64("startID", startID), + zap.Int64("endID", endID)) + return nil, errors.New("table id does not match") + } + } + if out := rangeTree.InsertRange(rg); out != nil { + return nil, errors.Errorf("ranges overlapped: %s, %s", out, rg) + } + } + sortedRanges := make([]Range, 0, len(ranges)) + rangeTree.Ascend(func(rg *Range) bool { + if rg == nil { + return false + } + sortedRanges = append(sortedRanges, *rg) + return true + }) + return sortedRanges, nil +} + +// RangeTree stores the ranges in an orderly manner. +// All the ranges it stored do not overlap. +type RangeTree struct { + tree *btree.BTree +} + +// NewRangeTree returns a new RangeTree. +func NewRangeTree() *RangeTree { + return &RangeTree{tree: btree.New(32)} +} + +// Find returns nil or a range in the range tree +func (rt *RangeTree) Find(key []byte) *Range { + var ret *Range + r := &Range{ + StartKey: key, + } + rt.tree.DescendLessOrEqual(r, func(i btree.Item) bool { + ret = i.(*Range) + return false + }) + if ret == nil || !ret.contains(key) { + return nil + } + return ret +} + +// InsertRange inserts ranges into the range tree. +// it returns true if all ranges inserted successfully. +// it returns false if there are some overlapped ranges. +func (rt *RangeTree) InsertRange(rg Range) btree.Item { + return rt.tree.ReplaceOrInsert(&rg) +} + +// RangeIterator allows callers of Ascend to iterate in-order over portions of +// the tree. When this function returns false, iteration will stop and the +// associated Ascend function will immediately return. +type RangeIterator func(rg *Range) bool + +// Ascend calls the iterator for every value in the tree within [first, last], +// until the iterator returns false. +func (rt *RangeTree) Ascend(iterator RangeIterator) { + rt.tree.Ascend(func(i btree.Item) bool { + return iterator(i.(*Range)) + }) +} + +// RegionInfo includes a region and the leader of the region. +type RegionInfo struct { + Region *metapb.Region + Leader *metapb.Peer +} + +// RewriteRules contains rules for rewriting keys of tables. +type RewriteRules struct { + Table []*import_sstpb.RewriteRule + Data []*import_sstpb.RewriteRule +} diff --git a/pkg/restore/range_test.go b/pkg/restore/range_test.go new file mode 100644 index 000000000..a9edc5b82 --- /dev/null +++ b/pkg/restore/range_test.go @@ -0,0 +1,75 @@ +package restore + +import ( + "bytes" + + . "github.com/pingcap/check" + "github.com/pingcap/kvproto/pkg/import_sstpb" + "github.com/pingcap/tidb/tablecodec" +) + +type testRangeSuite struct{} + +var _ = Suite(&testRangeSuite{}) + +type rangeEquals struct { + *CheckerInfo +} + +var RangeEquals Checker = &rangeEquals{ + &CheckerInfo{Name: "RangeEquals", Params: []string{"obtained", "expected"}}, +} + +func (checker *rangeEquals) Check(params []interface{}, names []string) (result bool, error string) { + obtained := params[0].([]Range) + expected := params[1].([]Range) + if len(obtained) != len(expected) { + return false, "" + } + for i := range obtained { + if !bytes.Equal(obtained[i].StartKey, expected[i].StartKey) || + !bytes.Equal(obtained[i].EndKey, expected[i].EndKey) { + return false, "" + } + } + return true, "" +} + +func (s *testRangeSuite) TestSortRange(c *C) { + dataRules := []*import_sstpb.RewriteRule{ + {OldKeyPrefix: tablecodec.GenTableRecordPrefix(1), NewKeyPrefix: tablecodec.GenTableRecordPrefix(4)}, + {OldKeyPrefix: tablecodec.GenTableRecordPrefix(2), NewKeyPrefix: tablecodec.GenTableRecordPrefix(5)}, + } + rewriteRules := &RewriteRules{ + Table: make([]*import_sstpb.RewriteRule, 0), + Data: dataRules, + } + ranges1 := []Range{ + {append(tablecodec.GenTableRecordPrefix(1), []byte("aaa")...), + append(tablecodec.GenTableRecordPrefix(1), []byte("bbb")...)}, + } + rs1, err := sortRanges(ranges1, rewriteRules) + c.Assert(err, IsNil, Commentf("sort range1 failed: %v", err)) + c.Assert(rs1, RangeEquals, []Range{ + {append(tablecodec.GenTableRecordPrefix(4), []byte("aaa")...), + append(tablecodec.GenTableRecordPrefix(4), []byte("bbb")...)}, + }) + + ranges2 := []Range{ + {append(tablecodec.GenTableRecordPrefix(1), []byte("aaa")...), + append(tablecodec.GenTableRecordPrefix(2), []byte("bbb")...)}, + } + _, err = sortRanges(ranges2, rewriteRules) + c.Assert(err, ErrorMatches, ".*table id does not match.*") + + ranges3 := initRanges() + rewriteRules1 := initRewriteRules() + rs3, err := sortRanges(ranges3, rewriteRules1) + c.Assert(err, IsNil, Commentf("sort range1 failed: %v", err)) + c.Assert(rs3, RangeEquals, []Range{ + {[]byte("bbd"), []byte("bbf")}, + {[]byte("bbf"), []byte("bbj")}, + {[]byte("xxa"), []byte("xxe")}, + {[]byte("xxe"), []byte("xxz")}, + }) +} diff --git a/pkg/restore/split.go b/pkg/restore/split.go new file mode 100644 index 000000000..31b23a60f --- /dev/null +++ b/pkg/restore/split.go @@ -0,0 +1,305 @@ +package restore + +import ( + "bytes" + "context" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/import_sstpb" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/log" + "github.com/pingcap/tidb/util/codec" + "go.uber.org/zap" +) + +// Constants for split retry machinery. +const ( + SplitRetryTimes = 32 + SplitRetryInterval = 50 * time.Millisecond + SplitMaxRetryInterval = time.Second + + SplitCheckMaxRetryTimes = 64 + SplitCheckInterval = 8 * time.Millisecond + SplitMaxCheckInterval = time.Second + + ScatterWaitMaxRetryTimes = 64 + ScatterWaitInterval = 50 * time.Millisecond + ScatterMaxWaitInterval = time.Second + + ScatterWaitUpperInterval = 180 * time.Second +) + +// RegionSplitter is a executor of region split by rules. +type RegionSplitter struct { + client SplitClient +} + +// NewRegionSplitter returns a new RegionSplitter. +func NewRegionSplitter(client SplitClient) *RegionSplitter { + return &RegionSplitter{ + client: client, + } +} + +// OnSplitFunc is called before split a range. +type OnSplitFunc func(key [][]byte) + +// Split executes a region split. It will split regions by the rewrite rules, +// then it will split regions by the end key of each range. +// tableRules includes the prefix of a table, since some ranges may have +// a prefix with record sequence or index sequence. +// note: all ranges and rewrite rules must have raw key. +func (rs *RegionSplitter) Split( + ctx context.Context, + ranges []Range, + rewriteRules *RewriteRules, + onSplit OnSplitFunc, +) error { + if len(ranges) == 0 { + return nil + } + startTime := time.Now() + // Sort the range for getting the min and max key of the ranges + sortedRanges, err := sortRanges(ranges, rewriteRules) + if err != nil { + return errors.Trace(err) + } + minKey := codec.EncodeBytes([]byte{}, sortedRanges[0].StartKey) + maxKey := codec.EncodeBytes([]byte{}, sortedRanges[len(sortedRanges)-1].EndKey) + for _, rule := range rewriteRules.Table { + if bytes.Compare(minKey, rule.GetNewKeyPrefix()) > 0 { + minKey = rule.GetNewKeyPrefix() + } + if bytes.Compare(maxKey, rule.GetNewKeyPrefix()) < 0 { + maxKey = rule.GetNewKeyPrefix() + } + } + for _, rule := range rewriteRules.Data { + if bytes.Compare(minKey, rule.GetNewKeyPrefix()) > 0 { + minKey = rule.GetNewKeyPrefix() + } + if bytes.Compare(maxKey, rule.GetNewKeyPrefix()) < 0 { + maxKey = rule.GetNewKeyPrefix() + } + } + interval := SplitRetryInterval + scatterRegions := make([]*RegionInfo, 0) +SplitRegions: + for i := 0; i < SplitRetryTimes; i++ { + var regions []*RegionInfo + regions, err = rs.client.ScanRegions(ctx, minKey, maxKey, 0) + if err != nil { + return errors.Trace(err) + } + if len(regions) == 0 { + log.Warn("cannot scan any region") + return nil + } + splitKeyMap := getSplitKeys(rewriteRules, sortedRanges, regions) + regionMap := make(map[uint64]*RegionInfo) + for _, region := range regions { + regionMap[region.Region.GetId()] = region + } + for regionID, keys := range splitKeyMap { + var newRegions []*RegionInfo + newRegions, err = rs.splitAndScatterRegions(ctx, regionMap[regionID], keys) + if err != nil { + interval = 2 * interval + if interval > SplitMaxRetryInterval { + interval = SplitMaxRetryInterval + } + time.Sleep(interval) + if i > 3 { + log.Warn("splitting regions failed, retry it", zap.Error(err)) + } + continue SplitRegions + } + scatterRegions = append(scatterRegions, newRegions...) + onSplit(keys) + } + break + } + if err != nil { + return errors.Trace(err) + } + log.Info("splitting regions done, wait for scattering regions", + zap.Int("regions", len(scatterRegions)), zap.Duration("take", time.Since(startTime))) + startTime = time.Now() + scatterCount := 0 + for _, region := range scatterRegions { + rs.waitForScatterRegion(ctx, region) + if time.Since(startTime) > ScatterWaitUpperInterval { + break + } + scatterCount++ + } + if scatterCount == len(scatterRegions) { + log.Info("waiting for scattering regions done", + zap.Int("regions", len(scatterRegions)), zap.Duration("take", time.Since(startTime))) + } else { + log.Warn("waiting for scattering regions timeout", + zap.Int("scatterCount", scatterCount), + zap.Int("regions", len(scatterRegions)), + zap.Duration("take", time.Since(startTime))) + } + return nil +} + +func (rs *RegionSplitter) hasRegion(ctx context.Context, regionID uint64) (bool, error) { + regionInfo, err := rs.client.GetRegionByID(ctx, regionID) + if err != nil { + return false, err + } + return regionInfo != nil, nil +} + +func (rs *RegionSplitter) isScatterRegionFinished(ctx context.Context, regionID uint64) (bool, error) { + resp, err := rs.client.GetOperator(ctx, regionID) + if err != nil { + return false, err + } + // Heartbeat may not be sent to PD + if respErr := resp.GetHeader().GetError(); respErr != nil { + if respErr.GetType() == pdpb.ErrorType_REGION_NOT_FOUND { + return true, nil + } + return false, errors.Errorf("get operator error: %s", respErr.GetType()) + } + retryTimes := ctx.Value(retryTimes).(int) + if retryTimes > 3 { + log.Warn("get operator", zap.Uint64("regionID", regionID), zap.Stringer("resp", resp)) + } + // If the current operator of the region is not 'scatter-region', we could assume + // that 'scatter-operator' has finished or timeout + ok := string(resp.GetDesc()) != "scatter-region" || resp.GetStatus() != pdpb.OperatorStatus_RUNNING + return ok, nil +} + +func (rs *RegionSplitter) waitForSplit(ctx context.Context, regionID uint64) { + interval := SplitCheckInterval + for i := 0; i < SplitCheckMaxRetryTimes; i++ { + ok, err := rs.hasRegion(ctx, regionID) + if err != nil { + log.Warn("wait for split failed", zap.Error(err)) + return + } + if ok { + break + } + interval = 2 * interval + if interval > SplitMaxCheckInterval { + interval = SplitMaxCheckInterval + } + time.Sleep(interval) + } +} + +type retryTimeKey struct{} + +var retryTimes = new(retryTimeKey) + +func (rs *RegionSplitter) waitForScatterRegion(ctx context.Context, regionInfo *RegionInfo) { + interval := ScatterWaitInterval + regionID := regionInfo.Region.GetId() + for i := 0; i < ScatterWaitMaxRetryTimes; i++ { + ctx1 := context.WithValue(ctx, retryTimes, i) + ok, err := rs.isScatterRegionFinished(ctx1, regionID) + if err != nil { + log.Warn("scatter region failed: do not have the region", + zap.Stringer("region", regionInfo.Region)) + return + } + if ok { + break + } + interval = 2 * interval + if interval > ScatterMaxWaitInterval { + interval = ScatterMaxWaitInterval + } + time.Sleep(interval) + } +} + +func (rs *RegionSplitter) splitAndScatterRegions( + ctx context.Context, regionInfo *RegionInfo, keys [][]byte, +) ([]*RegionInfo, error) { + newRegions, err := rs.client.BatchSplitRegions(ctx, regionInfo, keys) + if err != nil { + return nil, err + } + for _, region := range newRegions { + // Wait for a while until the regions successfully splits. + rs.waitForSplit(ctx, region.Region.Id) + if err = rs.client.ScatterRegion(ctx, region); err != nil { + log.Warn("scatter region failed", zap.Stringer("region", region.Region), zap.Error(err)) + } + } + return newRegions, nil +} + +// getSplitKeys checks if the regions should be split by the new prefix of the rewrites rule and the end key of +// the ranges, groups the split keys by region id +func getSplitKeys(rewriteRules *RewriteRules, ranges []Range, regions []*RegionInfo) map[uint64][][]byte { + splitKeyMap := make(map[uint64][][]byte) + checkKeys := make([][]byte, 0) + for _, rule := range rewriteRules.Table { + checkKeys = append(checkKeys, rule.GetNewKeyPrefix()) + } + for _, rule := range rewriteRules.Data { + checkKeys = append(checkKeys, rule.GetNewKeyPrefix()) + } + for _, rg := range ranges { + checkKeys = append(checkKeys, rg.EndKey) + } + for _, key := range checkKeys { + if region := needSplit(key, regions); region != nil { + splitKeys, ok := splitKeyMap[region.Region.GetId()] + if !ok { + splitKeys = make([][]byte, 0, 1) + } + splitKeyMap[region.Region.GetId()] = append(splitKeys, key) + } + } + return splitKeyMap +} + +// needSplit checks whether a key is necessary to split, if true returns the split region +func needSplit(splitKey []byte, regions []*RegionInfo) *RegionInfo { + // If splitKey is the max key. + if len(splitKey) == 0 { + return nil + } + splitKey = codec.EncodeBytes([]byte{}, splitKey) + for _, region := range regions { + // If splitKey is the boundary of the region + if bytes.Equal(splitKey, region.Region.GetStartKey()) { + return nil + } + // If splitKey is in a region + if bytes.Compare(splitKey, region.Region.GetStartKey()) > 0 && beforeEnd(splitKey, region.Region.GetEndKey()) { + return region + } + } + return nil +} + +func beforeEnd(key []byte, end []byte) bool { + return bytes.Compare(key, end) < 0 || len(end) == 0 +} + +func replacePrefix(s []byte, rewriteRules *RewriteRules) ([]byte, *import_sstpb.RewriteRule) { + // We should search the dataRules firstly. + for _, rule := range rewriteRules.Data { + if bytes.HasPrefix(s, rule.GetOldKeyPrefix()) { + return append(append([]byte{}, rule.GetNewKeyPrefix()...), s[len(rule.GetOldKeyPrefix()):]...), rule + } + } + for _, rule := range rewriteRules.Table { + if bytes.HasPrefix(s, rule.GetOldKeyPrefix()) { + return append(append([]byte{}, rule.GetNewKeyPrefix()...), s[len(rule.GetOldKeyPrefix()):]...), rule + } + } + + return s, nil +} diff --git a/pkg/restore/split_client.go b/pkg/restore/split_client.go new file mode 100644 index 000000000..8a618a191 --- /dev/null +++ b/pkg/restore/split_client.go @@ -0,0 +1,353 @@ +package restore + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "path" + "strconv" + "strings" + "sync" + + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/kvproto/pkg/tikvpb" + pd "github.com/pingcap/pd/client" + "github.com/pingcap/pd/server/schedule/placement" + "google.golang.org/grpc" +) + +// SplitClient is an external client used by RegionSplitter. +type SplitClient interface { + // GetStore gets a store by a store id. + GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) + // GetRegion gets a region which includes a specified key. + GetRegion(ctx context.Context, key []byte) (*RegionInfo, error) + // GetRegionByID gets a region by a region id. + GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) + // SplitRegion splits a region from a key, if key is not included in the region, it will return nil. + // note: the key should not be encoded + SplitRegion(ctx context.Context, regionInfo *RegionInfo, key []byte) (*RegionInfo, error) + // BatchSplitRegions splits a region from a batch of keys. + // note: the keys should not be encoded + BatchSplitRegions(ctx context.Context, regionInfo *RegionInfo, keys [][]byte) ([]*RegionInfo, error) + // ScatterRegion scatters a specified region. + ScatterRegion(ctx context.Context, regionInfo *RegionInfo) error + // GetOperator gets the status of operator of the specified region. + GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) + // ScanRegion gets a list of regions, starts from the region that contains key. + // Limit limits the maximum number of regions returned. + ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error) + // GetPlacementRule loads a placement rule from PD. + GetPlacementRule(ctx context.Context, groupID, ruleID string) (placement.Rule, error) + // SetPlacementRule insert or update a placement rule to PD. + SetPlacementRule(ctx context.Context, rule placement.Rule) error + // DeletePlacementRule removes a placement rule from PD. + DeletePlacementRule(ctx context.Context, groupID, ruleID string) error + // SetStoreLabel add or update specified label of stores. If labelValue + // is empty, it clears the label. + SetStoresLabel(ctx context.Context, stores []uint64, labelKey, labelValue string) error +} + +// pdClient is a wrapper of pd client, can be used by RegionSplitter. +type pdClient struct { + mu sync.Mutex + client pd.Client + storeCache map[uint64]*metapb.Store +} + +// NewSplitClient returns a client used by RegionSplitter. +func NewSplitClient(client pd.Client) SplitClient { + return &pdClient{ + client: client, + storeCache: make(map[uint64]*metapb.Store), + } +} + +func (c *pdClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) { + c.mu.Lock() + defer c.mu.Unlock() + store, ok := c.storeCache[storeID] + if ok { + return store, nil + } + store, err := c.client.GetStore(ctx, storeID) + if err != nil { + return nil, err + } + c.storeCache[storeID] = store + return store, nil + +} + +func (c *pdClient) GetRegion(ctx context.Context, key []byte) (*RegionInfo, error) { + region, leader, err := c.client.GetRegion(ctx, key) + if err != nil { + return nil, err + } + if region == nil { + return nil, nil + } + return &RegionInfo{ + Region: region, + Leader: leader, + }, nil +} + +func (c *pdClient) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) { + region, leader, err := c.client.GetRegionByID(ctx, regionID) + if err != nil { + return nil, err + } + if region == nil { + return nil, nil + } + return &RegionInfo{ + Region: region, + Leader: leader, + }, nil +} + +func (c *pdClient) SplitRegion(ctx context.Context, regionInfo *RegionInfo, key []byte) (*RegionInfo, error) { + var peer *metapb.Peer + if regionInfo.Leader != nil { + peer = regionInfo.Leader + } else { + if len(regionInfo.Region.Peers) == 0 { + return nil, errors.New("region does not have peer") + } + peer = regionInfo.Region.Peers[0] + } + storeID := peer.GetStoreId() + store, err := c.GetStore(ctx, storeID) + if err != nil { + return nil, err + } + conn, err := grpc.Dial(store.GetAddress(), grpc.WithInsecure()) + if err != nil { + return nil, err + } + defer conn.Close() + + client := tikvpb.NewTikvClient(conn) + resp, err := client.SplitRegion(ctx, &kvrpcpb.SplitRegionRequest{ + Context: &kvrpcpb.Context{ + RegionId: regionInfo.Region.Id, + RegionEpoch: regionInfo.Region.RegionEpoch, + Peer: peer, + }, + SplitKey: key, + }) + if err != nil { + return nil, err + } + if resp.RegionError != nil { + return nil, errors.Errorf("split region failed: region=%v, key=%x, err=%v", regionInfo.Region, key, resp.RegionError) + } + + // BUG: Left is deprecated, it may be nil even if split is succeed! + // Assume the new region is the left one. + newRegion := resp.GetLeft() + if newRegion == nil { + regions := resp.GetRegions() + for _, r := range regions { + if bytes.Equal(r.GetStartKey(), regionInfo.Region.GetStartKey()) { + newRegion = r + break + } + } + } + if newRegion == nil { + return nil, errors.New("split region failed: new region is nil") + } + var leader *metapb.Peer + // Assume the leaders will be at the same store. + if regionInfo.Leader != nil { + for _, p := range newRegion.GetPeers() { + if p.GetStoreId() == regionInfo.Leader.GetStoreId() { + leader = p + break + } + } + } + return &RegionInfo{ + Region: newRegion, + Leader: leader, + }, nil +} + +func (c *pdClient) BatchSplitRegions( + ctx context.Context, regionInfo *RegionInfo, keys [][]byte, +) ([]*RegionInfo, error) { + var peer *metapb.Peer + if regionInfo.Leader != nil { + peer = regionInfo.Leader + } else { + if len(regionInfo.Region.Peers) == 0 { + return nil, errors.New("region does not have peer") + } + peer = regionInfo.Region.Peers[0] + } + + storeID := peer.GetStoreId() + store, err := c.GetStore(ctx, storeID) + if err != nil { + return nil, err + } + conn, err := grpc.Dial(store.GetAddress(), grpc.WithInsecure()) + if err != nil { + return nil, err + } + defer conn.Close() + client := tikvpb.NewTikvClient(conn) + resp, err := client.SplitRegion(ctx, &kvrpcpb.SplitRegionRequest{ + Context: &kvrpcpb.Context{ + RegionId: regionInfo.Region.Id, + RegionEpoch: regionInfo.Region.RegionEpoch, + Peer: peer, + }, + SplitKeys: keys, + }) + if err != nil { + return nil, err + } + if resp.RegionError != nil { + return nil, errors.Errorf("split region failed: region=%v, err=%v", regionInfo.Region, resp.RegionError) + } + + regions := resp.GetRegions() + newRegionInfos := make([]*RegionInfo, 0, len(regions)) + for _, region := range regions { + // Skip the original region + if region.GetId() == regionInfo.Region.GetId() { + continue + } + var leader *metapb.Peer + // Assume the leaders will be at the same store. + if regionInfo.Leader != nil { + for _, p := range region.GetPeers() { + if p.GetStoreId() == regionInfo.Leader.GetStoreId() { + leader = p + break + } + } + } + newRegionInfos = append(newRegionInfos, &RegionInfo{ + Region: region, + Leader: leader, + }) + } + return newRegionInfos, nil +} + +func (c *pdClient) ScatterRegion(ctx context.Context, regionInfo *RegionInfo) error { + return c.client.ScatterRegion(ctx, regionInfo.Region.GetId()) +} + +func (c *pdClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { + return c.client.GetOperator(ctx, regionID) +} + +func (c *pdClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error) { + regions, leaders, err := c.client.ScanRegions(ctx, key, endKey, limit) + if err != nil { + return nil, err + } + regionInfos := make([]*RegionInfo, 0, len(regions)) + for i := range regions { + regionInfos = append(regionInfos, &RegionInfo{ + Region: regions[i], + Leader: leaders[i], + }) + } + return regionInfos, nil +} + +func (c *pdClient) GetPlacementRule(ctx context.Context, groupID, ruleID string) (placement.Rule, error) { + var rule placement.Rule + addr := c.getPDAPIAddr() + if addr == "" { + return rule, errors.New("failed to add stores labels: no leader") + } + req, _ := http.NewRequestWithContext(ctx, "GET", addr+path.Join("/pd/api/v1/config/rule", groupID, ruleID), nil) + res, err := http.DefaultClient.Do(req) + if err != nil { + return rule, errors.WithStack(err) + } + b, err := ioutil.ReadAll(res.Body) + if err != nil { + return rule, errors.WithStack(err) + } + res.Body.Close() + err = json.Unmarshal(b, &rule) + if err != nil { + return rule, errors.WithStack(err) + } + return rule, nil +} + +func (c *pdClient) SetPlacementRule(ctx context.Context, rule placement.Rule) error { + addr := c.getPDAPIAddr() + if addr == "" { + return errors.New("failed to add stores labels: no leader") + } + m, _ := json.Marshal(rule) + req, _ := http.NewRequestWithContext(ctx, "POST", addr+path.Join("/pd/api/v1/config/rule"), bytes.NewReader(m)) + res, err := http.DefaultClient.Do(req) + if err != nil { + return errors.WithStack(err) + } + return errors.Trace(res.Body.Close()) +} + +func (c *pdClient) DeletePlacementRule(ctx context.Context, groupID, ruleID string) error { + addr := c.getPDAPIAddr() + if addr == "" { + return errors.New("failed to add stores labels: no leader") + } + req, _ := http.NewRequestWithContext(ctx, "DELETE", addr+path.Join("/pd/api/v1/config/rule", groupID, ruleID), nil) + res, err := http.DefaultClient.Do(req) + if err != nil { + return errors.WithStack(err) + } + return errors.Trace(res.Body.Close()) +} + +func (c *pdClient) SetStoresLabel( + ctx context.Context, stores []uint64, labelKey, labelValue string, +) error { + b := []byte(fmt.Sprintf(`{"%s": "%s"}`, labelKey, labelValue)) + addr := c.getPDAPIAddr() + if addr == "" { + return errors.New("failed to add stores labels: no leader") + } + for _, id := range stores { + req, _ := http.NewRequestWithContext( + ctx, "POST", + addr+path.Join("/pd/api/v1/store", strconv.FormatUint(id, 10), "label"), + bytes.NewReader(b), + ) + res, err := http.DefaultClient.Do(req) + if err != nil { + return errors.WithStack(err) + } + err = res.Body.Close() + if err != nil { + return errors.Trace(err) + } + } + return nil +} + +func (c *pdClient) getPDAPIAddr() string { + addr := c.client.GetLeaderAddr() + if addr != "" && !strings.HasPrefix(addr, "http") { + addr = "http://" + addr + } + return strings.TrimRight(addr, "/") +} diff --git a/pkg/restore/split_test.go b/pkg/restore/split_test.go new file mode 100644 index 000000000..509c4cfa0 --- /dev/null +++ b/pkg/restore/split_test.go @@ -0,0 +1,301 @@ +package restore + +import ( + "bytes" + "context" + "sync" + + . "github.com/pingcap/check" + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/import_sstpb" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/pd/server/schedule/placement" + "github.com/pingcap/tidb/util/codec" +) + +type testClient struct { + mu sync.RWMutex + stores map[uint64]*metapb.Store + regions map[uint64]*RegionInfo + nextRegionID uint64 +} + +func newTestClient(stores map[uint64]*metapb.Store, regions map[uint64]*RegionInfo, nextRegionID uint64) *testClient { + return &testClient{ + stores: stores, + regions: regions, + nextRegionID: nextRegionID, + } +} + +func (c *testClient) GetAllRegions() map[uint64]*RegionInfo { + c.mu.RLock() + defer c.mu.RUnlock() + return c.regions +} + +func (c *testClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) { + c.mu.RLock() + defer c.mu.RUnlock() + store, ok := c.stores[storeID] + if !ok { + return nil, errors.Errorf("store not found") + } + return store, nil +} + +func (c *testClient) GetRegion(ctx context.Context, key []byte) (*RegionInfo, error) { + c.mu.RLock() + defer c.mu.RUnlock() + for _, region := range c.regions { + if bytes.Compare(key, region.Region.StartKey) >= 0 && + (len(region.Region.EndKey) == 0 || bytes.Compare(key, region.Region.EndKey) < 0) { + return region, nil + } + } + return nil, errors.Errorf("region not found: key=%s", string(key)) +} + +func (c *testClient) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) { + c.mu.RLock() + defer c.mu.RUnlock() + region, ok := c.regions[regionID] + if !ok { + return nil, errors.Errorf("region not found: id=%d", regionID) + } + return region, nil +} + +func (c *testClient) SplitRegion(ctx context.Context, regionInfo *RegionInfo, key []byte) (*RegionInfo, error) { + c.mu.Lock() + defer c.mu.Unlock() + var target *RegionInfo + splitKey := codec.EncodeBytes([]byte{}, key) + for _, region := range c.regions { + if bytes.Compare(splitKey, region.Region.StartKey) >= 0 && + (len(region.Region.EndKey) == 0 || bytes.Compare(splitKey, region.Region.EndKey) < 0) { + target = region + } + } + if target == nil { + return nil, errors.Errorf("region not found: key=%s", string(key)) + } + newRegion := &RegionInfo{ + Region: &metapb.Region{ + Peers: target.Region.Peers, + Id: c.nextRegionID, + StartKey: target.Region.StartKey, + EndKey: splitKey, + }, + } + c.regions[c.nextRegionID] = newRegion + c.nextRegionID++ + target.Region.StartKey = splitKey + c.regions[target.Region.Id] = target + return newRegion, nil +} + +func (c *testClient) BatchSplitRegions( + ctx context.Context, regionInfo *RegionInfo, keys [][]byte, +) ([]*RegionInfo, error) { + c.mu.Lock() + defer c.mu.Unlock() + newRegions := make([]*RegionInfo, 0) + for _, key := range keys { + var target *RegionInfo + splitKey := codec.EncodeBytes([]byte{}, key) + for _, region := range c.regions { + if bytes.Compare(splitKey, region.Region.GetStartKey()) > 0 && + beforeEnd(splitKey, region.Region.GetEndKey()) { + target = region + } + } + if target == nil { + continue + } + newRegion := &RegionInfo{ + Region: &metapb.Region{ + Peers: target.Region.Peers, + Id: c.nextRegionID, + StartKey: target.Region.StartKey, + EndKey: splitKey, + }, + } + c.regions[c.nextRegionID] = newRegion + c.nextRegionID++ + target.Region.StartKey = splitKey + c.regions[target.Region.Id] = target + newRegions = append(newRegions, newRegion) + } + return newRegions, nil +} + +func (c *testClient) ScatterRegion(ctx context.Context, regionInfo *RegionInfo) error { + return nil +} + +func (c *testClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { + return &pdpb.GetOperatorResponse{ + Header: new(pdpb.ResponseHeader), + }, nil +} + +func (c *testClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error) { + regions := make([]*RegionInfo, 0) + for _, region := range c.regions { + if limit > 0 && len(regions) >= limit { + break + } + if (len(region.Region.GetEndKey()) != 0 && bytes.Compare(region.Region.GetEndKey(), key) <= 0) || + bytes.Compare(region.Region.GetStartKey(), endKey) > 0 { + continue + } + regions = append(regions, region) + } + return regions, nil +} + +func (c *testClient) GetPlacementRule(ctx context.Context, groupID, ruleID string) (r placement.Rule, err error) { + return +} + +func (c *testClient) SetPlacementRule(ctx context.Context, rule placement.Rule) error { + return nil +} + +func (c *testClient) DeletePlacementRule(ctx context.Context, groupID, ruleID string) error { + return nil +} + +func (c *testClient) SetStoresLabel(ctx context.Context, stores []uint64, labelKey, labelValue string) error { + return nil +} + +// region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, ) +// range: [aaa, aae), [aae, aaz), [ccd, ccf), [ccf, ccj) +// rewrite rules: aa -> xx, cc -> bb +// expected regions after split: +// [, aay), [aay, bb), [bb, bba), [bba, bbf), [bbf, bbh), [bbh, bbj), +// [bbj, cca), [cca, xx), [xx, xxe), [xxe, xxz), [xxz, ) +func (s *testRestoreUtilSuite) TestSplit(c *C) { + client := initTestClient() + ranges := initRanges() + rewriteRules := initRewriteRules() + regionSplitter := NewRegionSplitter(client) + + ctx := context.Background() + err := regionSplitter.Split(ctx, ranges, rewriteRules, func(key [][]byte) {}) + if err != nil { + c.Assert(err, IsNil, Commentf("split regions failed: %v", err)) + } + regions := client.GetAllRegions() + if !validateRegions(regions) { + for _, region := range regions { + c.Logf("region: %v\n", region.Region) + } + c.Log("get wrong result") + c.Fail() + } +} + +// region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, ) +func initTestClient() *testClient { + peers := make([]*metapb.Peer, 1) + peers[0] = &metapb.Peer{ + Id: 1, + StoreId: 1, + } + keys := [6]string{"", "aay", "bba", "bbh", "cca", ""} + regions := make(map[uint64]*RegionInfo) + for i := uint64(1); i < 6; i++ { + startKey := []byte(keys[i-1]) + if len(startKey) != 0 { + startKey = codec.EncodeBytes([]byte{}, startKey) + } + endKey := []byte(keys[i]) + if len(endKey) != 0 { + endKey = codec.EncodeBytes([]byte{}, endKey) + } + regions[i] = &RegionInfo{ + Region: &metapb.Region{ + Id: i, + Peers: peers, + StartKey: startKey, + EndKey: endKey, + }, + } + } + stores := make(map[uint64]*metapb.Store) + stores[1] = &metapb.Store{ + Id: 1, + } + return newTestClient(stores, regions, 6) +} + +// range: [aaa, aae), [aae, aaz), [ccd, ccf), [ccf, ccj) +func initRanges() []Range { + var ranges [4]Range + ranges[0] = Range{ + StartKey: []byte("aaa"), + EndKey: []byte("aae"), + } + ranges[1] = Range{ + StartKey: []byte("aae"), + EndKey: []byte("aaz"), + } + ranges[2] = Range{ + StartKey: []byte("ccd"), + EndKey: []byte("ccf"), + } + ranges[3] = Range{ + StartKey: []byte("ccf"), + EndKey: []byte("ccj"), + } + return ranges[:] +} + +func initRewriteRules() *RewriteRules { + var rules [2]*import_sstpb.RewriteRule + rules[0] = &import_sstpb.RewriteRule{ + OldKeyPrefix: []byte("aa"), + NewKeyPrefix: []byte("xx"), + } + rules[1] = &import_sstpb.RewriteRule{ + OldKeyPrefix: []byte("cc"), + NewKeyPrefix: []byte("bb"), + } + return &RewriteRules{ + Table: rules[:], + Data: rules[:], + } +} + +// expected regions after split: +// [, aay), [aay, bb), [bb, bba), [bba, bbf), [bbf, bbh), [bbh, bbj), +// [bbj, cca), [cca, xx), [xx, xxe), [xxe, xxz), [xxz, ) +func validateRegions(regions map[uint64]*RegionInfo) bool { + keys := [12]string{"", "aay", "bb", "bba", "bbf", "bbh", "bbj", "cca", "xx", "xxe", "xxz", ""} + if len(regions) != 11 { + return false + } +FindRegion: + for i := 1; i < 12; i++ { + for _, region := range regions { + startKey := []byte(keys[i-1]) + if len(startKey) != 0 { + startKey = codec.EncodeBytes([]byte{}, startKey) + } + endKey := []byte(keys[i]) + if len(endKey) != 0 { + endKey = codec.EncodeBytes([]byte{}, endKey) + } + if bytes.Equal(region.Region.GetStartKey(), startKey) && + bytes.Equal(region.Region.GetEndKey(), endKey) { + continue FindRegion + } + } + return false + } + return true +} diff --git a/pkg/restore/util.go b/pkg/restore/util.go index 126e864fd..a2e9e3e38 100644 --- a/pkg/restore/util.go +++ b/pkg/restore/util.go @@ -13,7 +13,6 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/pingcap/parser/model" - restore_util "github.com/pingcap/tidb-tools/pkg/restore-util" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/codec" "go.uber.org/zap" @@ -76,7 +75,7 @@ func GetRewriteRules( newTable *model.TableInfo, oldTable *model.TableInfo, newTimeStamp uint64, -) *restore_util.RewriteRules { +) *RewriteRules { tableIDs := make(map[int64]int64) tableIDs[oldTable.ID] = newTable.ID if oldTable.Partition != nil { @@ -119,7 +118,7 @@ func GetRewriteRules( } } - return &restore_util.RewriteRules{ + return &RewriteRules{ Table: tableRules, Data: dataRules, } @@ -196,9 +195,9 @@ func withRetry( // ValidateFileRanges checks and returns the ranges of the files. func ValidateFileRanges( files []*backup.File, - rewriteRules *restore_util.RewriteRules, -) ([]restore_util.Range, error) { - ranges := make([]restore_util.Range, 0, len(files)) + rewriteRules *RewriteRules, +) ([]Range, error) { + ranges := make([]Range, 0, len(files)) fileAppended := make(map[string]bool) for _, file := range files { @@ -217,7 +216,7 @@ func ValidateFileRanges( zap.Stringer("file", file)) return nil, errors.New("table ids dont match") } - ranges = append(ranges, restore_util.Range{ + ranges = append(ranges, Range{ StartKey: file.GetStartKey(), EndKey: file.GetEndKey(), }) @@ -228,7 +227,7 @@ func ValidateFileRanges( } // ValidateFileRewriteRule uses rewrite rules to validate the ranges of a file -func ValidateFileRewriteRule(file *backup.File, rewriteRules *restore_util.RewriteRules) error { +func ValidateFileRewriteRule(file *backup.File, rewriteRules *RewriteRules) error { // Check if the start key has a matched rewrite key _, startRule := rewriteRawKey(file.GetStartKey(), rewriteRules) if rewriteRules != nil && startRule == nil { @@ -269,7 +268,7 @@ func ValidateFileRewriteRule(file *backup.File, rewriteRules *restore_util.Rewri } // Rewrites a raw key and returns a encoded key -func rewriteRawKey(key []byte, rewriteRules *restore_util.RewriteRules) ([]byte, *import_sstpb.RewriteRule) { +func rewriteRawKey(key []byte, rewriteRules *RewriteRules) ([]byte, *import_sstpb.RewriteRule) { if rewriteRules == nil { return codec.EncodeBytes([]byte{}, key), nil } @@ -281,7 +280,7 @@ func rewriteRawKey(key []byte, rewriteRules *restore_util.RewriteRules) ([]byte, return nil, nil } -func matchOldPrefix(key []byte, rewriteRules *restore_util.RewriteRules) *import_sstpb.RewriteRule { +func matchOldPrefix(key []byte, rewriteRules *RewriteRules) *import_sstpb.RewriteRule { for _, rule := range rewriteRules.Data { if bytes.HasPrefix(key, rule.GetOldKeyPrefix()) { return rule @@ -295,7 +294,7 @@ func matchOldPrefix(key []byte, rewriteRules *restore_util.RewriteRules) *import return nil } -func matchNewPrefix(key []byte, rewriteRules *restore_util.RewriteRules) *import_sstpb.RewriteRule { +func matchNewPrefix(key []byte, rewriteRules *RewriteRules) *import_sstpb.RewriteRule { for _, rule := range rewriteRules.Data { if bytes.HasPrefix(key, rule.GetNewKeyPrefix()) { return rule @@ -319,8 +318,8 @@ func truncateTS(key []byte) []byte { func SplitRanges( ctx context.Context, client *Client, - ranges []restore_util.Range, - rewriteRules *restore_util.RewriteRules, + ranges []Range, + rewriteRules *RewriteRules, updateCh chan<- struct{}, ) error { start := time.Now() @@ -328,7 +327,7 @@ func SplitRanges( elapsed := time.Since(start) summary.CollectDuration("split region", elapsed) }() - splitter := restore_util.NewRegionSplitter(restore_util.NewClient(client.GetPDClient())) + splitter := NewRegionSplitter(NewSplitClient(client.GetPDClient())) return splitter.Split(ctx, ranges, rewriteRules, func(keys [][]byte) { for range keys { updateCh <- struct{}{} @@ -336,7 +335,7 @@ func SplitRanges( }) } -func rewriteFileKeys(file *backup.File, rewriteRules *restore_util.RewriteRules) (startKey, endKey []byte, err error) { +func rewriteFileKeys(file *backup.File, rewriteRules *RewriteRules) (startKey, endKey []byte, err error) { startID := tablecodec.DecodeTableID(file.GetStartKey()) endID := tablecodec.DecodeTableID(file.GetEndKey()) var rule *import_sstpb.RewriteRule diff --git a/pkg/restore/util_test.go b/pkg/restore/util_test.go index 5da5c9ab7..bc4da9168 100644 --- a/pkg/restore/util_test.go +++ b/pkg/restore/util_test.go @@ -5,7 +5,6 @@ import ( "github.com/pingcap/kvproto/pkg/backup" "github.com/pingcap/kvproto/pkg/import_sstpb" "github.com/pingcap/kvproto/pkg/metapb" - restore_util "github.com/pingcap/tidb-tools/pkg/restore-util" "github.com/pingcap/tidb/tablecodec" ) @@ -34,7 +33,7 @@ func (s *testRestoreUtilSuite) TestGetSSTMetaFromFile(c *C) { } func (s *testRestoreUtilSuite) TestValidateFileRanges(c *C) { - rules := &restore_util.RewriteRules{ + rules := &RewriteRules{ Table: []*import_sstpb.RewriteRule{&import_sstpb.RewriteRule{ OldKeyPrefix: []byte(tablecodec.EncodeTablePrefix(1)), NewKeyPrefix: []byte(tablecodec.EncodeTablePrefix(2)), From 3863a3a477ee9875944ba851285a3fd42b34a5e5 Mon Sep 17 00:00:00 2001 From: Kolbe Kegel Date: Fri, 31 Jan 2020 23:40:50 -0800 Subject: [PATCH 06/10] Fixed handling for a dbName that do not exist in the backup being restored (#148) * Fixed handling for a dbName that do not exist in the backup being restored * Fixed handling for a dbName that do not exist in the backup being restored --- cmd/restore.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/restore.go b/cmd/restore.go index eee65ba86..4f66e47de 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -111,6 +111,9 @@ func runRestore(flagSet *flag.FlagSet, cmdName, dbName, tableName string) error case len(dbName) != 0 && len(tableName) == 0: // database restore db := client.GetDatabase(dbName) + if db == nil { + return errors.Errorf("database %s not found in backup", dbName) + } err = client.CreateDatabase(db.Schema) if err != nil { return errors.Trace(err) @@ -122,6 +125,9 @@ func runRestore(flagSet *flag.FlagSet, cmdName, dbName, tableName string) error case len(dbName) != 0 && len(tableName) != 0: // table restore db := client.GetDatabase(dbName) + if db == nil { + return errors.Errorf("database %s not found in backup", dbName) + } err = client.CreateDatabase(db.Schema) if err != nil { return errors.Trace(err) From 6b65080b967947c6d816a8a1320ee594b921fb54 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Wed, 5 Feb 2020 10:09:01 +0800 Subject: [PATCH 07/10] validate: fix debug meta test ci (#153) * validate: fix debug meta test ci --- tests/br_debug_meta/run.sh | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/br_debug_meta/run.sh b/tests/br_debug_meta/run.sh index 1dcfccefe..8dc3ef5a3 100644 --- a/tests/br_debug_meta/run.sh +++ b/tests/br_debug_meta/run.sh @@ -15,28 +15,33 @@ set -eu DB="$TEST_NAME" +TABLE="usertable1" run_sql "CREATE DATABASE $DB;" -run_sql "CREATE TABLE $DB.usertable1 ( \ +run_sql "CREATE TABLE $DB.$TABLE( \ YCSB_KEY varchar(64) NOT NULL, \ FIELD0 varchar(1) DEFAULT NULL, \ PRIMARY KEY (YCSB_KEY) \ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;" -run_sql "INSERT INTO $DB.usertable1 VALUES (\"a\", \"b\");" -run_sql "INSERT INTO $DB.usertable1 VALUES (\"aa\", \"b\");" +run_sql "INSERT INTO $DB.$TABLE VALUES (\"a\", \"b\");" +run_sql "INSERT INTO $DB.$TABLE VALUES (\"aa\", \"b\");" + +row_count_ori=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') # backup table echo "backup start..." run_br --pd $PD_ADDR backup table --db $DB --table usertable1 -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 +run_sql "DROP DATABASE $DB;" + # Test validate decode run_br validate decode -s "local://$TEST_DIR/$DB" # should generate backupmeta.json if [ ! -f "$TEST_DIR/$DB/backupmeta.json" ]; then - echo "TEST: [$TEST_NAME] failed!" + echo "TEST: [$TEST_NAME] decode failed!" exit 1 fi @@ -45,14 +50,21 @@ run_br validate encode -s "local://$TEST_DIR/$DB" # should generate backupmeta_from_json if [ ! -f "$TEST_DIR/$DB/backupmeta_from_json" ]; then - echo "TEST: [$TEST_NAME] failed!" + echo "TEST: [$TEST_NAME] encode failed!" exit 1 fi -DIFF=$(diff $TEST_DIR/$DB/backupmeta_from_json $TEST_DIR/$DB/backupmeta) -if [ "$DIFF" != "" ] -then - echo "TEST: [$TEST_NAME] failed!" +# replace backupmeta +mv "$TEST_DIR/$DB/backupmeta_from_json" "$TEST_DIR/$DB/backupmeta" + +# restore table +echo "restore start..." +run_br --pd $PD_ADDR restore table --db $DB --table usertable1 -s "local://$TEST_DIR/$DB" + +row_count_new=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}') + +if [ "${row_count_ori}" != "${row_count_new}" ];then + echo "TEST: [$TEST_NAME] failed!, row count not equal after restore" exit 1 fi From 0dfe82d7cacc8553f8744b6b54852b9466852542 Mon Sep 17 00:00:00 2001 From: kennytm Date: Mon, 10 Feb 2020 14:36:31 +0800 Subject: [PATCH 08/10] *: extracts runBackup/runRestore in cmd into pkg/task (#156) * *: extracts runBackup/runRestore in cmd into pkg/task Defines a "Config" structure to store the parsed flags. Use the "black-white-list" structure to define what tables/databases to backup/restore. * go.mod: update tidb to v4.0.0-beta --- cmd/backup.go | 203 ++---------------------- cmd/cmd.go | 62 +------- cmd/restore.go | 322 ++------------------------------------ cmd/validate.go | 75 +++------ go.mod | 19 +-- go.sum | 96 ++++++++---- pkg/backup/client.go | 87 +++------- pkg/backup/client_test.go | 20 +-- pkg/backup/schema_test.go | 34 ++-- pkg/restore/db_test.go | 2 +- pkg/restore/util.go | 7 +- pkg/storage/flags.go | 46 +----- pkg/storage/gcs.go | 28 ++-- pkg/storage/gcs_test.go | 17 +- pkg/storage/s3.go | 29 ++-- pkg/storage/s3_test.go | 4 +- pkg/storage/storage.go | 6 +- pkg/task/backup.go | 157 +++++++++++++++++++ pkg/task/common.go | 236 ++++++++++++++++++++++++++++ pkg/task/restore.go | 254 ++++++++++++++++++++++++++++++ 20 files changed, 871 insertions(+), 833 deletions(-) create mode 100644 pkg/task/backup.go create mode 100644 pkg/task/common.go create mode 100644 pkg/task/restore.go diff --git a/cmd/backup.go b/cmd/backup.go index 73ae6106f..39aa4fd28 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -1,176 +1,21 @@ package cmd import ( - "context" - - "github.com/pingcap/errors" - "github.com/pingcap/log" "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/session" "github.com/spf13/cobra" - "github.com/spf13/pflag" - "github.com/pingcap/br/pkg/backup" - "github.com/pingcap/br/pkg/storage" "github.com/pingcap/br/pkg/summary" + "github.com/pingcap/br/pkg/task" "github.com/pingcap/br/pkg/utils" ) -const ( - flagBackupTimeago = "timeago" - flagBackupRateLimit = "ratelimit" - flagBackupRateLimitUnit = "ratelimit-unit" - flagBackupConcurrency = "concurrency" - flagBackupChecksum = "checksum" - flagLastBackupTS = "lastbackupts" -) - -func defineBackupFlags(flagSet *pflag.FlagSet) { - flagSet.StringP( - flagBackupTimeago, "", "", - "The history version of the backup task, e.g. 1m, 1h. Do not exceed GCSafePoint") - flagSet.Uint64P( - flagBackupRateLimit, "", 0, "The rate limit of the backup task, MB/s per node") - flagSet.Uint32P( - flagBackupConcurrency, "", 4, "The size of thread pool on each node that execute the backup task") - flagSet.BoolP(flagBackupChecksum, "", true, - "Run checksum after backup") - flagSet.Uint64P(flagLastBackupTS, "", 0, "the last time backup ts") - _ = flagSet.MarkHidden(flagLastBackupTS) - - // Test only flag. - flagSet.Uint64P( - flagBackupRateLimitUnit, "", utils.MB, "The unit of rate limit of the backup task") - _ = flagSet.MarkHidden(flagBackupRateLimitUnit) -} - -func runBackup(flagSet *pflag.FlagSet, cmdName, db, table string) error { - ctx, cancel := context.WithCancel(defaultContext) - defer cancel() - - mgr, err := GetDefaultMgr() - if err != nil { - return err - } - defer mgr.Close() - - timeago, err := flagSet.GetString(flagBackupTimeago) - if err != nil { - return err - } - - ratelimit, err := flagSet.GetUint64(flagBackupRateLimit) - if err != nil { - return err - } - ratelimitUnit, err := flagSet.GetUint64(flagBackupRateLimitUnit) - if err != nil { - return err - } - ratelimit *= ratelimitUnit - - concurrency, err := flagSet.GetUint32(flagBackupConcurrency) - if err != nil { - return err - } - if concurrency == 0 { - err = errors.New("at least one thread required") +func runBackupCommand(command *cobra.Command, cmdName string) error { + cfg := task.BackupConfig{Config: task.Config{LogProgress: HasLogFile()}} + if err := cfg.ParseFromFlags(command.Flags()); err != nil { return err } - - checksum, err := flagSet.GetBool(flagBackupChecksum) - if err != nil { - return err - } - - lastBackupTS, err := flagSet.GetUint64(flagLastBackupTS) - if err != nil { - return nil - } - - u, err := storage.ParseBackendFromFlags(flagSet, FlagStorage) - if err != nil { - return err - } - - client, err := backup.NewBackupClient(ctx, mgr) - if err != nil { - return nil - } - - err = client.SetStorage(ctx, u) - if err != nil { - return err - } - - backupTS, err := client.GetTS(ctx, timeago) - if err != nil { - return err - } - - defer summary.Summary(cmdName) - - ranges, backupSchemas, err := backup.BuildBackupRangeAndSchema( - mgr.GetDomain(), mgr.GetTiKV(), backupTS, db, table) - if err != nil { - return err - } - - // The number of regions need to backup - approximateRegions := 0 - for _, r := range ranges { - var regionCount int - regionCount, err = mgr.GetRegionCount(ctx, r.StartKey, r.EndKey) - if err != nil { - return err - } - approximateRegions += regionCount - } - - summary.CollectInt("backup total regions", approximateRegions) - // Backup - // Redirect to log if there is no log file to avoid unreadable output. - updateCh := utils.StartProgress( - ctx, cmdName, int64(approximateRegions), !HasLogFile()) - err = client.BackupRanges( - ctx, ranges, lastBackupTS, backupTS, ratelimit, concurrency, updateCh) - if err != nil { - return err - } - // Backup has finished - close(updateCh) - - // Checksum - backupSchemasConcurrency := backup.DefaultSchemaConcurrency - if backupSchemas.Len() < backupSchemasConcurrency { - backupSchemasConcurrency = backupSchemas.Len() - } - updateCh = utils.StartProgress( - ctx, "Checksum", int64(backupSchemas.Len()), !HasLogFile()) - backupSchemas.SetSkipChecksum(!checksum) - backupSchemas.Start( - ctx, mgr.GetTiKV(), backupTS, uint(backupSchemasConcurrency), updateCh) - - err = client.CompleteMeta(backupSchemas) - if err != nil { - return err - } - - valid, err := client.FastChecksum() - if err != nil { - return err - } - if !valid { - log.Error("backup FastChecksum failed!") - } - // Checksum has finished - close(updateCh) - - err = client.SaveBackupMeta(ctx) - if err != nil { - return err - } - return nil + return task.RunBackup(GetDefaultContext(), cmdName, &cfg) } // NewBackupCommand return a full backup subcommand. @@ -200,7 +45,7 @@ func NewBackupCommand() *cobra.Command { newTableBackupCommand(), ) - defineBackupFlags(command.PersistentFlags()) + task.DefineBackupFlags(command.PersistentFlags()) return command } @@ -211,7 +56,7 @@ func newFullBackupCommand() *cobra.Command { Short: "backup all database", RunE: func(command *cobra.Command, _ []string) error { // empty db/table means full backup. - return runBackup(command.Flags(), "Full backup", "", "") + return runBackupCommand(command, "Full backup") }, } return command @@ -223,19 +68,10 @@ func newDbBackupCommand() *cobra.Command { Use: "db", Short: "backup a database", RunE: func(command *cobra.Command, _ []string) error { - db, err := command.Flags().GetString(flagDatabase) - if err != nil { - return err - } - if len(db) == 0 { - return errors.Errorf("empty database name is not allowed") - } - return runBackup(command.Flags(), "Database backup", db, "") + return runBackupCommand(command, "Database backup") }, } - command.Flags().StringP(flagDatabase, "", "", "backup a table in the specific db") - _ = command.MarkFlagRequired(flagDatabase) - + task.DefineDatabaseFlags(command) return command } @@ -245,26 +81,9 @@ func newTableBackupCommand() *cobra.Command { Use: "table", Short: "backup a table", RunE: func(command *cobra.Command, _ []string) error { - db, err := command.Flags().GetString(flagDatabase) - if err != nil { - return err - } - if len(db) == 0 { - return errors.Errorf("empty database name is not allowed") - } - table, err := command.Flags().GetString(flagTable) - if err != nil { - return err - } - if len(table) == 0 { - return errors.Errorf("empty table name is not allowed") - } - return runBackup(command.Flags(), "Table backup", db, table) + return runBackupCommand(command, "Table backup") }, } - command.Flags().StringP(flagDatabase, "", "", "backup a table in the specific db") - command.Flags().StringP(flagTable, "t", "", "backup the specific table") - _ = command.MarkFlagRequired(flagDatabase) - _ = command.MarkFlagRequired(flagTable) + task.DefineTableFlags(command) return command } diff --git a/cmd/cmd.go b/cmd/cmd.go index 468c35232..fdadaa6f8 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -2,47 +2,28 @@ package cmd import ( "context" - "fmt" "net/http" "net/http/pprof" "sync" "sync/atomic" - "github.com/pingcap/errors" "github.com/pingcap/log" - "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/util/logutil" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "go.uber.org/zap" - "github.com/pingcap/br/pkg/conn" - "github.com/pingcap/br/pkg/storage" + "github.com/pingcap/br/pkg/task" "github.com/pingcap/br/pkg/utils" ) var ( initOnce = sync.Once{} defaultContext context.Context - pdAddress string hasLogFile uint64 - - connOnce = sync.Once{} - defaultMgr *conn.Mgr ) const ( - // FlagPD is the name of url flag. - FlagPD = "pd" - // FlagCA is the name of CA flag. - FlagCA = "ca" - // FlagCert is the name of cert flag. - FlagCert = "cert" - // FlagKey is the name of key flag. - FlagKey = "key" - // FlagStorage is the name of storage flag. - FlagStorage = "storage" // FlagLogLevel is the name of log-level flag. FlagLogLevel = "log-level" // FlagLogFile is the name of log-file flag. @@ -52,9 +33,6 @@ const ( // FlagSlowLogFile is the name of slow-log-file flag. FlagSlowLogFile = "slow-log-file" - flagDatabase = "db" - flagTable = "table" - flagVersion = "version" flagVersionShort = "V" ) @@ -65,19 +43,13 @@ func AddFlags(cmd *cobra.Command) { cmd.Flags().BoolP(flagVersion, flagVersionShort, false, "Display version information about BR") cmd.SetVersionTemplate("{{printf \"%s\" .Version}}\n") - cmd.PersistentFlags().StringP(FlagPD, "u", "127.0.0.1:2379", "PD address") - cmd.PersistentFlags().String(FlagCA, "", "CA certificate path for TLS connection") - cmd.PersistentFlags().String(FlagCert, "", "Certificate path for TLS connection") - cmd.PersistentFlags().String(FlagKey, "", "Private key path for TLS connection") - cmd.PersistentFlags().StringP(FlagStorage, "s", "", - `specify the url where backup storage, eg, "local:///path/to/save"`) cmd.PersistentFlags().StringP(FlagLogLevel, "L", "info", "Set the log level") cmd.PersistentFlags().String(FlagLogFile, "", "Set the log file path. If not set, logs will output to stdout") cmd.PersistentFlags().String(FlagStatusAddr, "", "Set the HTTP listening address for the status report service. Set to empty string to disable") - storage.DefineFlags(cmd.PersistentFlags()) + task.DefineCommonFlags(cmd.PersistentFlags()) cmd.PersistentFlags().StringP(FlagSlowLogFile, "", "", "Set the slow log file path. If not set, discard slow logs") @@ -140,12 +112,6 @@ func Init(cmd *cobra.Command) (err error) { } } }() - // Set the PD server address. - pdAddress, e = cmd.Flags().GetString(FlagPD) - if e != nil { - err = e - return - } }) return err } @@ -155,30 +121,6 @@ func HasLogFile() bool { return atomic.LoadUint64(&hasLogFile) != uint64(0) } -// GetDefaultMgr returns the default mgr for command line usage. -func GetDefaultMgr() (*conn.Mgr, error) { - if pdAddress == "" { - return nil, errors.New("pd address can not be empty") - } - - // Lazy initialize and defaultMgr - var err error - connOnce.Do(func() { - var storage kv.Storage - storage, err = tikv.Driver{}.Open( - // Disable GC because TiDB enables GC already. - fmt.Sprintf("tikv://%s?disableGC=true", pdAddress)) - if err != nil { - return - } - defaultMgr, err = conn.NewMgr(defaultContext, pdAddress, storage.(tikv.Storage)) - }) - if err != nil { - return nil, err - } - return defaultMgr, nil -} - // SetDefaultContext sets the default context for command line usage. func SetDefaultContext(ctx context.Context) { defaultContext = ctx diff --git a/cmd/restore.go b/cmd/restore.go index 4f66e47de..2dfec9846 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -1,33 +1,20 @@ package cmd import ( - "context" - "strings" - - "github.com/gogo/protobuf/proto" - "github.com/pingcap/errors" - "github.com/pingcap/kvproto/pkg/backup" - "github.com/pingcap/log" "github.com/pingcap/tidb/session" "github.com/spf13/cobra" - flag "github.com/spf13/pflag" - "go.uber.org/zap" - "github.com/pingcap/br/pkg/conn" - "github.com/pingcap/br/pkg/restore" - "github.com/pingcap/br/pkg/storage" "github.com/pingcap/br/pkg/summary" + "github.com/pingcap/br/pkg/task" "github.com/pingcap/br/pkg/utils" ) -var schedulers = map[string]struct{}{ - "balance-leader-scheduler": {}, - "balance-hot-region-scheduler": {}, - "balance-region-scheduler": {}, - - "shuffle-leader-scheduler": {}, - "shuffle-region-scheduler": {}, - "shuffle-hot-region-scheduler": {}, +func runRestoreCommand(command *cobra.Command, cmdName string) error { + cfg := task.RestoreConfig{Config: task.Config{LogProgress: HasLogFile()}} + if err := cfg.ParseFromFlags(command.Flags()); err != nil { + return err + } + return task.RunRestore(GetDefaultContext(), cmdName, &cfg) } // NewRestoreCommand returns a restore subcommand @@ -54,172 +41,17 @@ func NewRestoreCommand() *cobra.Command { newDbRestoreCommand(), newTableRestoreCommand(), ) - - command.PersistentFlags().Uint("concurrency", 128, - "The size of thread pool that execute the restore task") - command.PersistentFlags().Uint64("ratelimit", 0, - "The rate limit of the restore task, MB/s per node. Set to 0 for unlimited speed.") - command.PersistentFlags().BoolP("checksum", "", true, - "Run checksum after restore") - command.PersistentFlags().BoolP("online", "", false, - "Whether online when restore") - // TODO remove hidden flag if it's stable - _ = command.PersistentFlags().MarkHidden("online") + task.DefineRestoreFlags(command.PersistentFlags()) return command } -func runRestore(flagSet *flag.FlagSet, cmdName, dbName, tableName string) error { - ctx, cancel := context.WithCancel(GetDefaultContext()) - defer cancel() - - mgr, err := GetDefaultMgr() - if err != nil { - return err - } - defer mgr.Close() - - client, err := restore.NewRestoreClient( - ctx, mgr.GetPDClient(), mgr.GetTiKV()) - if err != nil { - return errors.Trace(err) - } - defer client.Close() - err = initRestoreClient(ctx, client, flagSet) - if err != nil { - return errors.Trace(err) - } - - files := make([]*backup.File, 0) - tables := make([]*utils.Table, 0) - - defer summary.Summary(cmdName) - - switch { - case len(dbName) == 0 && len(tableName) == 0: - // full restore - for _, db := range client.GetDatabases() { - err = client.CreateDatabase(db.Schema) - if err != nil { - return errors.Trace(err) - } - for _, table := range db.Tables { - files = append(files, table.Files...) - } - tables = append(tables, db.Tables...) - } - case len(dbName) != 0 && len(tableName) == 0: - // database restore - db := client.GetDatabase(dbName) - if db == nil { - return errors.Errorf("database %s not found in backup", dbName) - } - err = client.CreateDatabase(db.Schema) - if err != nil { - return errors.Trace(err) - } - for _, table := range db.Tables { - files = append(files, table.Files...) - } - tables = db.Tables - case len(dbName) != 0 && len(tableName) != 0: - // table restore - db := client.GetDatabase(dbName) - if db == nil { - return errors.Errorf("database %s not found in backup", dbName) - } - err = client.CreateDatabase(db.Schema) - if err != nil { - return errors.Trace(err) - } - table := db.GetTable(tableName) - files = table.Files - tables = append(tables, table) - default: - return errors.New("must set db when table was set") - } - var newTS uint64 - if client.IsIncremental() { - newTS, err = client.GetTS(ctx) - if err != nil { - return err - } - } - summary.CollectInt("restore files", len(files)) - rewriteRules, newTables, err := client.CreateTables(mgr.GetDomain(), tables, newTS) - if err != nil { - return errors.Trace(err) - } - ranges, err := restore.ValidateFileRanges(files, rewriteRules) - if err != nil { - return err - } - summary.CollectInt("restore ranges", len(ranges)) - - // Redirect to log if there is no log file to avoid unreadable output. - updateCh := utils.StartProgress( - ctx, - cmdName, - // Split/Scatter + Download/Ingest - int64(len(ranges)+len(files)), - !HasLogFile()) - - err = restore.SplitRanges(ctx, client, ranges, rewriteRules, updateCh) - if err != nil { - log.Error("split regions failed", zap.Error(err)) - return errors.Trace(err) - } - - if !client.IsIncremental() { - var pdAddr string - pdAddr, err = flagSet.GetString(FlagPD) - if err != nil { - return errors.Trace(err) - } - pdAddrs := strings.Split(pdAddr, ",") - err = client.ResetTS(pdAddrs) - if err != nil { - log.Error("reset pd TS failed", zap.Error(err)) - return errors.Trace(err) - } - } - - removedSchedulers, err := RestorePrepareWork(ctx, client, mgr) - if err != nil { - return errors.Trace(err) - } - - err = client.RestoreAll(rewriteRules, updateCh) - if err != nil { - return errors.Trace(err) - } - - err = RestorePostWork(ctx, client, mgr, removedSchedulers) - if err != nil { - return errors.Trace(err) - } - // Restore has finished. - close(updateCh) - - // Checksum - updateCh = utils.StartProgress( - ctx, "Checksum", int64(len(newTables)), !HasLogFile()) - err = client.ValidateChecksum( - ctx, mgr.GetTiKV().GetClient(), tables, newTables, updateCh) - if err != nil { - return err - } - close(updateCh) - - return nil -} - func newFullRestoreCommand() *cobra.Command { command := &cobra.Command{ Use: "full", Short: "restore all tables", RunE: func(cmd *cobra.Command, _ []string) error { - return runRestore(cmd.Flags(), "Full Restore", "", "") + return runRestoreCommand(cmd, "Full restore") }, } return command @@ -230,18 +62,10 @@ func newDbRestoreCommand() *cobra.Command { Use: "db", Short: "restore tables in a database", RunE: func(cmd *cobra.Command, _ []string) error { - db, err := cmd.Flags().GetString(flagDatabase) - if err != nil { - return err - } - if len(db) == 0 { - return errors.New("empty database name is not allowed") - } - return runRestore(cmd.Flags(), "Database Restore", db, "") + return runRestoreCommand(cmd, "Database restore") }, } - command.Flags().String(flagDatabase, "", "database name") - _ = command.MarkFlagRequired(flagDatabase) + task.DefineDatabaseFlags(command) return command } @@ -250,129 +74,9 @@ func newTableRestoreCommand() *cobra.Command { Use: "table", Short: "restore a table", RunE: func(cmd *cobra.Command, _ []string) error { - db, err := cmd.Flags().GetString(flagDatabase) - if err != nil { - return err - } - if len(db) == 0 { - return errors.New("empty database name is not allowed") - } - table, err := cmd.Flags().GetString(flagTable) - if err != nil { - return err - } - if len(table) == 0 { - return errors.New("empty table name is not allowed") - } - return runRestore(cmd.Flags(), "Table Restore", db, table) + return runRestoreCommand(cmd, "Table restore") }, } - - command.Flags().String(flagDatabase, "", "database name") - command.Flags().String(flagTable, "", "table name") - - _ = command.MarkFlagRequired(flagDatabase) - _ = command.MarkFlagRequired(flagTable) + task.DefineTableFlags(command) return command } - -func initRestoreClient(ctx context.Context, client *restore.Client, flagSet *flag.FlagSet) error { - u, err := storage.ParseBackendFromFlags(flagSet, FlagStorage) - if err != nil { - return err - } - rateLimit, err := flagSet.GetUint64("ratelimit") - if err != nil { - return err - } - client.SetRateLimit(rateLimit * utils.MB) - s, err := storage.Create(ctx, u) - if err != nil { - return errors.Trace(err) - } - metaData, err := s.Read(ctx, utils.MetaFile) - if err != nil { - return errors.Trace(err) - } - backupMeta := &backup.BackupMeta{} - err = proto.Unmarshal(metaData, backupMeta) - if err != nil { - return errors.Trace(err) - } - err = client.InitBackupMeta(backupMeta, u) - if err != nil { - return errors.Trace(err) - } - - concurrency, err := flagSet.GetUint("concurrency") - if err != nil { - return err - } - client.SetConcurrency(concurrency) - - isOnline, err := flagSet.GetBool("online") - if err != nil { - return err - } - if isOnline { - client.EnableOnline() - } - - return nil -} - -// RestorePrepareWork execute some prepare work before restore -func RestorePrepareWork(ctx context.Context, client *restore.Client, mgr *conn.Mgr) ([]string, error) { - if client.IsOnline() { - return nil, nil - } - err := client.SwitchToImportMode(ctx) - if err != nil { - return nil, errors.Trace(err) - } - existSchedulers, err := mgr.ListSchedulers(ctx) - if err != nil { - return nil, errors.Trace(err) - } - needRemoveSchedulers := make([]string, 0, len(existSchedulers)) - for _, s := range existSchedulers { - if _, ok := schedulers[s]; ok { - needRemoveSchedulers = append(needRemoveSchedulers, s) - } - } - return removePDLeaderScheduler(ctx, mgr, needRemoveSchedulers) -} - -func removePDLeaderScheduler(ctx context.Context, mgr *conn.Mgr, existSchedulers []string) ([]string, error) { - removedSchedulers := make([]string, 0, len(existSchedulers)) - for _, scheduler := range existSchedulers { - err := mgr.RemoveScheduler(ctx, scheduler) - if err != nil { - return nil, err - } - removedSchedulers = append(removedSchedulers, scheduler) - } - return removedSchedulers, nil -} - -// RestorePostWork execute some post work after restore -func RestorePostWork(ctx context.Context, client *restore.Client, mgr *conn.Mgr, removedSchedulers []string) error { - if client.IsOnline() { - return nil - } - err := client.SwitchToNormalMode(ctx) - if err != nil { - return errors.Trace(err) - } - return addPDLeaderScheduler(ctx, mgr, removedSchedulers) -} - -func addPDLeaderScheduler(ctx context.Context, mgr *conn.Mgr, removedSchedulers []string) error { - for _, scheduler := range removedSchedulers { - err := mgr.AddScheduler(ctx, scheduler) - if err != nil { - return err - } - } - return nil -} diff --git a/cmd/validate.go b/cmd/validate.go index 8ba72b372..559cb9983 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -19,7 +19,7 @@ import ( "go.uber.org/zap" "github.com/pingcap/br/pkg/restore" - "github.com/pingcap/br/pkg/storage" + "github.com/pingcap/br/pkg/task" "github.com/pingcap/br/pkg/utils" ) @@ -54,24 +54,14 @@ func newCheckSumCommand() *cobra.Command { ctx, cancel := context.WithCancel(GetDefaultContext()) defer cancel() - u, err := storage.ParseBackendFromFlags(cmd.Flags(), FlagStorage) - if err != nil { + var cfg task.Config + if err := cfg.ParseFromFlags(cmd.Flags()); err != nil { return err } - s, err := storage.Create(ctx, u) - if err != nil { - return errors.Trace(err) - } - - metaData, err := s.Read(ctx, utils.MetaFile) - if err != nil { - return errors.Trace(err) - } - backupMeta := &backup.BackupMeta{} - err = proto.Unmarshal(metaData, backupMeta) + _, s, backupMeta, err := task.ReadBackupMeta(ctx, &cfg) if err != nil { - return errors.Trace(err) + return err } dbs, err := utils.LoadBackupTables(backupMeta) @@ -152,24 +142,14 @@ func newBackupMetaCommand() *cobra.Command { if err != nil { return err } - u, err := storage.ParseBackendFromFlags(cmd.Flags(), FlagStorage) - if err != nil { - return err - } - s, err := storage.Create(ctx, u) - if err != nil { - log.Error("create storage failed", zap.Error(err)) - return errors.Trace(err) - } - data, err := s.Read(ctx, utils.MetaFile) - if err != nil { - log.Error("load backupmeta failed", zap.Error(err)) + + var cfg task.Config + if err = cfg.ParseFromFlags(cmd.Flags()); err != nil { return err } - backupMeta := &backup.BackupMeta{} - err = proto.Unmarshal(data, backupMeta) + _, _, backupMeta, err := task.ReadBackupMeta(ctx, &cfg) if err != nil { - log.Error("parse backupmeta failed", zap.Error(err)) + log.Error("read backupmeta failed", zap.Error(err)) return err } dbs, err := utils.LoadBackupTables(backupMeta) @@ -241,8 +221,7 @@ func newBackupMetaCommand() *cobra.Command { return nil }, } - command.Flags().String("path", "", "the path of backupmeta") - command.Flags().Uint64P("offset", "", 0, "the offset of table id alloctor") + command.Flags().Uint64("offset", 0, "the offset of table id alloctor") command.Hidden = true return command } @@ -254,24 +233,16 @@ func decodeBackupMetaCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithCancel(GetDefaultContext()) defer cancel() - u, err := storage.ParseBackendFromFlags(cmd.Flags(), FlagStorage) - if err != nil { - return errors.Trace(err) - } - s, err := storage.Create(ctx, u) - if err != nil { - return errors.Trace(err) + + var cfg task.Config + if err := cfg.ParseFromFlags(cmd.Flags()); err != nil { + return err } - metaData, err := s.Read(ctx, utils.MetaFile) + _, s, backupMeta, err := task.ReadBackupMeta(ctx, &cfg) if err != nil { - return errors.Trace(err) + return err } - backupMeta := &backup.BackupMeta{} - err = proto.Unmarshal(metaData, backupMeta) - if err != nil { - return errors.Trace(err) - } backupMetaJSON, err := json.Marshal(backupMeta) if err != nil { return errors.Trace(err) @@ -309,14 +280,16 @@ func encodeBackupMetaCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithCancel(GetDefaultContext()) defer cancel() - u, err := storage.ParseBackendFromFlags(cmd.Flags(), FlagStorage) - if err != nil { - return errors.Trace(err) + + var cfg task.Config + if err := cfg.ParseFromFlags(cmd.Flags()); err != nil { + return err } - s, err := storage.Create(ctx, u) + _, s, err := task.GetStorage(ctx, &cfg) if err != nil { - return errors.Trace(err) + return err } + metaData, err := s.Read(ctx, utils.MetaJSONFile) if err != nil { return errors.Trace(err) diff --git a/go.mod b/go.mod index 9951c2922..850750f09 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/cheggaaa/pb/v3 v3.0.1 github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect - github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 // indirect github.com/fsouza/fake-gcs-server v1.15.0 github.com/go-sql-driver/mysql v1.4.1 github.com/gogo/protobuf v1.3.1 @@ -17,25 +16,23 @@ require ( github.com/google/uuid v1.1.1 github.com/onsi/ginkgo v1.10.3 // indirect github.com/onsi/gomega v1.7.1 // indirect - github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 - github.com/pingcap/errors v0.11.4 - github.com/pingcap/kvproto v0.0.0-20191212110315-d6a9d626988c + github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12 + github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 + github.com/pingcap/kvproto v0.0.0-20200108025604-a4dc183d2af5 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 - github.com/pingcap/parser v0.0.0-20191210060830-bdf23a7ade01 - github.com/pingcap/pd v1.1.0-beta.0.20191212045800-234784c7a9c5 - github.com/pingcap/tidb v1.1.0-beta.0.20191213040028-9009da737834 - github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33 + github.com/pingcap/parser v0.0.0-20200109073933-a9496438d77d + github.com/pingcap/pd v1.1.0-beta.0.20191219054547-4d65bbefbc6d + github.com/pingcap/tidb v1.1.0-beta.0.20200110130413-8c3ee37c1938 + github.com/pingcap/tidb-tools v4.0.0-beta+incompatible + github.com/pingcap/tipb v0.0.0-20191227083941-3996eff010dc github.com/prometheus/client_golang v1.0.0 github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.3 go.opencensus.io v0.22.2 // indirect - go.uber.org/atomic v1.5.1 // indirect go.uber.org/zap v1.13.0 - golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect golang.org/x/net v0.0.0-20191011234655-491137f69257 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 - golang.org/x/tools v0.0.0-20191213032237-7093a17b0467 // indirect google.golang.org/api v0.14.0 google.golang.org/grpc v1.25.1 ) diff --git a/go.sum b/go.sum index 085e00355..d5e9c891d 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= @@ -36,6 +38,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cheggaaa/pb/v3 v3.0.1 h1:m0BngUk2LuSRYdx4fujDKNRXNDpbNCfptPfVT2m6OJY= github.com/cheggaaa/pb/v3 v3.0.1/go.mod h1:SqqeMF/pMOIu3xgGoxtPYhMNQP258xE4x/XRTYua+KU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -48,10 +52,12 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -60,18 +66,23 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cznic/golex v0.0.0-20181122101858-9c343928389c/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= +github.com/cznic/parser v0.0.0-20160622100904-31edd927e5b1/go.mod h1:2B43mz36vGZNZEwkWi8ayRSSUXLfjL8OkbzwW4NcPMM= github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 h1:LpMLYGyy67BoAFGda1NeOBQwqlv7nUXpm+rIVHGxZZ4= github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= +github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= +github.com/cznic/y v0.0.0-20170802143616-045f81c6662a/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.1 h1:cJwdnj42uV8Jg4+KLrYovLiCgIfz9wtWm6E6KA+1tLs= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA= -github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -122,6 +133,7 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -169,7 +181,9 @@ github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -191,6 +205,7 @@ github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSg github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -219,7 +234,9 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -237,9 +254,11 @@ github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3 github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -248,45 +267,48 @@ github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKw github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.3.0 h1:e5+lF2E4Y2WCIxBefVowBuB0iHrUH4HZ8q+6mGF7fJc= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0= github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= +github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12 h1:rfD9v3+ppLPzoQBgZev0qYCpegrwyFx/BUpkApEiKdY= +github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 h1:KH4f4Si9XK6/IW50HtoaiLIFHGkapOM6w83za47UYik= github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c h1:hvQd3aOLKLF7xvRV6DzvPkKY4QXzfVbjU1BhW0d9yL8= -github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= +github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 h1:58naV4XMEqm0hl9LcYo6cZoGBGiLtefMQMF/vo3XLgQ= +github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d h1:F8vp38kTAckN+v8Jlc98uMBvKIzr1a+UhnLyVYn8Q5Q= +github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d h1:rCmRK0lCRrHMUbS99BKFYhK9YxJDNw0xB033cQbYo0s= github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ= -github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= -github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= -github.com/pingcap/kvproto v0.0.0-20191030021250-51b332bcb20b/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20191121022655-4c654046831d/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20191202044712-32be31591b03 h1:IyJl+qesVPf3UfFFmKtX69y1K5KC8uXlot3U0QgH7V4= -github.com/pingcap/kvproto v0.0.0-20191202044712-32be31591b03/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20191212110315-d6a9d626988c h1:CwVCq7XA/NvTQ6X9ZAhZlvcEvseUsHiPFQf2mL3LVl4= -github.com/pingcap/kvproto v0.0.0-20191212110315-d6a9d626988c/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd h1:hWDol43WY5PGhsh3+8794bFHY1bPrmu6bTalpssCrGg= -github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= +github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= +github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= +github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20191213111810-93cb7c623c8b/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20200108025604-a4dc183d2af5 h1:RUxQExD5yubAjWGnw8kcxfO9abbiVHIE1rbuCyQCWDE= +github.com/pingcap/kvproto v0.0.0-20200108025604-a4dc183d2af5/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/parser v0.0.0-20191210060830-bdf23a7ade01 h1:q1rGnV/296//bArDP7cDWWaSrhaeEKZY+gIo+Jb0Gyk= -github.com/pingcap/parser v0.0.0-20191210060830-bdf23a7ade01/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= -github.com/pingcap/pd v1.1.0-beta.0.20191210055626-676ddd3fbd2d/go.mod h1:Z/VMtXHpkOP+MnYnk4TL5VHc3ZwO1qHwc89zDuf5n8Q= -github.com/pingcap/pd v1.1.0-beta.0.20191212045800-234784c7a9c5 h1:sbpL1uNynq4yjGh0Xxb8MMePaOOXb9fdml3kB1NMQu4= -github.com/pingcap/pd v1.1.0-beta.0.20191212045800-234784c7a9c5/go.mod h1:NJYtcyKOqSWTJXoMF9CDdQc1xymxyBuQ8QSH6jJWqgc= -github.com/pingcap/sysutil v0.0.0-20191126040022-986c5b3ed9a3 h1:HCNif3lukL83gNC2EBAoh2Qbz36+2p0bm0LjgnNfl1s= -github.com/pingcap/sysutil v0.0.0-20191126040022-986c5b3ed9a3/go.mod h1:Futrrmuw98pEsbEmoPsjw8aKLCmixwHEmT2rF+AsXGw= -github.com/pingcap/tidb v1.1.0-beta.0.20191213040028-9009da737834 h1:eNf7bDY39moIzzcs5+PhLLW0BM2D2yrzFbjW/X42y0s= -github.com/pingcap/tidb v1.1.0-beta.0.20191213040028-9009da737834/go.mod h1:VWx47QOXISBHHtZeWrDQlBOdbvth9TE9gei6QpoqJ4g= +github.com/pingcap/parser v0.0.0-20200109073933-a9496438d77d h1:4QwSJRxmBjTB9ssJNWg2f2bDm5rqnHCUUjMh4N1QOOY= +github.com/pingcap/parser v0.0.0-20200109073933-a9496438d77d/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= +github.com/pingcap/pd v1.1.0-beta.0.20191219054547-4d65bbefbc6d h1:Ui80aiLTyd0EZD56o2tjFRYpHfhazBjtBdKeR8UoTFY= +github.com/pingcap/pd v1.1.0-beta.0.20191219054547-4d65bbefbc6d/go.mod h1:CML+b1JVjN+VbDijaIcUSmuPgpDjXEY7UiOx5yDP8eE= +github.com/pingcap/sysutil v0.0.0-20191216090214-5f9620d22b3b h1:EEyo/SCRswLGuSk+7SB86Ak1p8bS6HL1Mi4Dhyuv6zg= +github.com/pingcap/sysutil v0.0.0-20191216090214-5f9620d22b3b/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= +github.com/pingcap/tidb v1.1.0-beta.0.20200110130413-8c3ee37c1938 h1:Jt9ENNiS1ZNC9jV2Pd3wdegXQYFq3U6z1xFlzZNMNC8= +github.com/pingcap/tidb v1.1.0-beta.0.20200110130413-8c3ee37c1938/go.mod h1:DlMN+GGqC/WpREnzcH8xgxbXnntjybLhT84AbUSvMVM= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33 h1:cTSaVv1hue17BCPqt+sURADTFSMpSD26ZuvKRyYIjJs= -github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tidb-tools v4.0.0-beta+incompatible h1:+XJdcVLCM8GDgXiMS6lFV59N3XPVOqtNHeWNLVrb2pg= +github.com/pingcap/tidb-tools v4.0.0-beta+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20191227083941-3996eff010dc h1:IOKsFObJ4GZwAgyuhdJKg3oKCzWcoBFfHhpq2TOn5H0= +github.com/pingcap/tipb v0.0.0-20191227083941-3996eff010dc/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -297,7 +319,6 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -310,14 +331,16 @@ github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7 h1:FUL3b97ZY2EPqg2NbXKuMHs5pXJB9hjj1fDHnF2vl28= -github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.19.10+incompatible h1:lA4Pi29JEVIQIgATSeftHSY0rMGI9CLrl2ZvDLiahto= github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 h1:SWV2fHctRpRrp49VXJ6UZja7gU9QLHwRpIPBN89SKEo= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca h1:3fECS8atRjByijiI8yYiuwLwQ2ZxXobW7ua/8GRB3pI= @@ -331,11 +354,14 @@ github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= @@ -348,7 +374,6 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/struCoder/pidusage v0.1.2/go.mod h1:pWBlW3YuSwRl6h7R5KbvA4N8oOqe9LjaKW5CwT1SPjI= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ= @@ -374,6 +399,7 @@ github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKn github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yookoala/realpath v1.0.0 h1:7OA9pj4FZd+oZDsyvXWQvjn5oBdcHRTV44PpdMSuImQ= github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= @@ -382,6 +408,7 @@ go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf/go.mod h1:KSGwdbiFchh5KIC9My2 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4gLeh/DGvBYVaUYHuiB93teOmwgc= go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -415,6 +442,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -442,6 +470,7 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -457,6 +486,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc= golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191011234655-491137f69257 h1:ry8e2D+cwaV6hk7lb3aRTjjZo24shrbK0e11QEOkTIg= golang.org/x/net v0.0.0-20191011234655-491137f69257/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -525,9 +555,10 @@ golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2 h1:EtTFh6h4SAKemS+CURDMTDIANuduG5zKEXShyy18bGA= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191213032237-7093a17b0467 h1:Jybbe55FT+YYZIJGWmJIA4ZGcglFuZOduakIW3+gHXY= -golang.org/x/tools v0.0.0-20191213032237-7093a17b0467/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -585,6 +616,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/backup/client.go b/pkg/backup/client.go index 5cba2d9bf..49e48638d 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -16,12 +16,14 @@ import ( "github.com/pingcap/log" "github.com/pingcap/parser/model" pd "github.com/pingcap/pd/client" + "github.com/pingcap/tidb-tools/pkg/filter" "github.com/pingcap/tidb/distsql" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/oracle" + "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/ranger" "go.uber.org/zap" @@ -67,21 +69,17 @@ func NewBackupClient(ctx context.Context, mgr ClientMgr) (*Client, error) { } // GetTS returns the latest timestamp. -func (bc *Client) GetTS(ctx context.Context, timeAgo string) (uint64, error) { +func (bc *Client) GetTS(ctx context.Context, duration time.Duration) (uint64, error) { p, l, err := bc.mgr.GetPDClient().GetTS(ctx) if err != nil { return 0, errors.Trace(err) } backupTS := oracle.ComposeTS(p, l) - if timeAgo != "" { - duration, err := time.ParseDuration(timeAgo) - if err != nil { - return 0, errors.Trace(err) - } - if duration <= 0 { - return 0, errors.New("negative timeago is not allowed") - } + switch { + case duration < 0: + return 0, errors.New("negative timeago is not allowed") + case duration > 0: log.Info("backup time ago", zap.Duration("timeago", duration)) backupTime := oracle.GetTimeFromTS(backupTS) @@ -102,9 +100,9 @@ func (bc *Client) GetTS(ctx context.Context, timeAgo string) (uint64, error) { } // SetStorage set ExternalStorage for client -func (bc *Client) SetStorage(ctx context.Context, backend *backup.StorageBackend) error { +func (bc *Client) SetStorage(ctx context.Context, backend *backup.StorageBackend, sendCreds bool) error { var err error - bc.storage, err = storage.Create(ctx, backend) + bc.storage, err = storage.Create(ctx, backend, sendCreds) if err != nil { return err } @@ -173,63 +171,27 @@ func appendRanges(tbl *model.TableInfo, tblID int64) ([]kv.KeyRange, error) { func BuildBackupRangeAndSchema( dom *domain.Domain, storage kv.Storage, + tableFilter *filter.Filter, backupTS uint64, - dbName, tableName string, ) ([]Range, *Schemas, error) { - SystemDatabases := [3]string{ - "information_schema", - "performance_schema", - "mysql", - } - info, err := dom.GetSnapshotInfoSchema(backupTS) if err != nil { return nil, nil, errors.Trace(err) } - var dbInfos []*model.DBInfo - var cTableName model.CIStr - switch { - case len(dbName) == 0 && len(tableName) != 0: - return nil, nil, errors.New("no database is not specified") - case len(dbName) != 0 && len(tableName) == 0: - // backup database - cDBName := model.NewCIStr(dbName) - dbInfo, exist := info.SchemaByName(cDBName) - if !exist { - return nil, nil, errors.Errorf("schema %s not found", dbName) - } - dbInfos = append(dbInfos, dbInfo) - case len(dbName) != 0 && len(tableName) != 0: - // backup table - cTableName = model.NewCIStr(tableName) - cDBName := model.NewCIStr(dbName) - dbInfo, exist := info.SchemaByName(cDBName) - if !exist { - return nil, nil, errors.Errorf("schema %s not found", dbName) - } - dbInfos = append(dbInfos, dbInfo) - case len(dbName) == 0 && len(tableName) == 0: - // backup full - dbInfos = info.AllSchemas() - } ranges := make([]Range, 0) backupSchemas := newBackupSchemas() -LoadDb: - for _, dbInfo := range dbInfos { + for _, dbInfo := range info.AllSchemas() { // skip system databases - for _, sysDbName := range SystemDatabases { - if sysDbName == dbInfo.Name.L { - continue LoadDb - } - } - dbData, err := json.Marshal(dbInfo) - if err != nil { - return nil, nil, errors.Trace(err) + if util.IsMemOrSysDB(dbInfo.Name.L) { + continue } - idAlloc := autoid.NewAllocator(storage, dbInfo.ID, false) + + var dbData []byte + idAlloc := autoid.NewAllocator(storage, dbInfo.ID, false, autoid.RowIDAllocType) + for _, tableInfo := range dbInfo.Tables { - if len(cTableName.L) != 0 && cTableName.L != tableInfo.Name.L { + if !tableFilter.Match(&filter.Table{Schema: dbInfo.Name.L, Name: tableInfo.Name.L}) { // Skip tables other than the given table. continue } @@ -243,6 +205,12 @@ LoadDb: zap.Stringer("table", tableInfo.Name), zap.Int64("AutoIncID", globalAutoID)) + if dbData == nil { + dbData, err = json.Marshal(dbInfo) + if err != nil { + return nil, nil, errors.Trace(err) + } + } tableData, err := json.Marshal(tableInfo) if err != nil { return nil, nil, errors.Trace(err) @@ -267,11 +235,8 @@ LoadDb: } } - if len(cTableName.L) != 0 { - // Must find the given table. - if backupSchemas.Len() == 0 { - return nil, nil, errors.Errorf("table %s not found", cTableName) - } + if backupSchemas.Len() == 0 { + return nil, nil, errors.New("nothing to backup") } return ranges, backupSchemas, nil } diff --git a/pkg/backup/client_test.go b/pkg/backup/client_test.go index 44ca1ad5a..ddff45299 100644 --- a/pkg/backup/client_test.go +++ b/pkg/backup/client_test.go @@ -50,16 +50,10 @@ func (r *testBackup) TestGetTS(c *C) { deviation = 100 ) - // timeago not valid - timeAgo := "invalid" - _, err = r.backupClient.GetTS(r.ctx, timeAgo) - c.Assert(err, ErrorMatches, "time: invalid duration invalid") - // timeago not work - timeAgo = "" expectedDuration := 0 currentTs := time.Now().UnixNano() / int64(time.Millisecond) - ts, err := r.backupClient.GetTS(r.ctx, timeAgo) + ts, err := r.backupClient.GetTS(r.ctx, 0) c.Assert(err, IsNil) pdTs := oracle.ExtractPhysical(ts) duration := int(currentTs - pdTs) @@ -67,10 +61,9 @@ func (r *testBackup) TestGetTS(c *C) { c.Assert(duration, Less, expectedDuration+deviation) // timeago = "1.5m" - timeAgo = "1.5m" expectedDuration = 90000 currentTs = time.Now().UnixNano() / int64(time.Millisecond) - ts, err = r.backupClient.GetTS(r.ctx, timeAgo) + ts, err = r.backupClient.GetTS(r.ctx, 90*time.Second) c.Assert(err, IsNil) pdTs = oracle.ExtractPhysical(ts) duration = int(currentTs - pdTs) @@ -78,13 +71,11 @@ func (r *testBackup) TestGetTS(c *C) { c.Assert(duration, Less, expectedDuration+deviation) // timeago = "-1m" - timeAgo = "-1m" - _, err = r.backupClient.GetTS(r.ctx, timeAgo) + _, err = r.backupClient.GetTS(r.ctx, -time.Minute) c.Assert(err, ErrorMatches, "negative timeago is not allowed") // timeago = "1000000h" overflows - timeAgo = "1000000h" - _, err = r.backupClient.GetTS(r.ctx, timeAgo) + _, err = r.backupClient.GetTS(r.ctx, 1000000*time.Hour) c.Assert(err, ErrorMatches, "backup ts overflow.*") // timeago = "10h" exceed GCSafePoint @@ -93,8 +84,7 @@ func (r *testBackup) TestGetTS(c *C) { now := oracle.ComposeTS(p, l) _, err = r.backupClient.mgr.GetPDClient().UpdateGCSafePoint(r.ctx, now) c.Assert(err, IsNil) - timeAgo = "10h" - _, err = r.backupClient.GetTS(r.ctx, timeAgo) + _, err = r.backupClient.GetTS(r.ctx, 10*time.Hour) c.Assert(err, ErrorMatches, "GC safepoint [0-9]+ exceed TS [0-9]+") } diff --git a/pkg/backup/schema_test.go b/pkg/backup/schema_test.go index 3d10fd967..f657310bf 100644 --- a/pkg/backup/schema_test.go +++ b/pkg/backup/schema_test.go @@ -5,6 +5,7 @@ import ( "math" . "github.com/pingcap/check" + "github.com/pingcap/tidb-tools/pkg/filter" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" @@ -34,28 +35,32 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) { tk := testkit.NewTestKit(c, s.mock.Storage) // Table t1 is not exist. + testFilter, err := filter.New(false, &filter.Rules{ + DoTables: []*filter.Table{{Schema: "test", Name: "t1"}}, + }) + c.Assert(err, IsNil) _, backupSchemas, err := BuildBackupRangeAndSchema( - s.mock.Domain, s.mock.Storage, math.MaxUint64, "test", "t1") + s.mock.Domain, s.mock.Storage, testFilter, math.MaxUint64) c.Assert(err, NotNil) c.Assert(backupSchemas, IsNil) // Database is not exist. + fooFilter, err := filter.New(false, &filter.Rules{ + DoTables: []*filter.Table{{Schema: "foo", Name: "t1"}}, + }) + c.Assert(err, IsNil) _, backupSchemas, err = BuildBackupRangeAndSchema( - s.mock.Domain, s.mock.Storage, math.MaxUint64, "foo", "t1") + s.mock.Domain, s.mock.Storage, fooFilter, math.MaxUint64) c.Assert(err, NotNil) c.Assert(backupSchemas, IsNil) // Empty databse. - _, backupSchemas, err = BuildBackupRangeAndSchema( - s.mock.Domain, s.mock.Storage, math.MaxUint64, "", "") - c.Assert(err, IsNil) - c.Assert(backupSchemas, NotNil) - c.Assert(backupSchemas.Len(), Equals, 0) - updateCh := make(chan struct{}, 2) - backupSchemas.Start(context.Background(), s.mock.Storage, math.MaxUint64, 1, updateCh) - schemas, err := backupSchemas.finishTableChecksum() + noFilter, err := filter.New(false, &filter.Rules{}) c.Assert(err, IsNil) - c.Assert(len(schemas), Equals, 0) + _, backupSchemas, err = BuildBackupRangeAndSchema( + s.mock.Domain, s.mock.Storage, noFilter, math.MaxUint64) + c.Assert(err, NotNil) + c.Assert(backupSchemas, IsNil) tk.MustExec("use test") tk.MustExec("drop table if exists t1;") @@ -63,11 +68,12 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) { tk.MustExec("insert into t1 values (10);") _, backupSchemas, err = BuildBackupRangeAndSchema( - s.mock.Domain, s.mock.Storage, math.MaxUint64, "test", "t1") + s.mock.Domain, s.mock.Storage, testFilter, math.MaxUint64) c.Assert(err, IsNil) c.Assert(backupSchemas.Len(), Equals, 1) + updateCh := make(chan struct{}, 2) backupSchemas.Start(context.Background(), s.mock.Storage, math.MaxUint64, 1, updateCh) - schemas, err = backupSchemas.finishTableChecksum() + schemas, err := backupSchemas.finishTableChecksum() <-updateCh c.Assert(err, IsNil) c.Assert(len(schemas), Equals, 1) @@ -82,7 +88,7 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) { tk.MustExec("insert into t2 values (11);") _, backupSchemas, err = BuildBackupRangeAndSchema( - s.mock.Domain, s.mock.Storage, math.MaxUint64, "", "") + s.mock.Domain, s.mock.Storage, noFilter, math.MaxUint64) c.Assert(err, IsNil) c.Assert(backupSchemas.Len(), Equals, 2) backupSchemas.Start(context.Background(), s.mock.Storage, math.MaxUint64, 2, updateCh) diff --git a/pkg/restore/db_test.go b/pkg/restore/db_test.go index 9583f7f8c..98341f510 100644 --- a/pkg/restore/db_test.go +++ b/pkg/restore/db_test.go @@ -64,7 +64,7 @@ func (s *testRestoreSchemaSuite) TestRestoreAutoIncID(c *C) { Db: dbInfo, } // Get the next AutoIncID - idAlloc := autoid.NewAllocator(s.mock.Storage, dbInfo.ID, false) + idAlloc := autoid.NewAllocator(s.mock.Storage, dbInfo.ID, false, autoid.RowIDAllocType) globalAutoID, err := idAlloc.NextGlobalAutoID(table.Schema.ID) c.Assert(err, IsNil, Commentf("Error allocate next auto id")) c.Assert(autoIncID, Equals, uint64(globalAutoID)) diff --git a/pkg/restore/util.go b/pkg/restore/util.go index a2e9e3e38..ea8629470 100644 --- a/pkg/restore/util.go +++ b/pkg/restore/util.go @@ -13,6 +13,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/codec" "go.uber.org/zap" @@ -50,7 +51,7 @@ func newIDAllocator(id int64) *idAllocator { return &idAllocator{id: id} } -func (alloc *idAllocator) Alloc(tableID int64, n uint64) (min int64, max int64, err error) { +func (alloc *idAllocator) Alloc(tableID int64, n uint64, increment, offset int64) (min int64, max int64, err error) { return alloc.id, alloc.id, nil } @@ -70,6 +71,10 @@ func (alloc *idAllocator) NextGlobalAutoID(tableID int64) (int64, error) { return alloc.id, nil } +func (alloc *idAllocator) GetType() autoid.AllocatorType { + return autoid.RowIDAllocType +} + // GetRewriteRules returns the rewrite rule of the new table and the old table. func GetRewriteRules( newTable *model.TableInfo, diff --git a/pkg/storage/flags.go b/pkg/storage/flags.go index 51fd98af1..2340467ba 100644 --- a/pkg/storage/flags.go +++ b/pkg/storage/flags.go @@ -1,55 +1,19 @@ package storage import ( - "github.com/pingcap/errors" - "github.com/pingcap/kvproto/pkg/backup" "github.com/spf13/pflag" ) -const ( - // flagSendCredentialOption specify whether to send credentials to tikv - flagSendCredentialOption = "send-credentials-to-tikv" -) - -var ( - sendCredential bool -) - // DefineFlags adds flags to the flag set corresponding to all backend options. func DefineFlags(flags *pflag.FlagSet) { - flags.BoolP(flagSendCredentialOption, "c", true, - "Whether send credentials to tikv") defineS3Flags(flags) defineGCSFlags(flags) } -// GetBackendOptionsFromFlags obtains the backend options from the flag set. -func GetBackendOptionsFromFlags(flags *pflag.FlagSet) (options BackendOptions, err error) { - sendCredential, err = flags.GetBool(flagSendCredentialOption) - if err != nil { - err = errors.Trace(err) - return - } - - if options.S3, err = getBackendOptionsFromS3Flags(flags); err != nil { - return - } - if options.GCS, err = getBackendOptionsFromGCSFlags(flags); err != nil { - return - } - return -} - -// ParseBackendFromFlags is a convenient function to consecutively call -// GetBackendOptionsFromFlags and ParseBackend. -func ParseBackendFromFlags(flags *pflag.FlagSet, storageFlag string) (*backup.StorageBackend, error) { - u, err := flags.GetString(storageFlag) - if err != nil { - return nil, errors.Trace(err) - } - opts, err := GetBackendOptionsFromFlags(flags) - if err != nil { - return nil, err +// ParseFromFlags obtains the backend options from the flag set. +func (options *BackendOptions) ParseFromFlags(flags *pflag.FlagSet) error { + if err := options.S3.parseFromFlags(flags); err != nil { + return err } - return ParseBackend(u, &opts) + return options.GCS.parseFromFlags(flags) } diff --git a/pkg/storage/gcs.go b/pkg/storage/gcs.go index a0df5b03e..2eb310c3a 100644 --- a/pkg/storage/gcs.go +++ b/pkg/storage/gcs.go @@ -70,31 +70,28 @@ https://console.cloud.google.com/apis/credentials.`) _ = flags.MarkHidden(gcsCredentialsFile) } -func getBackendOptionsFromGCSFlags(flags *pflag.FlagSet) (options GCSBackendOptions, err error) { +func (options *GCSBackendOptions) parseFromFlags(flags *pflag.FlagSet) error { + var err error options.Endpoint, err = flags.GetString(gcsEndpointOption) if err != nil { - err = errors.Trace(err) - return + return errors.Trace(err) } options.StorageClass, err = flags.GetString(gcsStorageClassOption) if err != nil { - err = errors.Trace(err) - return + return errors.Trace(err) } options.PredefinedACL, err = flags.GetString(gcsPredefinedACL) if err != nil { - err = errors.Trace(err) - return + return errors.Trace(err) } options.CredentialsFile, err = flags.GetString(gcsCredentialsFile) if err != nil { - err = errors.Trace(err) - return + return errors.Trace(err) } - return + return nil } type gcsStorage struct { @@ -142,11 +139,16 @@ func (s *gcsStorage) FileExists(ctx context.Context, name string) (bool, error) return true, nil } -func newGCSStorage(ctx context.Context, gcs *backup.GCS) (*gcsStorage, error) { - return newGCSStorageWithHTTPClient(ctx, gcs, nil) +func newGCSStorage(ctx context.Context, gcs *backup.GCS, sendCredential bool) (*gcsStorage, error) { + return newGCSStorageWithHTTPClient(ctx, gcs, nil, sendCredential) } -func newGCSStorageWithHTTPClient(ctx context.Context, gcs *backup.GCS, hclient *http.Client) (*gcsStorage, error) { +func newGCSStorageWithHTTPClient( // revive:disable-line:flag-parameter + ctx context.Context, + gcs *backup.GCS, + hclient *http.Client, + sendCredential bool, +) (*gcsStorage, error) { var clientOps []option.ClientOption if gcs.CredentialsBlob == "" { creds, err := google.FindDefaultCredentials(ctx, storage.ScopeReadWrite) diff --git a/pkg/storage/gcs_test.go b/pkg/storage/gcs_test.go index da990cfe7..10bb44371 100644 --- a/pkg/storage/gcs_test.go +++ b/pkg/storage/gcs_test.go @@ -28,7 +28,7 @@ func (r *testStorageSuite) TestGCS(c *C) { PredefinedAcl: "private", CredentialsBlob: "Fake Credentials", } - stg, err := newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient()) + stg, err := newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), false) c.Assert(err, IsNil) err = stg.Write(ctx, "key", []byte("data")) @@ -66,7 +66,6 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) { server.CreateBucket(bucketName) { - sendCredential = true gcs := &backup.GCS{ Bucket: bucketName, Prefix: "a/b/", @@ -74,13 +73,12 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) { PredefinedAcl: "private", CredentialsBlob: "FakeCredentials", } - _, err := newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient()) + _, err := newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), true) c.Assert(err, IsNil) c.Assert(gcs.CredentialsBlob, Equals, "FakeCredentials") } { - sendCredential = false gcs := &backup.GCS{ Bucket: bucketName, Prefix: "a/b/", @@ -88,7 +86,7 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) { PredefinedAcl: "private", CredentialsBlob: "FakeCredentials", } - _, err := newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient()) + _, err := newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), false) c.Assert(err, IsNil) c.Assert(gcs.CredentialsBlob, Equals, "") } @@ -106,7 +104,6 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) { defer os.Unsetenv("GOOGLE_APPLICATION_CREDENTIALS") c.Assert(err, IsNil) - sendCredential = true gcs := &backup.GCS{ Bucket: bucketName, Prefix: "a/b/", @@ -114,7 +111,7 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) { PredefinedAcl: "private", CredentialsBlob: "", } - _, err = newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient()) + _, err = newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), true) c.Assert(err, IsNil) c.Assert(gcs.CredentialsBlob, Equals, `{"type": "service_account"}`) } @@ -132,7 +129,6 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) { defer os.Unsetenv("GOOGLE_APPLICATION_CREDENTIALS") c.Assert(err, IsNil) - sendCredential = false gcs := &backup.GCS{ Bucket: bucketName, Prefix: "a/b/", @@ -140,13 +136,12 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) { PredefinedAcl: "private", CredentialsBlob: "", } - _, err = newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient()) + _, err = newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), false) c.Assert(err, IsNil) c.Assert(gcs.CredentialsBlob, Equals, "") } { - sendCredential = true os.Unsetenv("GOOGLE_APPLICATION_CREDENTIALS") gcs := &backup.GCS{ Bucket: bucketName, @@ -155,7 +150,7 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) { PredefinedAcl: "private", CredentialsBlob: "", } - _, err = newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient()) + _, err = newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), true) c.Assert(err, NotNil) } } diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go index 5db54556c..8e04769b5 100644 --- a/pkg/storage/s3.go +++ b/pkg/storage/s3.go @@ -117,44 +117,41 @@ func defineS3Flags(flags *pflag.FlagSet) { _ = flags.MarkHidden(s3ProviderOption) } -func getBackendOptionsFromS3Flags(flags *pflag.FlagSet) (options S3BackendOptions, err error) { +func (options *S3BackendOptions) parseFromFlags(flags *pflag.FlagSet) error { + var err error options.Endpoint, err = flags.GetString(s3EndpointOption) if err != nil { - err = errors.Trace(err) - return + return errors.Trace(err) } options.Region, err = flags.GetString(s3RegionOption) if err != nil { - err = errors.Trace(err) - return + return errors.Trace(err) } options.SSE, err = flags.GetString(s3SSEOption) if err != nil { - err = errors.Trace(err) - return + return errors.Trace(err) } options.ACL, err = flags.GetString(s3ACLOption) if err != nil { - err = errors.Trace(err) - return + return errors.Trace(err) } options.StorageClass, err = flags.GetString(s3StorageClassOption) if err != nil { - err = errors.Trace(err) - return + return errors.Trace(err) } options.ForcePathStyle = true options.Provider, err = flags.GetString(s3ProviderOption) if err != nil { - err = errors.Trace(err) - return + return errors.Trace(err) } - - return options, err + return nil } // newS3Storage initialize a new s3 storage for metadata -func newS3Storage(backend *backup.S3) (*S3Storage, error) { +func newS3Storage( // revive:disable-line:flag-parameter + backend *backup.S3, + sendCredential bool, +) (*S3Storage, error) { qs := *backend awsConfig := aws.NewConfig(). WithMaxRetries(maxRetries). diff --git a/pkg/storage/s3_test.go b/pkg/storage/s3_test.go index 92a5a8737..3eaf1c206 100644 --- a/pkg/storage/s3_test.go +++ b/pkg/storage/s3_test.go @@ -236,7 +236,7 @@ func (r *testStorageSuite) TestS3Storage(c *C) { testFn := func(test *testcase, c *C) { c.Log(test.name) ctx := aws.BackgroundContext() - sendCredential = test.sendCredential + sendCredential := test.sendCredential if test.hackCheck { checkS3Bucket = func(svc *s3.S3, bucket string) error { return nil } } @@ -245,7 +245,7 @@ func (r *testStorageSuite) TestS3Storage(c *C) { S3: test.s3, }, } - _, err := Create(ctx, s3) + _, err := Create(ctx, s3, sendCredential) if test.errReturn { c.Assert(err, NotNil) return diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 173638bdd..f9ae368ae 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -18,7 +18,7 @@ type ExternalStorage interface { } // Create creates ExternalStorage -func Create(ctx context.Context, backend *backup.StorageBackend) (ExternalStorage, error) { +func Create(ctx context.Context, backend *backup.StorageBackend, sendCreds bool) (ExternalStorage, error) { switch backend := backend.Backend.(type) { case *backup.StorageBackend_Local: return newLocalStorage(backend.Local.Path) @@ -26,14 +26,14 @@ func Create(ctx context.Context, backend *backup.StorageBackend) (ExternalStorag if backend.S3 == nil { return nil, errors.New("s3 config not found") } - return newS3Storage(backend.S3) + return newS3Storage(backend.S3, sendCreds) case *backup.StorageBackend_Noop: return newNoopStorage(), nil case *backup.StorageBackend_Gcs: if backend.Gcs == nil { return nil, errors.New("GCS config not found") } - return newGCSStorage(ctx, backend.Gcs) + return newGCSStorage(ctx, backend.Gcs, sendCreds) default: return nil, errors.Errorf("storage %T is not supported yet", backend) } diff --git a/pkg/task/backup.go b/pkg/task/backup.go new file mode 100644 index 000000000..b9613cd56 --- /dev/null +++ b/pkg/task/backup.go @@ -0,0 +1,157 @@ +package task + +import ( + "context" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/spf13/pflag" + + "github.com/pingcap/br/pkg/backup" + "github.com/pingcap/br/pkg/storage" + "github.com/pingcap/br/pkg/summary" + "github.com/pingcap/br/pkg/utils" +) + +const ( + flagBackupTimeago = "timeago" + flagLastBackupTS = "lastbackupts" +) + +// BackupConfig is the configuration specific for backup tasks. +type BackupConfig struct { + Config + + TimeAgo time.Duration `json:"time-ago" toml:"time-ago"` + LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"` +} + +// DefineBackupFlags defines common flags for the backup command. +func DefineBackupFlags(flags *pflag.FlagSet) { + flags.Duration( + flagBackupTimeago, 0, + "The history version of the backup task, e.g. 1m, 1h. Do not exceed GCSafePoint") + + flags.Uint64(flagLastBackupTS, 0, "the last time backup ts") + _ = flags.MarkHidden(flagLastBackupTS) +} + +// ParseFromFlags parses the backup-related flags from the flag set. +func (cfg *BackupConfig) ParseFromFlags(flags *pflag.FlagSet) error { + timeAgo, err := flags.GetDuration(flagBackupTimeago) + if err != nil { + return errors.Trace(err) + } + if timeAgo < 0 { + return errors.New("negative timeago is not allowed") + } + cfg.TimeAgo = timeAgo + cfg.LastBackupTS, err = flags.GetUint64(flagLastBackupTS) + if err != nil { + return errors.Trace(err) + } + if err = cfg.Config.ParseFromFlags(flags); err != nil { + return errors.Trace(err) + } + return nil +} + +// RunBackup starts a backup task inside the current goroutine. +func RunBackup(c context.Context, cmdName string, cfg *BackupConfig) error { + ctx, cancel := context.WithCancel(c) + defer cancel() + + u, err := storage.ParseBackend(cfg.Storage, &cfg.BackendOptions) + if err != nil { + return err + } + tableFilter, err := filter.New(cfg.CaseSensitive, &cfg.Filter) + if err != nil { + return err + } + mgr, err := newMgr(ctx, cfg.PD) + if err != nil { + return err + } + defer mgr.Close() + + client, err := backup.NewBackupClient(ctx, mgr) + if err != nil { + return err + } + if err = client.SetStorage(ctx, u, cfg.SendCreds); err != nil { + return err + } + + backupTS, err := client.GetTS(ctx, cfg.TimeAgo) + if err != nil { + return err + } + + defer summary.Summary(cmdName) + + ranges, backupSchemas, err := backup.BuildBackupRangeAndSchema( + mgr.GetDomain(), mgr.GetTiKV(), tableFilter, backupTS) + if err != nil { + return err + } + + // The number of regions need to backup + approximateRegions := 0 + for _, r := range ranges { + var regionCount int + regionCount, err = mgr.GetRegionCount(ctx, r.StartKey, r.EndKey) + if err != nil { + return err + } + approximateRegions += regionCount + } + + summary.CollectInt("backup total regions", approximateRegions) + + // Backup + // Redirect to log if there is no log file to avoid unreadable output. + updateCh := utils.StartProgress( + ctx, cmdName, int64(approximateRegions), !cfg.LogProgress) + err = client.BackupRanges( + ctx, ranges, cfg.LastBackupTS, backupTS, cfg.RateLimit, cfg.Concurrency, updateCh) + if err != nil { + return err + } + // Backup has finished + close(updateCh) + + // Checksum + backupSchemasConcurrency := backup.DefaultSchemaConcurrency + if backupSchemas.Len() < backupSchemasConcurrency { + backupSchemasConcurrency = backupSchemas.Len() + } + updateCh = utils.StartProgress( + ctx, "Checksum", int64(backupSchemas.Len()), !cfg.LogProgress) + backupSchemas.SetSkipChecksum(!cfg.Checksum) + backupSchemas.Start( + ctx, mgr.GetTiKV(), backupTS, uint(backupSchemasConcurrency), updateCh) + + err = client.CompleteMeta(backupSchemas) + if err != nil { + return err + } + + valid, err := client.FastChecksum() + if err != nil { + return err + } + if !valid { + log.Error("backup FastChecksum mismatch!") + } + // Checksum has finished + close(updateCh) + + err = client.SaveBackupMeta(ctx) + if err != nil { + return err + } + return nil +} diff --git a/pkg/task/common.go b/pkg/task/common.go new file mode 100644 index 000000000..2433d94b9 --- /dev/null +++ b/pkg/task/common.go @@ -0,0 +1,236 @@ +package task + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/gogo/protobuf/proto" + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/backup" + "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/store/tikv" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/pingcap/br/pkg/conn" + "github.com/pingcap/br/pkg/storage" + "github.com/pingcap/br/pkg/utils" +) + +const ( + // flagSendCreds specify whether to send credentials to tikv + flagSendCreds = "send-credentials-to-tikv" + // flagStorage is the name of storage flag. + flagStorage = "storage" + // flagPD is the name of PD url flag. + flagPD = "pd" + // flagCA is the name of TLS CA flag. + flagCA = "ca" + // flagCert is the name of TLS cert flag. + flagCert = "cert" + // flagKey is the name of TLS key flag. + flagKey = "key" + + flagDatabase = "db" + flagTable = "table" + + flagRateLimit = "ratelimit" + flagRateLimitUnit = "ratelimit-unit" + flagConcurrency = "concurrency" + flagChecksum = "checksum" +) + +// TLSConfig is the common configuration for TLS connection. +type TLSConfig struct { + CA string `json:"ca" toml:"ca"` + Cert string `json:"cert" toml:"cert"` + Key string `json:"key" toml:"key"` +} + +// Config is the common configuration for all BRIE tasks. +type Config struct { + storage.BackendOptions + + Storage string `json:"storage" toml:"storage"` + PD []string `json:"pd" toml:"pd"` + TLS TLSConfig `json:"tls" toml:"tls"` + RateLimit uint64 `json:"rate-limit" toml:"rate-limit"` + Concurrency uint32 `json:"concurrency" toml:"concurrency"` + Checksum bool `json:"checksum" toml:"checksum"` + SendCreds bool `json:"send-credentials-to-tikv" toml:"send-credentials-to-tikv"` + // LogProgress is true means the progress bar is printed to the log instead of stdout. + LogProgress bool `json:"log-progress" toml:"log-progress"` + + CaseSensitive bool `json:"case-sensitive" toml:"case-sensitive"` + Filter filter.Rules `json:"black-white-list" toml:"black-white-list"` +} + +// DefineCommonFlags defines the flags common to all BRIE commands. +func DefineCommonFlags(flags *pflag.FlagSet) { + flags.BoolP(flagSendCreds, "c", true, "Whether send credentials to tikv") + flags.StringP(flagStorage, "s", "", `specify the url where backup storage, eg, "local:///path/to/save"`) + flags.StringSliceP(flagPD, "u", []string{"127.0.0.1:2379"}, "PD address") + flags.String(flagCA, "", "CA certificate path for TLS connection") + flags.String(flagCert, "", "Certificate path for TLS connection") + flags.String(flagKey, "", "Private key path for TLS connection") + + flags.Uint64(flagRateLimit, 0, "The rate limit of the task, MB/s per node") + flags.Uint32(flagConcurrency, 4, "The size of thread pool on each node that executes the task") + flags.Bool(flagChecksum, true, "Run checksum at end of task") + + flags.Uint64(flagRateLimitUnit, utils.MB, "The unit of rate limit") + _ = flags.MarkHidden(flagRateLimitUnit) + + storage.DefineFlags(flags) +} + +// DefineDatabaseFlags defines the required --db flag. +func DefineDatabaseFlags(command *cobra.Command) { + command.Flags().String(flagDatabase, "", "database name") + _ = command.MarkFlagRequired(flagDatabase) +} + +// DefineTableFlags defines the required --db and --table flags. +func DefineTableFlags(command *cobra.Command) { + DefineDatabaseFlags(command) + command.Flags().StringP(flagTable, "t", "", "table name") + _ = command.MarkFlagRequired(flagTable) +} + +// ParseFromFlags parses the TLS config from the flag set. +func (tls *TLSConfig) ParseFromFlags(flags *pflag.FlagSet) error { + var err error + tls.CA, err = flags.GetString(flagCA) + if err != nil { + return errors.Trace(err) + } + tls.Cert, err = flags.GetString(flagCert) + if err != nil { + return errors.Trace(err) + } + tls.Key, err = flags.GetString(flagKey) + if err != nil { + return errors.Trace(err) + } + return nil +} + +// ParseFromFlags parses the config from the flag set. +func (cfg *Config) ParseFromFlags(flags *pflag.FlagSet) error { + var err error + cfg.Storage, err = flags.GetString(flagStorage) + if err != nil { + return errors.Trace(err) + } + cfg.SendCreds, err = flags.GetBool(flagSendCreds) + if err != nil { + return errors.Trace(err) + } + cfg.PD, err = flags.GetStringSlice(flagPD) + if err != nil { + return errors.Trace(err) + } + if len(cfg.PD) == 0 { + return errors.New("must provide at least one PD server address") + } + cfg.Concurrency, err = flags.GetUint32(flagConcurrency) + if err != nil { + return errors.Trace(err) + } + cfg.Checksum, err = flags.GetBool(flagChecksum) + if err != nil { + return errors.Trace(err) + } + + var rateLimit, rateLimitUnit uint64 + rateLimit, err = flags.GetUint64(flagRateLimit) + if err != nil { + return errors.Trace(err) + } + rateLimitUnit, err = flags.GetUint64(flagRateLimitUnit) + if err != nil { + return errors.Trace(err) + } + cfg.RateLimit = rateLimit * rateLimitUnit + + if dbFlag := flags.Lookup(flagDatabase); dbFlag != nil { + db := escapeFilterName(dbFlag.Value.String()) + if len(db) == 0 { + return errors.New("empty database name is not allowed") + } + if tblFlag := flags.Lookup(flagTable); tblFlag != nil { + tbl := escapeFilterName(tblFlag.Value.String()) + if len(tbl) == 0 { + return errors.New("empty table name is not allowed") + } + cfg.Filter.DoTables = []*filter.Table{{Schema: db, Name: tbl}} + } else { + cfg.Filter.DoDBs = []string{db} + } + } + + if err := cfg.BackendOptions.ParseFromFlags(flags); err != nil { + return err + } + return cfg.TLS.ParseFromFlags(flags) +} + +// newMgr creates a new mgr at the given PD address. +func newMgr(ctx context.Context, pds []string) (*conn.Mgr, error) { + pdAddress := strings.Join(pds, ",") + if len(pdAddress) == 0 { + return nil, errors.New("pd address can not be empty") + } + + // Disable GC because TiDB enables GC already. + store, err := tikv.Driver{}.Open(fmt.Sprintf("tikv://%s?disableGC=true", pdAddress)) + if err != nil { + return nil, err + } + return conn.NewMgr(ctx, pdAddress, store.(tikv.Storage)) +} + +// GetStorage gets the storage backend from the config. +func GetStorage( + ctx context.Context, + cfg *Config, +) (*backup.StorageBackend, storage.ExternalStorage, error) { + u, err := storage.ParseBackend(cfg.Storage, &cfg.BackendOptions) + if err != nil { + return nil, nil, err + } + s, err := storage.Create(ctx, u, cfg.SendCreds) + if err != nil { + return nil, nil, errors.Annotate(err, "create storage failed") + } + return u, s, nil +} + +// ReadBackupMeta reads the backupmeta file from the storage. +func ReadBackupMeta( + ctx context.Context, + cfg *Config, +) (*backup.StorageBackend, storage.ExternalStorage, *backup.BackupMeta, error) { + u, s, err := GetStorage(ctx, cfg) + if err != nil { + return nil, nil, nil, err + } + metaData, err := s.Read(ctx, utils.MetaFile) + if err != nil { + return nil, nil, nil, errors.Annotate(err, "load backupmeta failed") + } + backupMeta := &backup.BackupMeta{} + if err = proto.Unmarshal(metaData, backupMeta); err != nil { + return nil, nil, nil, errors.Annotate(err, "parse backupmeta failed") + } + return u, s, backupMeta, nil +} + +func escapeFilterName(name string) string { + if !strings.HasPrefix(name, "~") { + return name + } + return "~^" + regexp.QuoteMeta(name) + "$" +} diff --git a/pkg/task/restore.go b/pkg/task/restore.go new file mode 100644 index 000000000..f2f3caf43 --- /dev/null +++ b/pkg/task/restore.go @@ -0,0 +1,254 @@ +package task + +import ( + "context" + + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/backup" + "github.com/pingcap/log" + "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/spf13/pflag" + "go.uber.org/zap" + + "github.com/pingcap/br/pkg/conn" + "github.com/pingcap/br/pkg/restore" + "github.com/pingcap/br/pkg/summary" + "github.com/pingcap/br/pkg/utils" +) + +const ( + flagOnline = "online" +) + +var schedulers = map[string]struct{}{ + "balance-leader-scheduler": {}, + "balance-hot-region-scheduler": {}, + "balance-region-scheduler": {}, + + "shuffle-leader-scheduler": {}, + "shuffle-region-scheduler": {}, + "shuffle-hot-region-scheduler": {}, +} + +// RestoreConfig is the configuration specific for restore tasks. +type RestoreConfig struct { + Config + + Online bool `json:"online" toml:"online"` +} + +// DefineRestoreFlags defines common flags for the restore command. +func DefineRestoreFlags(flags *pflag.FlagSet) { + flags.Bool("online", false, "Whether online when restore") + // TODO remove hidden flag if it's stable + _ = flags.MarkHidden("online") +} + +// ParseFromFlags parses the restore-related flags from the flag set. +func (cfg *RestoreConfig) ParseFromFlags(flags *pflag.FlagSet) error { + var err error + cfg.Online, err = flags.GetBool(flagOnline) + if err != nil { + return errors.Trace(err) + } + return cfg.Config.ParseFromFlags(flags) +} + +// RunRestore starts a restore task inside the current goroutine. +func RunRestore(c context.Context, cmdName string, cfg *RestoreConfig) error { + ctx, cancel := context.WithCancel(c) + defer cancel() + + mgr, err := newMgr(ctx, cfg.PD) + if err != nil { + return err + } + defer mgr.Close() + + client, err := restore.NewRestoreClient(ctx, mgr.GetPDClient(), mgr.GetTiKV()) + if err != nil { + return err + } + defer client.Close() + + client.SetRateLimit(cfg.RateLimit) + client.SetConcurrency(uint(cfg.Concurrency)) + if cfg.Online { + client.EnableOnline() + } + + defer summary.Summary(cmdName) + + u, _, backupMeta, err := ReadBackupMeta(ctx, &cfg.Config) + if err != nil { + return err + } + if err = client.InitBackupMeta(backupMeta, u); err != nil { + return err + } + + files, tables, err := filterRestoreFiles(client, cfg) + if err != nil { + return err + } + if len(files) == 0 { + return errors.New("all files are filtered out from the backup archive, nothing to restore") + } + summary.CollectInt("restore files", len(files)) + + var newTS uint64 + if client.IsIncremental() { + newTS, err = client.GetTS(ctx) + if err != nil { + return err + } + } + rewriteRules, newTables, err := client.CreateTables(mgr.GetDomain(), tables, newTS) + if err != nil { + return err + } + + ranges, err := restore.ValidateFileRanges(files, rewriteRules) + if err != nil { + return err + } + summary.CollectInt("restore ranges", len(ranges)) + + // Redirect to log if there is no log file to avoid unreadable output. + updateCh := utils.StartProgress( + ctx, + cmdName, + // Split/Scatter + Download/Ingest + int64(len(ranges)+len(files)), + !cfg.LogProgress) + + err = restore.SplitRanges(ctx, client, ranges, rewriteRules, updateCh) + if err != nil { + log.Error("split regions failed", zap.Error(err)) + return err + } + + if !client.IsIncremental() { + if err = client.ResetTS(cfg.PD); err != nil { + log.Error("reset pd TS failed", zap.Error(err)) + return err + } + } + + removedSchedulers, err := restorePreWork(ctx, client, mgr) + if err != nil { + return err + } + err = client.RestoreAll(rewriteRules, updateCh) + // always run the post-work even on error, so we don't stuck in the import mode or emptied schedulers + postErr := restorePostWork(ctx, client, mgr, removedSchedulers) + + if err != nil { + return err + } + if postErr != nil { + return postErr + } + + // Restore has finished. + close(updateCh) + + // Checksum + updateCh = utils.StartProgress( + ctx, "Checksum", int64(len(newTables)), !cfg.LogProgress) + err = client.ValidateChecksum( + ctx, mgr.GetTiKV().GetClient(), tables, newTables, updateCh) + if err != nil { + return err + } + close(updateCh) + + return nil +} + +func filterRestoreFiles( + client *restore.Client, + cfg *RestoreConfig, +) (files []*backup.File, tables []*utils.Table, err error) { + tableFilter, err := filter.New(cfg.CaseSensitive, &cfg.Filter) + if err != nil { + return nil, nil, err + } + + for _, db := range client.GetDatabases() { + createdDatabase := false + for _, table := range db.Tables { + if !tableFilter.Match(&filter.Table{Schema: db.Schema.Name.O, Name: table.Schema.Name.O}) { + continue + } + + if !createdDatabase { + if err = client.CreateDatabase(db.Schema); err != nil { + return nil, nil, err + } + createdDatabase = true + } + + files = append(files, table.Files...) + tables = append(tables, table) + } + } + + return +} + +// restorePreWork executes some prepare work before restore +func restorePreWork(ctx context.Context, client *restore.Client, mgr *conn.Mgr) ([]string, error) { + if client.IsOnline() { + return nil, nil + } + + if err := client.SwitchToImportMode(ctx); err != nil { + return nil, err + } + + existSchedulers, err := mgr.ListSchedulers(ctx) + if err != nil { + return nil, errors.Trace(err) + } + needRemoveSchedulers := make([]string, 0, len(existSchedulers)) + for _, s := range existSchedulers { + if _, ok := schedulers[s]; ok { + needRemoveSchedulers = append(needRemoveSchedulers, s) + } + } + return removePDLeaderScheduler(ctx, mgr, needRemoveSchedulers) +} + +func removePDLeaderScheduler(ctx context.Context, mgr *conn.Mgr, existSchedulers []string) ([]string, error) { + removedSchedulers := make([]string, 0, len(existSchedulers)) + for _, scheduler := range existSchedulers { + err := mgr.RemoveScheduler(ctx, scheduler) + if err != nil { + return nil, err + } + removedSchedulers = append(removedSchedulers, scheduler) + } + return removedSchedulers, nil +} + +// restorePostWork executes some post work after restore +func restorePostWork(ctx context.Context, client *restore.Client, mgr *conn.Mgr, removedSchedulers []string) error { + if client.IsOnline() { + return nil + } + if err := client.SwitchToNormalMode(ctx); err != nil { + return err + } + return addPDLeaderScheduler(ctx, mgr, removedSchedulers) +} + +func addPDLeaderScheduler(ctx context.Context, mgr *conn.Mgr, removedSchedulers []string) error { + for _, scheduler := range removedSchedulers { + err := mgr.AddScheduler(ctx, scheduler) + if err != nil { + return err + } + } + return nil +} From 8c97452f3c27c641f4e0be5b980cbe491740056b Mon Sep 17 00:00:00 2001 From: 3pointer Date: Thu, 13 Feb 2020 10:24:23 +0800 Subject: [PATCH 09/10] restore: fix restore summary log (#150) Co-authored-by: kennytm --- pkg/restore/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/restore/client.go b/pkg/restore/client.go index 3030ba857..5e8df4418 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -242,8 +242,6 @@ func (rc *Client) RestoreTable( key := fmt.Sprintf("%s.%s", table.Db.Name.String(), table.Schema.Name.String()) if err != nil { summary.CollectFailureUnit(key, err) - } else { - summary.CollectSuccessUnit(key, elapsed) } }() @@ -342,6 +340,7 @@ func (rc *Client) RestoreAll( defer func() { elapsed := time.Since(start) log.Info("Restore All", zap.Duration("take", elapsed)) + summary.CollectSuccessUnit("restore all", elapsed) }() errCh := make(chan error, len(rc.databases)) wg := new(sync.WaitGroup) From 008ec45182220d6d7cc7e8b13f940bfea1e24fd6 Mon Sep 17 00:00:00 2001 From: 5kbpers <20279863+5kbpers@users.noreply.github.com> Date: Thu, 13 Feb 2020 15:43:18 +0800 Subject: [PATCH 10/10] restore: enhance error handling (#152) * restore: enhance error handling Signed-off-by: 5kbpers * unit test Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * fix region epoch error Signed-off-by: 5kbpers * address comments Signed-off-by: 5kbpers * remove `Restore*` Signed-off-by: 5kbpers * address lint Signed-off-by: 5kbpers * add debug log Signed-off-by: 5kbpers * Apply suggestions from code review Co-Authored-By: kennytm * Update pkg/restore/import.go Co-Authored-By: kennytm * fix retry error Signed-off-by: 5kbpers * handle RegionNotFound error Signed-off-by: 5kbpers Co-authored-by: Neil Shen Co-authored-by: kennytm --- go.mod | 3 + go.sum | 11 +++ pkg/restore/backoff.go | 117 +++++++++++++++++++++++++ pkg/restore/backoff_test.go | 58 +++++++++++++ pkg/restore/client.go | 133 +++++----------------------- pkg/restore/import.go | 168 ++++++++++++++++++------------------ pkg/restore/split.go | 3 +- pkg/restore/util.go | 52 +---------- pkg/task/restore.go | 2 +- pkg/utils/retry.go | 40 +++++++++ tests/br_full_ddl/run.sh | 2 +- 11 files changed, 338 insertions(+), 251 deletions(-) create mode 100644 pkg/restore/backoff.go create mode 100644 pkg/restore/backoff_test.go create mode 100644 pkg/utils/retry.go diff --git a/go.mod b/go.mod index 850750f09..180c58f93 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,14 @@ require ( github.com/cheggaaa/pb/v3 v3.0.1 github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect + github.com/fatih/color v1.9.0 // indirect github.com/fsouza/fake-gcs-server v1.15.0 github.com/go-sql-driver/mysql v1.4.1 github.com/gogo/protobuf v1.3.1 github.com/golang/snappy v0.0.1 // indirect github.com/google/btree v1.0.0 github.com/google/uuid v1.1.1 + github.com/mattn/go-runewidth v0.0.7 // indirect github.com/onsi/ginkgo v1.10.3 // indirect github.com/onsi/gomega v1.7.1 // indirect github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12 @@ -33,6 +35,7 @@ require ( go.uber.org/zap v1.13.0 golang.org/x/net v0.0.0-20191011234655-491137f69257 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 + golang.org/x/tools v0.0.0-20200107184032-11e9d9cc0042 // indirect google.golang.org/api v0.14.0 google.golang.org/grpc v1.25.1 ) diff --git a/go.sum b/go.sum index d5e9c891d..0fe4a3024 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/fake-gcs-server v1.15.0 h1:ss/ztlt10Y64A5qslmxZKsiqW/i28t5DkRtv6qSFaLQ= @@ -224,12 +226,18 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -518,6 +526,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190909082730-f460065e899a h1:mIzbOulag9/gXacgxKlFVwpCOWSfBT3/pDyyCwGA9as= golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -555,6 +564,8 @@ golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2 h1:EtTFh6h4SAKemS+CURDMTDIANuduG5zKEXShyy18bGA= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200107184032-11e9d9cc0042 h1:BKiPVwWbEdmAh+5CBwk13CYeVJQRDJpDnKgDyMOGz9M= +golang.org/x/tools v0.0.0-20200107184032-11e9d9cc0042/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/restore/backoff.go b/pkg/restore/backoff.go new file mode 100644 index 000000000..dae14e109 --- /dev/null +++ b/pkg/restore/backoff.go @@ -0,0 +1,117 @@ +package restore + +import ( + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + "go.uber.org/zap" + + "github.com/pingcap/br/pkg/utils" +) + +var ( + errNotLeader = errors.NewNoStackError("not leader") + errEpochNotMatch = errors.NewNoStackError("epoch not match") + errKeyNotInRegion = errors.NewNoStackError("key not in region") + errRegionNotFound = errors.NewNoStackError("region not found") + errResp = errors.NewNoStackError("response error") + errRewriteRuleNotFound = errors.NewNoStackError("rewrite rule not found") + errRangeIsEmpty = errors.NewNoStackError("range is empty") + errGrpc = errors.NewNoStackError("gRPC error") + + // TODO: add `error` field to `DownloadResponse` for distinguish the errors of gRPC + // and the errors of request + errBadFormat = errors.NewNoStackError("bad format") + errWrongKeyPrefix = errors.NewNoStackError("wrong key prefix") + errFileCorrupted = errors.NewNoStackError("file corrupted") + errCannotRead = errors.NewNoStackError("cannot read externel storage") +) + +const ( + importSSTRetryTimes = 16 + importSSTWaitInterval = 10 * time.Millisecond + importSSTMaxWaitInterval = 1 * time.Second + + downloadSSTRetryTimes = 8 + downloadSSTWaitInterval = 10 * time.Millisecond + downloadSSTMaxWaitInterval = 1 * time.Second + + resetTsRetryTime = 16 + resetTSWaitInterval = 50 * time.Millisecond + resetTSMaxWaitInterval = 500 * time.Millisecond +) + +type importerBackoffer struct { + attempt int + delayTime time.Duration + maxDelayTime time.Duration +} + +func newImportSSTBackoffer() utils.Backoffer { + return &importerBackoffer{ + attempt: importSSTRetryTimes, + delayTime: importSSTWaitInterval, + maxDelayTime: importSSTMaxWaitInterval, + } +} + +func newDownloadSSTBackoffer() utils.Backoffer { + return &importerBackoffer{ + attempt: downloadSSTRetryTimes, + delayTime: downloadSSTWaitInterval, + maxDelayTime: downloadSSTMaxWaitInterval, + } +} + +func (bo *importerBackoffer) NextBackoff(err error) time.Duration { + switch errors.Cause(err) { + case errResp, errGrpc, errEpochNotMatch, errNotLeader: + bo.delayTime = 2 * bo.delayTime + bo.attempt-- + case errRangeIsEmpty, errRewriteRuleNotFound: + // Excepted error, finish the operation + bo.delayTime = 0 + bo.attempt = 0 + default: + // Unexcepted error + bo.delayTime = 0 + bo.attempt = 0 + log.Warn("unexcepted error, stop to retry", zap.Error(err)) + } + if bo.delayTime > bo.maxDelayTime { + return bo.maxDelayTime + } + return bo.delayTime +} + +func (bo *importerBackoffer) Attempt() int { + return bo.attempt +} + +type resetTSBackoffer struct { + attempt int + delayTime time.Duration + maxDelayTime time.Duration +} + +func newResetTSBackoffer() utils.Backoffer { + return &resetTSBackoffer{ + attempt: resetTsRetryTime, + delayTime: resetTSWaitInterval, + maxDelayTime: resetTSMaxWaitInterval, + } +} + +func (bo *resetTSBackoffer) NextBackoff(err error) time.Duration { + bo.delayTime = 2 * bo.delayTime + bo.attempt-- + if bo.delayTime > bo.maxDelayTime { + return bo.maxDelayTime + } + return bo.delayTime +} + +func (bo *resetTSBackoffer) Attempt() int { + return bo.attempt +} diff --git a/pkg/restore/backoff_test.go b/pkg/restore/backoff_test.go new file mode 100644 index 000000000..537f0980c --- /dev/null +++ b/pkg/restore/backoff_test.go @@ -0,0 +1,58 @@ +package restore + +import ( + "context" + "time" + + . "github.com/pingcap/check" + "github.com/pingcap/tidb/util/testleak" + + "github.com/pingcap/br/pkg/utils" +) + +var _ = Suite(&testBackofferSuite{}) + +type testBackofferSuite struct { + mock *utils.MockCluster +} + +func (s *testBackofferSuite) SetUpSuite(c *C) { + var err error + s.mock, err = utils.NewMockCluster() + c.Assert(err, IsNil) +} + +func (s *testBackofferSuite) TearDownSuite(c *C) { + testleak.AfterTest(c)() +} + +func (s *testBackofferSuite) TestImporterBackoffer(c *C) { + var counter int + err := utils.WithRetry(context.Background(), func() error { + defer func() { counter++ }() + switch counter { + case 0: + return errGrpc + case 1: + return errResp + case 2: + return errRangeIsEmpty + } + return nil + }, newImportSSTBackoffer()) + c.Assert(counter, Equals, 3) + c.Assert(err, Equals, errRangeIsEmpty) + + counter = 0 + backoffer := importerBackoffer{ + attempt: 10, + delayTime: time.Nanosecond, + maxDelayTime: time.Nanosecond, + } + err = utils.WithRetry(context.Background(), func() error { + defer func() { counter++ }() + return errResp + }, &backoffer) + c.Assert(counter, Equals, 10) + c.Assert(err, Equals, errResp) +} diff --git a/pkg/restore/client.go b/pkg/restore/client.go index 5e8df4418..5402d78bc 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -2,7 +2,6 @@ package restore import ( "context" - "fmt" "math" "sync" "time" @@ -26,15 +25,9 @@ import ( "github.com/pingcap/br/pkg/utils" ) -const ( - resetTsRetryTime = 16 - resetTSWaitInterval = 50 * time.Millisecond - resetTSMaxWaitInterval = 500 * time.Millisecond - - // defaultChecksumConcurrency is the default number of the concurrent - // checksum tasks. - defaultChecksumConcurrency = 64 -) +// defaultChecksumConcurrency is the default number of the concurrent +// checksum tasks. +const defaultChecksumConcurrency = 64 // Client sends requests to restore files type Client struct { @@ -138,13 +131,10 @@ func (rc *Client) ResetTS(pdAddrs []string) error { restoreTS := rc.backupMeta.GetEndVersion() log.Info("reset pd timestamp", zap.Uint64("ts", restoreTS)) i := 0 - return withRetry(func() error { + return utils.WithRetry(rc.ctx, func() error { idx := i % len(pdAddrs) return utils.ResetTS(pdAddrs[idx], restoreTS) - }, func(e error) bool { - i++ - return true - }, resetTsRetryTime, resetTSWaitInterval, resetTSMaxWaitInterval) + }, newResetTSBackoffer()) } // GetDatabases returns all databases. @@ -228,29 +218,28 @@ func (rc *Client) setSpeedLimit() error { return nil } -// RestoreTable tries to restore the data of a table. -func (rc *Client) RestoreTable( - table *utils.Table, +// RestoreFiles tries to restore the files. +func (rc *Client) RestoreFiles( + files []*backup.File, rewriteRules *RewriteRules, updateCh chan<- struct{}, ) (err error) { start := time.Now() defer func() { elapsed := time.Since(start) - log.Info("restore table", - zap.Stringer("table", table.Schema.Name), zap.Duration("take", elapsed)) - key := fmt.Sprintf("%s.%s", table.Db.Name.String(), table.Schema.Name.String()) - if err != nil { - summary.CollectFailureUnit(key, err) + if err == nil { + log.Info("Restore Files", + zap.Int("files", len(files)), zap.Duration("take", elapsed)) + summary.CollectSuccessUnit("files", elapsed) + } else { + summary.CollectFailureUnit("files", err) } }() - log.Debug("start to restore table", - zap.Stringer("table", table.Schema.Name), - zap.Stringer("db", table.Db.Name), - zap.Array("files", files(table.Files)), + log.Debug("start to restore files", + zap.Int("files", len(files)), ) - errCh := make(chan error, len(table.Files)) + errCh := make(chan error, len(files)) wg := new(sync.WaitGroup) defer close(errCh) err = rc.setSpeedLimit() @@ -258,7 +247,7 @@ func (rc *Client) RestoreTable( return err } - for _, file := range table.Files { + for _, file := range files { wg.Add(1) fileReplica := file rc.workerPool.Apply( @@ -272,100 +261,18 @@ func (rc *Client) RestoreTable( } }) } - for range table.Files { + for range files { err := <-errCh if err != nil { rc.cancel() wg.Wait() log.Error( - "restore table failed", - zap.Stringer("table", table.Schema.Name), - zap.Stringer("db", table.Db.Name), + "restore files failed", zap.Error(err), ) return err } } - log.Info( - "finish to restore table", - zap.Stringer("table", table.Schema.Name), - zap.Stringer("db", table.Db.Name), - ) - return nil -} - -// RestoreDatabase tries to restore the data of a database -func (rc *Client) RestoreDatabase( - db *utils.Database, - rewriteRules *RewriteRules, - updateCh chan<- struct{}, -) (err error) { - start := time.Now() - defer func() { - elapsed := time.Since(start) - log.Info("Restore Database", zap.Stringer("db", db.Schema.Name), zap.Duration("take", elapsed)) - }() - errCh := make(chan error, len(db.Tables)) - wg := new(sync.WaitGroup) - defer close(errCh) - for _, table := range db.Tables { - wg.Add(1) - tblReplica := table - rc.tableWorkerPool.Apply(func() { - defer wg.Done() - select { - case <-rc.ctx.Done(): - errCh <- nil - case errCh <- rc.RestoreTable( - tblReplica, rewriteRules, updateCh): - } - }) - } - for range db.Tables { - err = <-errCh - if err != nil { - wg.Wait() - return err - } - } - return nil -} - -// RestoreAll tries to restore all the data of backup files. -func (rc *Client) RestoreAll( - rewriteRules *RewriteRules, - updateCh chan<- struct{}, -) (err error) { - start := time.Now() - defer func() { - elapsed := time.Since(start) - log.Info("Restore All", zap.Duration("take", elapsed)) - summary.CollectSuccessUnit("restore all", elapsed) - }() - errCh := make(chan error, len(rc.databases)) - wg := new(sync.WaitGroup) - defer close(errCh) - for _, db := range rc.databases { - wg.Add(1) - dbReplica := db - rc.tableWorkerPool.Apply(func() { - defer wg.Done() - select { - case <-rc.ctx.Done(): - errCh <- nil - case errCh <- rc.RestoreDatabase( - dbReplica, rewriteRules, updateCh): - } - }) - } - - for range rc.databases { - err = <-errCh - if err != nil { - wg.Wait() - return err - } - } return nil } diff --git a/pkg/restore/import.go b/pkg/restore/import.go index 77273ebab..01f8456ef 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -2,6 +2,7 @@ package restore import ( "context" + "strings" "sync" "time" @@ -16,25 +17,10 @@ import ( "google.golang.org/grpc" "github.com/pingcap/br/pkg/summary" + "github.com/pingcap/br/pkg/utils" ) -var ( - errNotLeader = errors.New("not leader") - errEpochNotMatch = errors.New("epoch not match") - errRewriteRuleNotFound = errors.New("rewrite rule not found") - errRangeIsEmpty = errors.New("range is empty") -) - -const ( - importScanResgionTime = 10 * time.Second - importFileRetryTimes = 16 - importFileWaitInterval = 10 * time.Millisecond - importFileMaxWaitInterval = 1 * time.Second - - downloadSSTRetryTimes = 8 - downloadSSTWaitInterval = 10 * time.Millisecond - downloadSSTMaxWaitInterval = 1 * time.Second -) +const importScanRegionTime = 10 * time.Second // ImporterClient is used to import a file to TiKV type ImporterClient interface { @@ -172,10 +158,9 @@ func (importer *FileImporter) Import(file *backup.File, rewriteRules *RewriteRul log.Debug("rewrite file keys", zap.Stringer("file", file), zap.Binary("startKey", startKey), - zap.Binary("endKey", endKey), - ) - err = withRetry(func() error { - ctx, cancel := context.WithTimeout(importer.ctx, importScanResgionTime) + zap.Binary("endKey", endKey)) + err = utils.WithRetry(importer.ctx, func() error { + ctx, cancel := context.WithTimeout(importer.ctx, importScanRegionTime) defer cancel() // Scan regions covered by the file range regionInfos, err1 := importer.metaClient.ScanRegions(ctx, startKey, endKey, 0) @@ -185,63 +170,56 @@ func (importer *FileImporter) Import(file *backup.File, rewriteRules *RewriteRul log.Debug("scan regions", zap.Stringer("file", file), zap.Int("count", len(regionInfos))) // Try to download and ingest the file in every region for _, regionInfo := range regionInfos { - var downloadMeta *import_sstpb.SSTMeta info := regionInfo // Try to download file. - err = withRetry(func() error { - var err2 error - var isEmpty bool - downloadMeta, isEmpty, err2 = importer.downloadSST(info, file, rewriteRules) - if err2 != nil { - if err != errRewriteRuleNotFound { - log.Warn("download file failed", - zap.Stringer("file", file), - zap.Stringer("region", info.Region), - zap.Binary("startKey", startKey), - zap.Binary("endKey", endKey), - zap.Error(err2), - ) - } - return err2 - } - if isEmpty { - log.Info( - "file don't have any key in this region, skip it", - zap.Stringer("file", file), - zap.Stringer("region", info.Region), - ) - return errRangeIsEmpty - } - return nil - }, func(e error) bool { - // Scan regions may return some regions which cannot match any rewrite rule, - // like [t{tableID}, t{tableID}_r), those regions should be skipped - return e != errRewriteRuleNotFound && e != errRangeIsEmpty - }, downloadSSTRetryTimes, downloadSSTWaitInterval, downloadSSTMaxWaitInterval) - if err != nil { - if err == errRewriteRuleNotFound || err == errRangeIsEmpty { + var downloadMeta *import_sstpb.SSTMeta + err1 = utils.WithRetry(importer.ctx, func() error { + var e error + downloadMeta, e = importer.downloadSST(info, file, rewriteRules) + return e + }, newDownloadSSTBackoffer()) + if err1 != nil { + if err1 == errRewriteRuleNotFound || err1 == errRangeIsEmpty { // Skip this region continue } - return err + log.Error("download file failed", + zap.Stringer("file", file), + zap.Stringer("region", info.Region), + zap.Binary("startKey", startKey), + zap.Binary("endKey", endKey), + zap.Error(err1)) + return err1 } - err = importer.ingestSST(downloadMeta, info) - if err != nil { - log.Warn("ingest file failed", + err1 = importer.ingestSST(downloadMeta, info) + // If error is `NotLeader`, update the region info and retry + for errors.Cause(err1) == errNotLeader { + log.Debug("ingest sst returns not leader error, retry it", + zap.Stringer("region", info.Region)) + var newInfo *RegionInfo + newInfo, err1 = importer.metaClient.GetRegion(importer.ctx, info.Region.GetStartKey()) + if err1 != nil { + break + } + if !checkRegionEpoch(newInfo, info) { + err1 = errEpochNotMatch + break + } + err1 = importer.ingestSST(downloadMeta, newInfo) + } + if err1 != nil { + log.Error("ingest file failed", zap.Stringer("file", file), zap.Stringer("range", downloadMeta.GetRange()), zap.Stringer("region", info.Region), - zap.Error(err), - ) - return err + zap.Error(err1)) + return err1 } summary.CollectSuccessUnit(summary.TotalKV, file.TotalKvs) summary.CollectSuccessUnit(summary.TotalBytes, file.TotalBytes) } return nil - }, func(e error) bool { - return true - }, importFileRetryTimes, importFileWaitInterval, importFileMaxWaitInterval) + }, newImportSSTBackoffer()) return err } @@ -257,33 +235,25 @@ func (importer *FileImporter) downloadSST( regionInfo *RegionInfo, file *backup.File, rewriteRules *RewriteRules, -) (*import_sstpb.SSTMeta, bool, error) { +) (*import_sstpb.SSTMeta, error) { id, err := uuid.New().MarshalBinary() if err != nil { - return nil, true, errors.Trace(err) + return nil, errors.Trace(err) } // Assume one region reflects to one rewrite rule _, key, err := codec.DecodeBytes(regionInfo.Region.GetStartKey()) if err != nil { - return nil, true, err + return nil, err } regionRule := matchNewPrefix(key, rewriteRules) if regionRule == nil { - log.Debug("cannot find rewrite rule, skip region", - zap.Stringer("region", regionInfo.Region), - zap.Array("tableRule", rules(rewriteRules.Table)), - zap.Array("dataRule", rules(rewriteRules.Data)), - zap.Binary("key", key), - ) - return nil, true, errRewriteRuleNotFound + return nil, errors.Trace(errRewriteRuleNotFound) } rule := import_sstpb.RewriteRule{ OldKeyPrefix: encodeKeyPrefix(regionRule.GetOldKeyPrefix()), NewKeyPrefix: encodeKeyPrefix(regionRule.GetNewKeyPrefix()), } sstMeta := getSSTMetaFromFile(id, file, regionInfo.Region, &rule) - sstMeta.RegionId = regionInfo.Region.GetId() - sstMeta.RegionEpoch = regionInfo.Region.GetRegionEpoch() req := &import_sstpb.DownloadRequest{ Sst: sstMeta, StorageBackend: importer.backend, @@ -298,15 +268,15 @@ func (importer *FileImporter) downloadSST( for _, peer := range regionInfo.Region.GetPeers() { resp, err = importer.importClient.DownloadSST(importer.ctx, peer.GetStoreId(), req) if err != nil { - return nil, true, err + return nil, extractDownloadSSTError(err) } if resp.GetIsEmpty() { - return &sstMeta, true, nil + return nil, errors.Trace(errRangeIsEmpty) } } sstMeta.Range.Start = truncateTS(resp.Range.GetStart()) sstMeta.Range.End = truncateTS(resp.Range.GetEnd()) - return &sstMeta, false, nil + return &sstMeta, nil } func (importer *FileImporter) ingestSST( @@ -329,17 +299,45 @@ func (importer *FileImporter) ingestSST( log.Debug("download SST", zap.Stringer("sstMeta", sstMeta)) resp, err := importer.importClient.IngestSST(importer.ctx, leader.GetStoreId(), req) if err != nil { - return err + if strings.Contains(err.Error(), "RegionNotFound") { + return errors.Trace(errRegionNotFound) + } + return errors.Trace(err) } respErr := resp.GetError() if respErr != nil { - if respErr.EpochNotMatch != nil { - return errEpochNotMatch + log.Debug("ingest sst resp error", zap.Stringer("error", respErr)) + if respErr.GetKeyNotInRegion() != nil { + return errors.Trace(errKeyNotInRegion) } - if respErr.NotLeader != nil { - return errNotLeader + if respErr.GetNotLeader() != nil { + return errors.Trace(errNotLeader) } - return errors.Errorf("ingest failed: %v", respErr) + return errors.Wrap(errResp, respErr.String()) } return nil } + +func checkRegionEpoch(new, old *RegionInfo) bool { + if new.Region.GetId() == old.Region.GetId() && + new.Region.GetRegionEpoch().GetVersion() == old.Region.GetRegionEpoch().GetVersion() && + new.Region.GetRegionEpoch().GetConfVer() == old.Region.GetRegionEpoch().GetConfVer() { + return true + } + return false +} + +func extractDownloadSSTError(e error) error { + err := errGrpc + switch { + case strings.Contains(e.Error(), "bad format"): + err = errBadFormat + case strings.Contains(e.Error(), "wrong prefix"): + err = errWrongKeyPrefix + case strings.Contains(e.Error(), "corrupted"): + err = errFileCorrupted + case strings.Contains(e.Error(), "Cannot read"): + err = errCannotRead + } + return errors.Trace(err) +} diff --git a/pkg/restore/split.go b/pkg/restore/split.go index 31b23a60f..3248fdd0d 100644 --- a/pkg/restore/split.go +++ b/pkg/restore/split.go @@ -111,7 +111,7 @@ SplitRegions: } time.Sleep(interval) if i > 3 { - log.Warn("splitting regions failed, retry it", zap.Error(err)) + log.Warn("splitting regions failed, retry it", zap.Error(err), zap.ByteStrings("keys", keys)) } continue SplitRegions } @@ -259,6 +259,7 @@ func getSplitKeys(rewriteRules *RewriteRules, ranges []Range, regions []*RegionI splitKeys = make([][]byte, 0, 1) } splitKeyMap[region.Region.GetId()] = append(splitKeys, key) + log.Debug("get key for split region", zap.Binary("key", key), zap.Stringer("region", region.Region)) } } return splitKeyMap diff --git a/pkg/restore/util.go b/pkg/restore/util.go index ea8629470..63ee92969 100644 --- a/pkg/restore/util.go +++ b/pkg/restore/util.go @@ -17,31 +17,12 @@ import ( "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/codec" "go.uber.org/zap" - "go.uber.org/zap/zapcore" "github.com/pingcap/br/pkg/summary" ) var recordPrefixSep = []byte("_r") -type files []*backup.File - -func (fs files) MarshalLogArray(arr zapcore.ArrayEncoder) error { - for i := range fs { - arr.AppendString(fs[i].String()) - } - return nil -} - -type rules []*import_sstpb.RewriteRule - -func (rs rules) MarshalLogArray(arr zapcore.ArrayEncoder) error { - for i := range rs { - arr.AppendString(rs[i].String()) - } - return nil -} - // idAllocator always returns a specified ID type idAllocator struct { id int64 @@ -163,40 +144,11 @@ func getSSTMetaFromFile( Start: rangeStart, End: rangeEnd, }, + RegionId: region.GetId(), + RegionEpoch: region.GetRegionEpoch(), } } -type retryableFunc func() error -type continueFunc func(error) bool - -func withRetry( - retryableFunc retryableFunc, - continueFunc continueFunc, - attempts uint, - delayTime time.Duration, - maxDelayTime time.Duration, -) error { - var lastErr error - for i := uint(0); i < attempts; i++ { - err := retryableFunc() - if err != nil { - lastErr = err - // If this is the last attempt, do not wait - if !continueFunc(err) || i == attempts-1 { - break - } - delayTime = 2 * delayTime - if delayTime > maxDelayTime { - delayTime = maxDelayTime - } - time.Sleep(delayTime) - } else { - return nil - } - } - return lastErr -} - // ValidateFileRanges checks and returns the ranges of the files. func ValidateFileRanges( files []*backup.File, diff --git a/pkg/task/restore.go b/pkg/task/restore.go index f2f3caf43..a56a1d6da 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -139,7 +139,7 @@ func RunRestore(c context.Context, cmdName string, cfg *RestoreConfig) error { if err != nil { return err } - err = client.RestoreAll(rewriteRules, updateCh) + err = client.RestoreFiles(files, rewriteRules, updateCh) // always run the post-work even on error, so we don't stuck in the import mode or emptied schedulers postErr := restorePostWork(ctx, client, mgr, removedSchedulers) diff --git a/pkg/utils/retry.go b/pkg/utils/retry.go new file mode 100644 index 000000000..a8f446764 --- /dev/null +++ b/pkg/utils/retry.go @@ -0,0 +1,40 @@ +package utils + +import ( + "context" + "time" +) + +// RetryableFunc presents a retryable opreation +type RetryableFunc func() error + +// Backoffer implements a backoff policy for retrying operations +type Backoffer interface { + // NextBackoff returns a duration to wait before retrying again + NextBackoff(err error) time.Duration + // Attempt returns the remain attempt times + Attempt() int +} + +// WithRetry retrys a given operation with a backoff policy +func WithRetry( + ctx context.Context, + retryableFunc RetryableFunc, + backoffer Backoffer, +) error { + var lastErr error + for backoffer.Attempt() > 0 { + err := retryableFunc() + if err != nil { + lastErr = err + select { + case <-ctx.Done(): + return lastErr + case <-time.After(backoffer.NextBackoff(err)): + } + } else { + return nil + } + } + return lastErr +} diff --git a/tests/br_full_ddl/run.sh b/tests/br_full_ddl/run.sh index 3db1ecd60..1e40415d7 100755 --- a/tests/br_full_ddl/run.sh +++ b/tests/br_full_ddl/run.sh @@ -28,7 +28,7 @@ for i in $(seq $DDL_COUNT); do run_sql "USE $DB; ALTER TABLE $TABLE ADD INDEX (FIELD$i);" done -for i in $(sql $DDL_COUNT); do +for i in $(seq $DDL_COUNT); do if (( RANDOM % 2 )); then run_sql "USE $DB; ALTER TABLE $TABLE DROP INDEX FIELD$i;" fi