Merge branch 'master' into release-4.0

cmd/backup.go

@@ -16,6 +16,7 @@ import (
 func runBackupCommand(command *cobra.Command, cmdName string) error {
 	cfg := task.BackupConfig{Config: task.Config{LogProgress: HasLogFile()}}
 	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		command.SilenceUsage = false
 		return err
 	}
 	return task.RunBackup(GetDefaultContext(), tidbGlue, cmdName, &cfg)
@@ -24,6 +25,7 @@ func runBackupCommand(command *cobra.Command, cmdName string) error {
 func runBackupRawCommand(command *cobra.Command, cmdName string) error {
 	cfg := task.RawKvConfig{Config: task.Config{LogProgress: HasLogFile()}}
 	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		command.SilenceUsage = false
 		return err
 	}
 	return task.RunBackupRaw(GetDefaultContext(), gluetikv.Glue{}, cmdName, &cfg)
@@ -34,7 +36,7 @@ func NewBackupCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:          "backup",
 		Short:        "backup a TiDB/TiKV cluster",
-		SilenceUsage: false,
+		SilenceUsage: true,
 		PersistentPreRunE: func(c *cobra.Command, args []string) error {
 			if err := Init(c); err != nil {
 				return err

cmd/restore.go

@@ -15,6 +15,7 @@ import (
 func runRestoreCommand(command *cobra.Command, cmdName string) error {
 	cfg := task.RestoreConfig{Config: task.Config{LogProgress: HasLogFile()}}
 	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		command.SilenceUsage = false
 		return err
 	}
 	return task.RunRestore(GetDefaultContext(), tidbGlue, cmdName, &cfg)
@@ -25,6 +26,7 @@ func runRestoreRawCommand(command *cobra.Command, cmdName string) error {
 		RawKvConfig: task.RawKvConfig{Config: task.Config{LogProgress: HasLogFile()}},
 	}
 	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		command.SilenceUsage = false
 		return err
 	}
 	return task.RunRestoreRaw(GetDefaultContext(), gluetikv.Glue{}, cmdName, &cfg)
@@ -33,6 +35,7 @@ func runRestoreRawCommand(command *cobra.Command, cmdName string) error {
 func runRestoreTiflashReplicaCommand(command *cobra.Command, cmdName string) error {
 	cfg := task.RestoreConfig{Config: task.Config{LogProgress: HasLogFile()}}
 	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		command.SilenceUsage = false
 		return err
 	}
 
@@ -44,7 +47,7 @@ func NewRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:          "restore",
 		Short:        "restore a TiDB/TiKV cluster",
-		SilenceUsage: false,
+		SilenceUsage: true,
 		PersistentPreRunE: func(c *cobra.Command, args []string) error {
 			if err := Init(c); err != nil {
 				return err

go.mod

@@ -23,9 +23,9 @@ require (
 	github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
 	github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75
 	github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd
-	github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84
+	github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87
 	github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3
-	github.com/pingcap/tidb v1.1.0-beta.0.20200310133602-7c39e5e5e0bc
+	github.com/pingcap/tidb v1.1.0-beta.0.20200325094938-30e1edae0897
 	github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible
 	github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60
 	github.com/prometheus/client_golang v1.0.0

go.sum

@@ -370,17 +370,17 @@ github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84 h1:u5FOwUw9muF8mBTZVV1dQhoAKiEo2Ci54CxN9XchEEY=
-github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4=
+github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87 h1:533jEUp3mtfWjk0el+awLbyGVxiHcUIGWcR1Y7gB+fg=
+github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4=
 github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 h1:Yrp99FnjHAEuDrSBql2l0IqCtJX7KwJbTsD5hIArkvk=
 github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3/go.mod h1:25GfNw6+Jcr9kca5rtmTb4gKCJ4jOpow2zV2S9Dgafs=
 github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20200302022240-21c8c70d0ab1 h1:YUnUZ914SHFMsOSe/xgH5DKK/thtRma8X8hcszRo3CA=
 github.com/pingcap/sysutil v0.0.0-20200302022240-21c8c70d0ab1/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20200309085538-962fd285f3bb h1:bDbgLaNTRNK6Qw7KjvEqqfCQstY8WMEcXyXTU7yzYKg=
 github.com/pingcap/sysutil v0.0.0-20200309085538-962fd285f3bb/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
-github.com/pingcap/tidb v1.1.0-beta.0.20200310133602-7c39e5e5e0bc h1:1aW3qTRJZjnosvXt1b75KL73b28XRJWBx6jtTtHsybg=
-github.com/pingcap/tidb v1.1.0-beta.0.20200310133602-7c39e5e5e0bc/go.mod h1:WTmfs5zrUGMPw3Enn5FI3buzkU8BDuJ6BhsO/JC239U=
+github.com/pingcap/tidb v1.1.0-beta.0.20200325094938-30e1edae0897 h1:wTNFJMM6GNmG09YoN3/3K8BqiK74zKkurjm4iY+m2mI=
+github.com/pingcap/tidb v1.1.0-beta.0.20200325094938-30e1edae0897/go.mod h1:4CGOiKZSaOU/Da3QYMtp0c3uBE2SxpcLOpESXmeQhcs=
 github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible h1:84F7MFMfdAYObrznvRslmVu43aoihrlL+7mMyMlOi0o=
 github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
@@ -500,6 +500,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yookoala/realpath v1.0.0 h1:7OA9pj4FZd+oZDsyvXWQvjn5oBdcHRTV44PpdMSuImQ=
 github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
@@ -675,8 +676,8 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200301222351-066e0c02454c h1:FD7jysxM+EJqg5UYYy3XYDsAiUickFsn4UiaanJkf8c=
 golang.org/x/tools v0.0.0-20200301222351-066e0c02454c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb h1:iKlO7ROJc6SttHKlxzwGytRtBUqX4VARrNTgP2YLX5M=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200313205530-4303120df7d8 h1:gkI/wGGwpcG5W4hLCzZNGxA4wzWBGGDStRI1MrjDl2Q=
+golang.org/x/tools v0.0.0-20200313205530-4303120df7d8/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

main.go

@@ -41,7 +41,7 @@ func main() {
 		Use:              "br",
 		Short:            "br is a TiDB/TiKV cluster backup restore tool.",
 		TraverseChildren: true,
-		SilenceUsage:     false,
+		SilenceUsage:     true,
 	}
 	cmd.AddFlags(rootCmd)
 	cmd.SetDefaultContext(ctx)

pkg/backup/client.go

@@ -255,7 +255,8 @@ func BuildBackupRangeAndSchema(
 	}
 
 	if backupSchemas.Len() == 0 {
-		return nil, nil, errors.New("nothing to backup")
+		log.Info("nothing to backup")
+		return nil, nil, nil
 	}
 	return ranges, backupSchemas, nil
 }

pkg/backup/schema_test.go

@@ -62,7 +62,7 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) {
 	c.Assert(err, IsNil)
 	_, backupSchemas, err := BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, testFilter, math.MaxUint64)
-	c.Assert(err, NotNil)
+	c.Assert(err, IsNil)
 	c.Assert(backupSchemas, IsNil)
 
 	// Database is not exist.
@@ -72,15 +72,15 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) {
 	c.Assert(err, IsNil)
 	_, backupSchemas, err = BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, fooFilter, math.MaxUint64)
-	c.Assert(err, NotNil)
+	c.Assert(err, IsNil)
 	c.Assert(backupSchemas, IsNil)
 
-	// Empty databse.
+	// Empty database.
 	noFilter, err := filter.New(false, &filter.Rules{})
 	c.Assert(err, IsNil)
 	_, backupSchemas, err = BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, noFilter, math.MaxUint64)
-	c.Assert(err, NotNil)
+	c.Assert(err, IsNil)
 	c.Assert(backupSchemas, IsNil)
 
 	tk.MustExec("use test")

pkg/restore/client.go

@@ -476,7 +476,7 @@ func (rc *Client) RestoreFiles(
 				defer wg.Done()
 				select {
 				case <-rc.ctx.Done():
-					errCh <- nil
+					errCh <- rc.ctx.Err()
 				case errCh <- rc.fileImporter.Import(fileReplica, rewriteRules):
 					updateCh.Inc()
 				}
@@ -527,7 +527,7 @@ func (rc *Client) RestoreRaw(startKey []byte, endKey []byte, files []*backup.Fil
 				defer wg.Done()
 				select {
 				case <-rc.ctx.Done():
-					errCh <- nil
+					errCh <- rc.ctx.Err()
 				case errCh <- rc.fileImporter.Import(fileReplica, emptyRules):
 					updateCh.Inc()
 				}

pkg/restore/db.go

@@ -203,19 +203,26 @@ func FilterDDLJobs(allDDLJobs []*model.Job, tables []*utils.Table) (ddlJobs []*m
 		}
 	}
 
+	type namePair struct {
+		db    string
+		table string
+	}
+
 	for _, table := range tables {
 		tableIDs := make(map[int64]bool)
 		tableIDs[table.Info.ID] = true
-		tableNames := make(map[string]bool)
-		tableNames[table.Info.Name.String()] = true
+		tableNames := make(map[namePair]bool)
+		name := namePair{table.Db.Name.String(), table.Info.Name.String()}
+		tableNames[name] = true
 		for _, job := range allDDLJobs {
 			if job.BinlogInfo.TableInfo != nil {
-				if tableIDs[job.TableID] || tableNames[job.BinlogInfo.TableInfo.Name.String()] {
+				name := namePair{job.SchemaName, job.BinlogInfo.TableInfo.Name.String()}
+				if tableIDs[job.TableID] || tableNames[name] {
 					ddlJobs = append(ddlJobs, job)
 					tableIDs[job.TableID] = true
 					// For truncate table, the id may be changed
 					tableIDs[job.BinlogInfo.TableInfo.ID] = true
-					tableNames[job.BinlogInfo.TableInfo.Name.String()] = true
+					tableNames[name] = true
 				}
 			}
 		}

pkg/task/backup.go

@@ -126,6 +126,10 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
 	if err != nil {
 		return err
 	}
+	// nothing to backup
+	if ranges == nil {
+		return nil
+	}
 
 	ddlJobs := make([]*model.Job, 0)
 	if cfg.LastBackupTS > 0 {

pkg/task/restore.go

@@ -118,13 +118,10 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 		return errors.New("cannot do transactional restore from raw kv data")
 	}
 
-	files, tables, err := filterRestoreFiles(client, cfg)
+	files, tables, dbs, err := filterRestoreFiles(client, cfg)
 	if err != nil {
 		return err
 	}
-	if len(files) == 0 {
-		return errors.New("all files are filtered out from the backup archive, nothing to restore")
-	}
 
 	var newTS uint64
 	if client.IsIncremental() {
@@ -137,10 +134,25 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 	if err != nil {
 		return err
 	}
+	// execute DDL first
 	err = client.ExecDDLs(ddlJobs)
 	if err != nil {
 		return errors.Trace(err)
 	}
+
+	// nothing to restore, maybe only ddl changes in incremental restore
+	if len(files) == 0 {
+		log.Info("all files are filtered out from the backup archive, nothing to restore")
+		return nil
+	}
+
+	for _, db := range dbs {
+		err = client.CreateDatabase(db.Info)
+		if err != nil {
+			return err
+		}
+	}
+
 	rewriteRules, newTables, err := client.CreateTables(mgr.GetDomain(), tables, newTS)
 	if err != nil {
 		return err
@@ -229,12 +241,16 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 
 	// Always run the post-work even on error, so we don't stuck in the import
 	// mode or emptied schedulers
-	err = restorePostWork(ctx, client, mgr, clusterCfg)
-	if err != nil {
-		return err
+	if errRestorePostWork := restorePostWork(ctx, client, mgr, clusterCfg); err == nil {
+		err = errRestorePostWork
 	}
 
-	if err = splitPostWork(ctx, client, newTables); err != nil {
+	if errSplitPostWork := splitPostWork(ctx, client, newTables); err == nil {
+		err = errSplitPostWork
+	}
+
+	// If any error happened, return now, don't execute checksum.
+	if err != nil {
 		return err
 	}
 
@@ -259,10 +275,10 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 func filterRestoreFiles(
 	client *restore.Client,
 	cfg *RestoreConfig,
-) (files []*backup.File, tables []*utils.Table, err error) {
+) (files []*backup.File, tables []*utils.Table, dbs []*utils.Database, err error) {
 	tableFilter, err := filter.New(cfg.CaseSensitive, &cfg.Filter)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	for _, db := range client.GetDatabases() {
@@ -273,17 +289,13 @@ func filterRestoreFiles(
 			}
 
 			if !createdDatabase {
-				if err = client.CreateDatabase(db.Info); err != nil {
-					return nil, nil, err
-				}
+				dbs = append(dbs, db)
 				createdDatabase = true
 			}
-
 			files = append(files, table.Files...)
 			tables = append(tables, table)
 		}
 	}
-
 	return
 }
 

pkg/task/restore_raw.go

@@ -46,7 +46,7 @@ func (cfg *RestoreRawConfig) ParseFromFlags(flags *pflag.FlagSet) error {
 }
 
 // RunRestoreRaw starts a raw kv restore task inside the current goroutine.
-func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreRawConfig) error {
+func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreRawConfig) (err error) {
 	defer summary.Summary(cmdName)
 	ctx, cancel := context.WithCancel(c)
 	defer cancel()
@@ -113,16 +113,18 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR
 	if err != nil {
 		return errors.Trace(err)
 	}
+	defer func() {
+		errPostWork := restorePostWork(ctx, client, mgr, removedSchedulers)
+		if err == nil {
+			err = errPostWork
+		}
+	}()
 
 	err = client.RestoreRaw(cfg.StartKey, cfg.EndKey, files, updateCh)
 	if err != nil {
 		return errors.Trace(err)
 	}
 
-	err = restorePostWork(ctx, client, mgr, removedSchedulers)
-	if err != nil {
-		return errors.Trace(err)
-	}
 	// Restore has finished.
 	updateCh.Close()