add skip create sqls (#211)

pkg/restore/client.go

@@ -64,6 +64,7 @@ type Client struct {
 	db              *DB
 	rateLimit       uint64
 	isOnline        bool
+	noSchema        bool
 	hasSpeedLimited bool
 
 	restoreStores []uint64
@@ -305,6 +306,10 @@ func (rc *Client) GetTableSchema(
 
 // CreateDatabase creates a database.
 func (rc *Client) CreateDatabase(db *model.DBInfo) error {
+	if rc.IsSkipCreateSQL() {
+		log.Info("skip create database", zap.Stringer("database", db.Name))
+		return nil
+	}
 	return rc.db.CreateDatabase(rc.ctx, db)
 }
 
@@ -320,9 +325,13 @@ func (rc *Client) CreateTables(
 	}
 	newTables := make([]*model.TableInfo, 0, len(tables))
 	for _, table := range tables {
-		err := rc.db.CreateTable(rc.ctx, table)
-		if err != nil {
-			return nil, nil, err
+		if rc.IsSkipCreateSQL() {
+			log.Info("skip create table and alter autoIncID", zap.Stringer("table", table.Info.Name))
+		} else {
+			err := rc.db.CreateTable(rc.ctx, table)
+			if err != nil {
+				return nil, nil, err
+			}
 		}
 		newTableInfo, err := rc.GetTableSchema(dom, table.Db.Name, table.Info.Name)
 		if err != nil {
@@ -847,3 +856,13 @@ func (rc *Client) IsIncremental() bool {
 	return !(rc.backupMeta.StartVersion == rc.backupMeta.EndVersion ||
 		rc.backupMeta.StartVersion == 0)
 }
+
+// EnableSkipCreateSQL sets switch of skip create schema and tables
+func (rc *Client) EnableSkipCreateSQL() {
+	rc.noSchema = true
+}
+
+// IsSkipCreateSQL returns whether we need skip create schema and tables in restore
+func (rc *Client) IsSkipCreateSQL() bool {
+	return rc.noSchema
+}

pkg/task/restore.go

@@ -23,7 +23,8 @@ import (
 )
 
 const (
-	flagOnline = "online"
+	flagOnline   = "online"
+	flagNoSchema = "no-schema"
 )
 
 var schedulers = map[string]struct{}{
@@ -45,13 +46,18 @@ const (
 type RestoreConfig struct {
 	Config
 
-	Online bool `json:"online" toml:"online"`
+	Online   bool `json:"online" toml:"online"`
+	NoSchema bool `json:"no-schema" toml:"no-schema"`
 }
 
 // DefineRestoreFlags defines common flags for the restore command.
 func DefineRestoreFlags(flags *pflag.FlagSet) {
 	// TODO remove experimental tag if it's stable
-	flags.Bool("online", false, "(experimental) Whether online when restore")
+	flags.Bool(flagOnline, false, "(experimental) Whether online when restore")
+	flags.Bool(flagNoSchema, false, "skip creating schemas and tables, reuse existing empty ones")
+
+	// Do not expose this flag
+	_ = flags.MarkHidden(flagNoSchema)
 }
 
 // ParseFromFlags parses the restore-related flags from the flag set.
@@ -61,6 +67,10 @@ func (cfg *RestoreConfig) ParseFromFlags(flags *pflag.FlagSet) error {
 	if err != nil {
 		return errors.Trace(err)
 	}
+	cfg.NoSchema, err = flags.GetBool(flagNoSchema)
+	if err != nil {
+		return errors.Trace(err)
+	}
 	err = cfg.Config.ParseFromFlags(flags)
 	if err != nil {
 		return errors.Trace(err)
@@ -101,6 +111,9 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 	if cfg.Online {
 		client.EnableOnline()
 	}
+	if cfg.NoSchema {
+		client.EnableSkipCreateSQL()
+	}
 	err = client.LoadRestoreStores(ctx)
 	if err != nil {
 		return err

tests/br_db_skip/run.sh

@@ -0,0 +1,72 @@
+#!/bin/sh
+#
+# Copyright 2019 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+DB="$TEST_NAME"
+
+run_sql "CREATE DATABASE $DB;"
+
+run_sql "CREATE TABLE $DB.usertable1 ( \
+  YCSB_KEY varchar(64) NOT NULL, \
+  FIELD0 varchar(1) DEFAULT NULL, \
+  PRIMARY KEY (YCSB_KEY) \
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
+
+run_sql "INSERT INTO $DB.usertable1 VALUES (\"a\", \"b\");"
+run_sql "INSERT INTO $DB.usertable1 VALUES (\"aa\", \"b\");"
+
+# backup db
+echo "backup start..."
+run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4
+
+run_sql "DROP DATABASE $DB;"
+
+run_sql "CREATE DATABASE $DB;"
+# restore db with skip-create-sql must failed
+echo "restore start but must failed"
+fail=false
+run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --no-schema=true || fail=true
+if $fail; then
+    # Error: [schema:1146]Table 'br_db_skip.usertable1' doesn't exist
+    echo "TEST: [$TEST_NAME] restore $DB with no-schema must failed"
+else
+    echo "TEST: [$TEST_NAME] restore $DB with no-schema not failed"
+    exit 1
+fi
+
+
+run_sql "CREATE TABLE $DB.usertable1 ( \
+  YCSB_KEY varchar(64) NOT NULL, \
+  FIELD0 varchar(1) DEFAULT NULL, \
+  PRIMARY KEY (YCSB_KEY) \
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"
+
+echo "restore start must succeed"
+fail=false
+run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --no-schema=true || fail=true
+if $fail; then
+    echo "TEST: [$TEST_NAME] restore $DB with no-schema failed"
+    exit 1
+else
+    echo "TEST: [$TEST_NAME] restore $DB with no-schema succeed"
+fi
+
+table_count=$(run_sql "use $DB; show tables;" | grep "Tables_in" | wc -l)
+if [ "$table_count" -ne "1" ];then
+    echo "TEST: [$TEST_NAME] failed!"
+    exit 1
+fi
+
+run_sql "DROP DATABASE $DB;"