From 8fbedbf39c13692c4efaeb15c6f4cb1a46d52245 Mon Sep 17 00:00:00 2001 From: 3pointer Date: Mon, 30 Mar 2020 17:46:16 +0800 Subject: [PATCH] add skip create sqls (#211) --- pkg/restore/client.go | 25 ++++++++++++-- pkg/task/restore.go | 19 +++++++++-- tests/br_db_skip/run.sh | 72 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 6 deletions(-) create mode 100755 tests/br_db_skip/run.sh diff --git a/pkg/restore/client.go b/pkg/restore/client.go index 56bb83233..76dbf5066 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -64,6 +64,7 @@ type Client struct { db *DB rateLimit uint64 isOnline bool + noSchema bool hasSpeedLimited bool restoreStores []uint64 @@ -305,6 +306,10 @@ func (rc *Client) GetTableSchema( // CreateDatabase creates a database. func (rc *Client) CreateDatabase(db *model.DBInfo) error { + if rc.IsSkipCreateSQL() { + log.Info("skip create database", zap.Stringer("database", db.Name)) + return nil + } return rc.db.CreateDatabase(rc.ctx, db) } @@ -320,9 +325,13 @@ func (rc *Client) CreateTables( } newTables := make([]*model.TableInfo, 0, len(tables)) for _, table := range tables { - err := rc.db.CreateTable(rc.ctx, table) - if err != nil { - return nil, nil, err + if rc.IsSkipCreateSQL() { + log.Info("skip create table and alter autoIncID", zap.Stringer("table", table.Info.Name)) + } else { + err := rc.db.CreateTable(rc.ctx, table) + if err != nil { + return nil, nil, err + } } newTableInfo, err := rc.GetTableSchema(dom, table.Db.Name, table.Info.Name) if err != nil { @@ -847,3 +856,13 @@ func (rc *Client) IsIncremental() bool { return !(rc.backupMeta.StartVersion == rc.backupMeta.EndVersion || rc.backupMeta.StartVersion == 0) } + +// EnableSkipCreateSQL sets switch of skip create schema and tables +func (rc *Client) EnableSkipCreateSQL() { + rc.noSchema = true +} + +// IsSkipCreateSQL returns whether we need skip create schema and tables in restore +func (rc *Client) IsSkipCreateSQL() bool { + return rc.noSchema +} diff --git a/pkg/task/restore.go b/pkg/task/restore.go index 33bac7a6c..336758dd7 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -23,7 +23,8 @@ import ( ) const ( - flagOnline = "online" + flagOnline = "online" + flagNoSchema = "no-schema" ) var schedulers = map[string]struct{}{ @@ -45,13 +46,18 @@ const ( type RestoreConfig struct { Config - Online bool `json:"online" toml:"online"` + Online bool `json:"online" toml:"online"` + NoSchema bool `json:"no-schema" toml:"no-schema"` } // DefineRestoreFlags defines common flags for the restore command. func DefineRestoreFlags(flags *pflag.FlagSet) { // TODO remove experimental tag if it's stable - flags.Bool("online", false, "(experimental) Whether online when restore") + flags.Bool(flagOnline, false, "(experimental) Whether online when restore") + flags.Bool(flagNoSchema, false, "skip creating schemas and tables, reuse existing empty ones") + + // Do not expose this flag + _ = flags.MarkHidden(flagNoSchema) } // ParseFromFlags parses the restore-related flags from the flag set. @@ -61,6 +67,10 @@ func (cfg *RestoreConfig) ParseFromFlags(flags *pflag.FlagSet) error { if err != nil { return errors.Trace(err) } + cfg.NoSchema, err = flags.GetBool(flagNoSchema) + if err != nil { + return errors.Trace(err) + } err = cfg.Config.ParseFromFlags(flags) if err != nil { return errors.Trace(err) @@ -101,6 +111,9 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf if cfg.Online { client.EnableOnline() } + if cfg.NoSchema { + client.EnableSkipCreateSQL() + } err = client.LoadRestoreStores(ctx) if err != nil { return err diff --git a/tests/br_db_skip/run.sh b/tests/br_db_skip/run.sh new file mode 100755 index 000000000..e126447c6 --- /dev/null +++ b/tests/br_db_skip/run.sh @@ -0,0 +1,72 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu +DB="$TEST_NAME" + +run_sql "CREATE DATABASE $DB;" + +run_sql "CREATE TABLE $DB.usertable1 ( \ + YCSB_KEY varchar(64) NOT NULL, \ + FIELD0 varchar(1) DEFAULT NULL, \ + PRIMARY KEY (YCSB_KEY) \ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;" + +run_sql "INSERT INTO $DB.usertable1 VALUES (\"a\", \"b\");" +run_sql "INSERT INTO $DB.usertable1 VALUES (\"aa\", \"b\");" + +# backup db +echo "backup start..." +run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 + +run_sql "DROP DATABASE $DB;" + +run_sql "CREATE DATABASE $DB;" +# restore db with skip-create-sql must failed +echo "restore start but must failed" +fail=false +run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --no-schema=true || fail=true +if $fail; then + # Error: [schema:1146]Table 'br_db_skip.usertable1' doesn't exist + echo "TEST: [$TEST_NAME] restore $DB with no-schema must failed" +else + echo "TEST: [$TEST_NAME] restore $DB with no-schema not failed" + exit 1 +fi + + +run_sql "CREATE TABLE $DB.usertable1 ( \ + YCSB_KEY varchar(64) NOT NULL, \ + FIELD0 varchar(1) DEFAULT NULL, \ + PRIMARY KEY (YCSB_KEY) \ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;" + +echo "restore start must succeed" +fail=false +run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --no-schema=true || fail=true +if $fail; then + echo "TEST: [$TEST_NAME] restore $DB with no-schema failed" + exit 1 +else + echo "TEST: [$TEST_NAME] restore $DB with no-schema succeed" +fi + +table_count=$(run_sql "use $DB; show tables;" | grep "Tables_in" | wc -l) +if [ "$table_count" -ne "1" ];then + echo "TEST: [$TEST_NAME] failed!" + exit 1 +fi + +run_sql "DROP DATABASE $DB;"