Support backup & restore views and sequences (#242)

pkg/backup/client.go

@@ -228,14 +228,23 @@ func BuildBackupRangeAndSchema(
 
 		var dbData []byte
 		idAlloc := autoid.NewAllocator(storage, dbInfo.ID, false, autoid.RowIDAllocType)
+		seqAlloc := autoid.NewAllocator(storage, dbInfo.ID, false, autoid.SequenceType)
 		randAlloc := autoid.NewAllocator(storage, dbInfo.ID, false, autoid.AutoRandomType)
 
 		for _, tableInfo := range dbInfo.Tables {
 			if !tableFilter.Match(&filter.Table{Schema: dbInfo.Name.L, Name: tableInfo.Name.L}) {
 				// Skip tables other than the given table.
 				continue
 			}
-			globalAutoID, err := idAlloc.NextGlobalAutoID(tableInfo.ID)
+			var globalAutoID int64
+			switch {
+			case tableInfo.IsSequence():
+				globalAutoID, err = seqAlloc.NextGlobalAutoID(tableInfo.ID)
+			case tableInfo.IsView():
+				// no auto ID for views.
+			default:
+				globalAutoID, err = idAlloc.NextGlobalAutoID(tableInfo.ID)
+			}
 			if err != nil {
 				return nil, nil, errors.Trace(err)
 			}

pkg/restore/db.go

@@ -106,16 +106,52 @@ func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error {
 			zap.Error(err))
 		return errors.Trace(err)
 	}
-	alterAutoIncIDSQL := fmt.Sprintf(
-		"alter table %s.%s auto_increment = %d",
-		utils.EncloseName(table.Db.Name.O),
-		utils.EncloseName(table.Info.Name.O),
-		table.Info.AutoIncID)
 
-	err = db.se.Execute(ctx, alterAutoIncIDSQL)
+	var restoreMetaSQL string
+	if table.Info.IsSequence() {
+		setValFormat := fmt.Sprintf("do setval(%s.%s, %%d);",
+			utils.EncloseName(table.Db.Name.O),
+			utils.EncloseName(table.Info.Name.O))
+		if table.Info.Sequence.Cycle {
+			increment := table.Info.Sequence.Increment
+			// TiDB sequence's behaviour is designed to keep the same pace
+			// among all nodes within the same cluster. so we need restore round.
+			// Here is a hack way to trigger sequence cycle round > 0 according to
+			// https://github.com/pingcap/br/pull/242#issuecomment-631307978
+			// TODO use sql to set cycle round
+			nextSeqSQL := fmt.Sprintf("do nextval(%s.%s);",
+				utils.EncloseName(table.Db.Name.O),
+				utils.EncloseName(table.Info.Name.O))
+			if increment < 0 {
+				restoreMetaSQL += fmt.Sprintf(setValFormat, table.Info.Sequence.MinValue)
+			} else {
+				restoreMetaSQL += fmt.Sprintf(setValFormat, table.Info.Sequence.MaxValue)
+			}
+			// trigger cycle round > 0
+			restoreMetaSQL += nextSeqSQL
+			restoreMetaSQL += fmt.Sprintf(setValFormat, table.Info.AutoIncID)
+		} else {
+			restoreMetaSQL = fmt.Sprintf(setValFormat, table.Info.AutoIncID)
+		}
+	} else {
+		var alterAutoIncIDFormat string
+		switch {
+		case table.Info.IsView():
+			return nil
+		default:
+			alterAutoIncIDFormat = "alter table %s.%s auto_increment = %d;"
+		}
+		restoreMetaSQL = fmt.Sprintf(
+			alterAutoIncIDFormat,
+			utils.EncloseName(table.Db.Name.O),
+			utils.EncloseName(table.Info.Name.O),
+			table.Info.AutoIncID)
+	}
+
+	err = db.se.Execute(ctx, restoreMetaSQL)
 	if err != nil {
-		log.Error("alter AutoIncID failed",
-			zap.String("query", alterAutoIncIDSQL),
+		log.Error("restore meta sql failed",
+			zap.String("query", restoreMetaSQL),
 			zap.Stringer("db", table.Db.Name),
 			zap.Stringer("table", table.Info.Name),
 			zap.Error(err))

tests/br_views_and_sequences/run.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+DB="$TEST_NAME"
+
+run_sql "create schema $DB;"
+run_sql "create view $DB.view_1 as select 331 as m;"
+run_sql "create view $DB.view_2 as select * from $DB.view_1;"
+run_sql "create sequence $DB.seq_1 nocache cycle maxvalue 40;"
+run_sql "create table $DB.table_1 (m int primary key default next value for $DB.seq_1, b int);"
+run_sql "insert into $DB.table_1 (b) values (8), (12), (16), (20);"
+run_sql "create sequence $DB.seq_2;"
+run_sql "create table $DB.table_2 (a int default next value for $DB.seq_1, b int default next value for $DB.seq_2, c int);"
+run_sql "insert into $DB.table_2 (c) values (24), (28), (32);"
+run_sql "create view $DB.view_3 as select m from $DB.table_1 union select a * b as m from $DB.table_2 union select m from $DB.view_2;"
+run_sql "drop view $DB.view_1;"
+run_sql "create view $DB.view_1 as select 133 as m;"
+
+echo "backup start..."
+run_br backup db --db "$DB" -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+
+run_sql "drop schema $DB;"
+
+echo "restore start..."
+run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+
+set -x
+
+views_count=$(run_sql "select count(*) c, sum(m) s from $DB.view_3;" | tail -2 | paste -sd ';')
+[ "$views_count" = 'c: 8;s: 181' ]
+
+run_sql "insert into $DB.table_2 (c) values (33);"
+seq_val=$(run_sql "select a >= 8 and b >= 4 as g from $DB.table_2 where c = 33;" | tail -1)
+[ "$seq_val" = 'g: 1' ]

tests/download_tools.sh

@@ -27,19 +27,18 @@ fi
 MISSING_TIDB_COMPONENTS=
 for COMPONENT in tidb-server pd-server tikv-server pd-ctl; do
     if [ ! -e "$BIN/$COMPONENT" ]; then
-        MISSING_TIDB_COMPONENTS="$MISSING_TIDB_COMPONENTS tidb-latest-linux-amd64/bin/$COMPONENT"
+        MISSING_TIDB_COMPONENTS="$MISSING_TIDB_COMPONENTS tidb-nightly-linux-amd64/bin/$COMPONENT"
     fi
 done
 
 if [ -n "$MISSING_TIDB_COMPONENTS" ]; then
     echo "Downloading latest TiDB bundle..."
-    # TODO: the url is going to change from 'latest' to 'nightly' someday.
-    curl -L -f -o "$BIN/tidb.tar.gz" "https://download.pingcap.org/tidb-latest-linux-amd64.tar.gz"
+    curl -L -f -o "$BIN/tidb.tar.gz" "https://download.pingcap.org/tidb-nightly-linux-amd64.tar.gz"
     tar -x -f "$BIN/tidb.tar.gz" -C "$BIN/" $MISSING_TIDB_COMPONENTS
     rm "$BIN/tidb.tar.gz"
-    mv "$BIN"/tidb-latest-linux-amd64/bin/* "$BIN/"
-    rmdir "$BIN/tidb-latest-linux-amd64/bin"
-    rmdir "$BIN/tidb-latest-linux-amd64"
+    mv "$BIN"/tidb-nightly-linux-amd64/bin/* "$BIN/"
+    rmdir "$BIN/tidb-nightly-linux-amd64/bin"
+    rmdir "$BIN/tidb-nightly-linux-amd64"
 fi
 
 if [ ! -e "$BIN/go-ycsb" ]; then