From 33a6660298c42ca54ee2a8a633d9dbb1e8f8e8e5 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sun, 28 Apr 2019 15:00:05 +0800 Subject: [PATCH] [2.1] Cherry-pick #170 and #172 (#174) * kv: fix handling of column default values (#170) * kv: fix handling of column default values * if the column is AUTO_INCREMENT, fill in with row_id (assume it is missing for the entire table instead of just a few values) * if the column has DEFAULT, fill in that value * tests: ensure DEFAULT CURRENT_TIMESTAMP works * tests,restore: re-enable the exotic_filenames test (#172) --- lightning/kv/sql2kv.go | 7 ++-- lightning/restore/tidb.go | 11 +++++-- tests/default-columns/config.toml | 18 ++++++++++ .../data/defcol-schema-create.sql | 1 + .../default-columns/data/defcol.t-schema.sql | 6 ++++ tests/default-columns/data/defcol.t.1.sql | 1 + tests/default-columns/data/defcol.t.2.sql | 1 + .../default-columns/data/defcol.u-schema.sql | 4 +++ tests/default-columns/data/defcol.u.1.sql | 1 + tests/default-columns/run.sh | 33 +++++++++++++++++++ ...272\253\360\237\245\263-schema-create.sql" | 1 + ...70\255\346\226\207\350\241\250-schema.sql" | 1 + ....\344\270\255\346\226\207\350\241\250.sql" | 1 + tests/exotic_filenames/run.sh | 6 ++-- 14 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 tests/default-columns/config.toml create mode 100644 tests/default-columns/data/defcol-schema-create.sql create mode 100644 tests/default-columns/data/defcol.t-schema.sql create mode 100644 tests/default-columns/data/defcol.t.1.sql create mode 100644 tests/default-columns/data/defcol.t.2.sql create mode 100644 tests/default-columns/data/defcol.u-schema.sql create mode 100644 tests/default-columns/data/defcol.u.1.sql create mode 100755 tests/default-columns/run.sh create mode 100644 "tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263-schema-create.sql" create mode 100644 "tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263.\344\270\255\346\226\207\350\241\250-schema.sql" create mode 100644 "tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263.\344\270\255\346\226\207\350\241\250.sql" diff --git a/lightning/kv/sql2kv.go b/lightning/kv/sql2kv.go index b73aa0674..58517f26c 100644 --- a/lightning/kv/sql2kv.go +++ b/lightning/kv/sql2kv.go @@ -69,13 +69,16 @@ func (kvcodec *TableKVEncoder) Encode( } for i, col := range cols { - if j := columnPermutation[i]; j >= 0 { + if j := columnPermutation[i]; j >= 0 && j < len(row) { value, err = table.CastValue(kvcodec.se, row[j], col.ToInfo()) if err == nil { value, err = col.HandleBadNull(value, kvcodec.se.vars.StmtCtx) } + } else if mysql.HasAutoIncrementFlag(col.Flag) { + // we still need a conversion, e.g. to catch overflow with a TINYINT column. + value, err = table.CastValue(kvcodec.se, types.NewIntDatum(rowID), col.ToInfo()) } else { - value, err = table.GetColOriginDefaultValue(kvcodec.se, col.ToInfo()) + value, err = table.GetColDefaultValue(kvcodec.se, col.ToInfo()) } if err != nil { return nil, errors.Trace(err) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index 31472ba1b..73462725e 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -20,6 +20,7 @@ import ( "net/http" "net/url" "regexp" + "strings" "time" "github.com/pingcap/errors" @@ -73,12 +74,18 @@ func (timgr *TiDBManager) Close() { } func (timgr *TiDBManager) InitSchema(ctx context.Context, database string, tablesSchema map[string]string) error { - createDatabase := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", database) + var createDatabaseBuilder strings.Builder + createDatabaseBuilder.WriteString("CREATE DATABASE IF NOT EXISTS ") + common.WriteMySQLIdentifier(&createDatabaseBuilder, database) + createDatabase := createDatabaseBuilder.String() err := common.ExecWithRetry(ctx, timgr.db, createDatabase, createDatabase) if err != nil { return errors.Trace(err) } - useDB := fmt.Sprintf("USE `%s`", database) + var useDBBuilder strings.Builder + useDBBuilder.WriteString("USE ") + common.WriteMySQLIdentifier(&useDBBuilder, database) + useDB := useDBBuilder.String() err = common.ExecWithRetry(ctx, timgr.db, useDB, useDB) if err != nil { return errors.Trace(err) diff --git a/tests/default-columns/config.toml b/tests/default-columns/config.toml new file mode 100644 index 000000000..bb9f5b054 --- /dev/null +++ b/tests/default-columns/config.toml @@ -0,0 +1,18 @@ +[lightning] +check-requirements = false +file = "/tmp/lightning_test_result/lightning.log" +level = "debug" + +[tikv-importer] +addr = "127.0.0.1:8808" + +[mydumper] +data-source-dir = "tests/default-columns/data" + +[tidb] +host = "127.0.0.1" +port = 4000 +user = "root" +status-port = 10080 +pd-addr = "127.0.0.1:2379" +log-level = "error" diff --git a/tests/default-columns/data/defcol-schema-create.sql b/tests/default-columns/data/defcol-schema-create.sql new file mode 100644 index 000000000..02bc0983c --- /dev/null +++ b/tests/default-columns/data/defcol-schema-create.sql @@ -0,0 +1 @@ +CREATE DATABASE defcol; diff --git a/tests/default-columns/data/defcol.t-schema.sql b/tests/default-columns/data/defcol.t-schema.sql new file mode 100644 index 000000000..05f2ae676 --- /dev/null +++ b/tests/default-columns/data/defcol.t-schema.sql @@ -0,0 +1,6 @@ +CREATE TABLE t ( + pk INT PRIMARY KEY AUTO_INCREMENT, + x INT NULL, + y INT NOT NULL DEFAULT 123, + z DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/tests/default-columns/data/defcol.t.1.sql b/tests/default-columns/data/defcol.t.1.sql new file mode 100644 index 000000000..aef45b86c --- /dev/null +++ b/tests/default-columns/data/defcol.t.1.sql @@ -0,0 +1 @@ +INSERT INTO t () VALUES (), (), (), (), (), (); diff --git a/tests/default-columns/data/defcol.t.2.sql b/tests/default-columns/data/defcol.t.2.sql new file mode 100644 index 000000000..253dfdd8d --- /dev/null +++ b/tests/default-columns/data/defcol.t.2.sql @@ -0,0 +1 @@ +INSERT INTO t VALUES (), (), (); diff --git a/tests/default-columns/data/defcol.u-schema.sql b/tests/default-columns/data/defcol.u-schema.sql new file mode 100644 index 000000000..c85dc9826 --- /dev/null +++ b/tests/default-columns/data/defcol.u-schema.sql @@ -0,0 +1,4 @@ +CREATE TABLE u ( + xx INT UNIQUE AUTO_INCREMENT, + yy INT PRIMARY KEY +); diff --git a/tests/default-columns/data/defcol.u.1.sql b/tests/default-columns/data/defcol.u.1.sql new file mode 100644 index 000000000..19c7c31cb --- /dev/null +++ b/tests/default-columns/data/defcol.u.1.sql @@ -0,0 +1 @@ +INSERT INTO u (yy) VALUES (40), (60); diff --git a/tests/default-columns/run.sh b/tests/default-columns/run.sh new file mode 100755 index 000000000..7a7130bbb --- /dev/null +++ b/tests/default-columns/run.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +run_sql 'DROP DATABASE IF EXISTS defcol' + +run_lightning + +run_sql 'SELECT min(pk), count(pk) FROM defcol.t' +check_contains 'min(pk): 1' +check_contains 'count(pk): 9' + +run_sql 'SELECT pk FROM defcol.t WHERE x IS NOT NULL OR y <> 123 OR z IS NULL OR z NOT BETWEEN now() - INTERVAL 5 MINUTE AND now()' +check_not_contains 'pk:' + +run_sql 'SELECT xx FROM defcol.u WHERE yy = 40' +check_contains 'xx: 1' + +run_sql 'SELECT xx FROM defcol.u WHERE yy = 60' +check_contains 'xx: 2' diff --git "a/tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263-schema-create.sql" "b/tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263-schema-create.sql" new file mode 100644 index 000000000..d18c9b3ff --- /dev/null +++ "b/tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263-schema-create.sql" @@ -0,0 +1 @@ +create database `中文庫🥳`; diff --git "a/tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263.\344\270\255\346\226\207\350\241\250-schema.sql" "b/tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263.\344\270\255\346\226\207\350\241\250-schema.sql" new file mode 100644 index 000000000..449584777 --- /dev/null +++ "b/tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263.\344\270\255\346\226\207\350\241\250-schema.sql" @@ -0,0 +1 @@ +create table 中文表(a int primary key); diff --git "a/tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263.\344\270\255\346\226\207\350\241\250.sql" "b/tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263.\344\270\255\346\226\207\350\241\250.sql" new file mode 100644 index 000000000..af77ea8c2 --- /dev/null +++ "b/tests/exotic_filenames/data/\344\270\255\346\226\207\345\272\253\360\237\245\263.\344\270\255\346\226\207\350\241\250.sql" @@ -0,0 +1 @@ +insert into 中文表 values (2345); diff --git a/tests/exotic_filenames/run.sh b/tests/exotic_filenames/run.sh index ba4d1b16b..cd58c36d5 100755 --- a/tests/exotic_filenames/run.sh +++ b/tests/exotic_filenames/run.sh @@ -16,11 +16,10 @@ # Confirm the behavior for some exotic filenames # Do not enable until https://github.com/pingcap/tidb/pull/8302 is merged. -exit 0 - set -eu run_sql 'DROP DATABASE IF EXISTS `x``f"n`;' +run_sql 'DROP DATABASE IF EXISTS `中文庫🥳`;' run_lightning echo 'Import finished' @@ -33,3 +32,6 @@ check_contains 'b > 80000: 1' run_sql 'SELECT _tidb_rowid > 80000, b > 80000 FROM `x``f"n`.`exotic``table````name` WHERE a = "gggggg"' check_contains '_tidb_rowid > 80000: 1' check_contains 'b > 80000: 1' + +run_sql 'SELECT * FROM `中文庫🥳`.中文表' +check_contains 'a: 2345'