Add rename/truncate on read/write test and alter on read test

contrib/client-c/include/pd/MockPDClient.h

@@ -1,46 +1,34 @@
 #pragma once
 
-#include <limits>
 #include <pd/IClient.h>
+#include <limits>
 
-namespace pingcap {
-namespace pd {
+namespace pingcap
+{
+namespace pd
+{
 
 using Clock = std::chrono::system_clock;
-using Seconds = std::chrono::seconds;
 
-class MockPDClient : public IClient {
+class MockPDClient : public IClient
+{
 public:
     MockPDClient() = default;
 
     ~MockPDClient() override {}
 
-    uint64_t getGCSafePoint() override
-    {
-        return (Clock::now() - Seconds(2)).time_since_epoch().count();
-    }
+    uint64_t getGCSafePoint() override { return 1000000000000000; }
 
-    uint64_t getTS() override
-    {
-        return Clock::now().time_since_epoch().count();
-    }
+    uint64_t getTS() override { return Clock::now().time_since_epoch().count(); }
 
-    std::tuple<metapb::Region, metapb::Peer, std::vector<metapb::Peer>> getRegion(std::string) override {
-        throw "not implemented";
-    }
+    std::tuple<metapb::Region, metapb::Peer, std::vector<metapb::Peer>> getRegion(std::string) override { throw "not implemented"; }
 
-    std::tuple<metapb::Region, metapb::Peer, std::vector<metapb::Peer>> getRegionByID(uint64_t) override {
-        throw "not implemented";
-    }
+    std::tuple<metapb::Region, metapb::Peer, std::vector<metapb::Peer>> getRegionByID(uint64_t) override { throw "not implemented"; }
 
-    metapb::Store getStore(uint64_t) override {
-        throw "not implemented";
-    }
+    metapb::Store getStore(uint64_t) override { throw "not implemented"; }
 
-    bool isMock() override {
-        return true;
-    }
+    bool isMock() override { return true; }
 };
 
-}
-}
+} // namespace pd
+} // namespace pingcap

dbms/src/Debug/MockTiDB.cpp

@@ -104,7 +104,7 @@ ColumnInfo getColumnInfoFromColumn(const NameAndTypePair & column, ColumnID id)
     return column_info;
 }
 
-TableID MockTiDB::newTable(const String & database_name, const String & table_name, const ColumnsDescription & columns)
+TableID MockTiDB::newTable(const String & database_name, const String & table_name, const ColumnsDescription & columns, Timestamp tso)
 {
     std::lock_guard lock(tables_mutex);
 
@@ -133,6 +133,7 @@ TableID MockTiDB::newTable(const String & database_name, const String & table_na
 
     table_info.pk_is_handle = false;
     table_info.comment = "Mocked.";
+    table_info.update_timestamp = tso;
 
     auto table = std::make_shared<Table>(database_name, table_name, std::move(table_info));
     tables_by_id.emplace(table->table_info.id, table);
@@ -141,7 +142,7 @@ TableID MockTiDB::newTable(const String & database_name, const String & table_na
     return table->table_info.id;
 }
 
-TableID MockTiDB::newPartition(const String & database_name, const String & table_name, const String & partition_name)
+TableID MockTiDB::newPartition(const String & database_name, const String & table_name, const String & partition_name, Timestamp tso)
 {
     std::lock_guard lock(tables_mutex);
 
@@ -162,6 +163,7 @@ TableID MockTiDB::newPartition(const String & database_name, const String & tabl
     partition_def.id = partition_id;
     partition_def.name = partition_name;
     table_info.partition.definitions.emplace_back(partition_def);
+    table_info.update_timestamp = tso;
 
     // Map the same table object with partition ID as key, so mock schema syncer behaves the same as TiDB,
     // i.e. gives the table info by partition ID.

dbms/src/Debug/MockTiDB.h

@@ -62,9 +62,9 @@ class MockTiDB : public ext::singleton<MockTiDB>
     using TablePtr = std::shared_ptr<Table>;
 
 public:
-    TableID newTable(const String & database_name, const String & table_name, const ColumnsDescription & columns);
+    TableID newTable(const String & database_name, const String & table_name, const ColumnsDescription & columns, Timestamp tso);
 
-    TableID newPartition(const String & database_name, const String & table_name, const String & partition_name);
+    TableID newPartition(const String & database_name, const String & table_name, const String & partition_name, Timestamp tso);
 
     void dropTable(const String & database_name, const String & table_name);
 

dbms/src/Debug/dbgFuncMockTiDBTable.cpp

@@ -41,24 +41,26 @@ void MockTiDBTable::dbgFuncMockTiDBTable(Context & context, const ASTs & args, D
         throw Exception("Invalid TiDB table schema", ErrorCodes::LOGICAL_ERROR);
     ColumnsDescription columns
         = InterpreterCreateQuery::getColumnsDescription(typeid_cast<const ASTExpressionList &>(*columns_ast), context);
+    auto tso = context.getTMTContext().getPDClient()->getTS();
 
-    TableID table_id = MockTiDB::instance().newTable(database_name, table_name, columns);
+    TableID table_id = MockTiDB::instance().newTable(database_name, table_name, columns, tso);
 
     std::stringstream ss;
     ss << "mock table #" << table_id;
     output(ss.str());
 }
 
-void MockTiDBTable::dbgFuncMockTiDBPartition(Context &, const ASTs & args, DBGInvoker::Printer output)
+void MockTiDBTable::dbgFuncMockTiDBPartition(Context & context, const ASTs & args, DBGInvoker::Printer output)
 {
     if (args.size() != 3)
         throw Exception("Args not matched, should be: database-name, table-name, partition-name", ErrorCodes::BAD_ARGUMENTS);
 
     const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
     const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
     const String & partition_name = typeid_cast<const ASTIdentifier &>(*args[2]).name;
+    auto tso = context.getTMTContext().getPDClient()->getTS();
 
-    TableID partition_id = MockTiDB::instance().newPartition(database_name, table_name, partition_name);
+    TableID partition_id = MockTiDB::instance().newPartition(database_name, table_name, partition_name, tso);
 
     std::stringstream ss;
     ss << "mock partition #" << partition_id;

tests/mutable-test/txn_schema/alter_on_read.test

@@ -0,0 +1,174 @@
+# Preparation.
+=> DBGInvoke __enable_schema_sync_service('false')
+
+=> DBGInvoke __drop_tidb_table(default, test)
+=> drop table if exists default.test
+
+=> DBGInvoke __set_flush_threshold(1000000, 1000000)
+=> DBGInvoke __mock_schema_syncer('true')
+
+=> DBGInvoke __enable_schema_sync_service('false')
+
+# Sync add column by reading.
+=> DBGInvoke __mock_tidb_table(default, test, 'col_1 String')
+=> DBGInvoke __refresh_schemas()
+=> DBGInvoke __put_region(4, 0, 100, default, test)
+=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_2 Nullable(Int8)')
+=> DBGInvoke __raft_insert_row(default, test, 4, 50, 'test1', 1)
+# Data in region cache with tso greater than read tso will be ignored.
+=> select * from default.test " --read_tso "1500000000000000
+# Data in region cache with tso less than read tso will be force decoded (extra column will be discarded), even under an out-of-date schema.
+=> select col_1 from default.test
+┌─col_1─┐
+│ test1 │
+└───────┘
+# Verify this schema is truely out-of-date.
+=> select col_2 from default.test
+Received exception from server (version {#WORD}):
+Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_2.
+# Read with specified bigger schema version will trigger schema sync.
+=> select col_2 from default.test " --schema_version "100
+┌─col_2─┐
+│     1 │
+└───────┘
+
+# Sync drop column by reading.
+=> DBGInvoke __try_flush_region(4)
+=> DBGInvoke __drop_column_from_tidb_table(default, test, col_1)
+=> DBGInvoke __raft_insert_row(default, test, 4, 51, 2)
+# Data in region cache with tso less than read tso will be force decoded (missing column will be filled with default value or null), even under an out-of-date schema.
+=> select col_1 from default.test
+┌─col_1─┐
+│ test1 │
+│       │
+└───────┘
+# Read with specified bigger schema version will trigger schema sync.
+=> select col_1 from default.test " --schema_version "100
+Received exception from server (version {#WORD}):
+Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_1.
+
+# Sync type change by checking sign overflow in CH when flushing.
+=> DBGInvoke __try_flush_region(4)
+=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int16)')
+=> DBGInvoke __raft_insert_row(default, test, 4, 52, -128)
+=> select col_2 from default.test
+┌─col_2─┐
+│     1 │
+│     2 │
+│  -128 │
+└───────┘
+=> desc default.test
+┌─name────────┬─type───────────┬─default_type─┬─default_expression─┐
+│ _tidb_rowid │ Int64          │              │                    │
+│ col_2       │ Nullable(Int8) │              │                    │
+└─────────────┴────────────────┴──────────────┴────────────────────┘
+=> DBGInvoke __raft_insert_row(default, test, 4, 53, 128)
+# 128 will overflow when decoding using out-of-date schema (Int8).
+=> select col_2 from default.test
+Received exception from server (version {#WORD}):
+Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 128 of type Nullable(Int8).
+# Read with specified bigger schema version will trigger schema sync.
+=> select col_2 from default.test " --schema_version "100
+┌─col_2─┐
+│     1 │
+│     2 │
+│  -128 │
+│   128 │
+└───────┘
+
+# Sync type change by checking value overflow in CH when flushing.
+=> DBGInvoke __try_flush_region(4)
+=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_2 Nullable(Int64)')
+=> DBGInvoke __raft_insert_row(default, test, 4, 54, 65536)
+# 65536 will overflow when decoding using out-of-date schema (Int16).
+=> select col_2 from default.test
+Received exception from server (version {#WORD}):
+Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 65536 of type Nullable(Int16).
+# Read with specified bigger schema version will trigger schema sync.
+=> select col_2 from default.test " --schema_version "100
+┌─col_2─┐
+│     1 │
+│     2 │
+│  -128 │
+│   128 │
+│ 65536 │
+└───────┘
+
+# Sync add column and type change together by checking value overflow in CH when flushing.
+=> DBGInvoke __try_flush_region(4)
+=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_3 UInt8')
+=> DBGInvoke __refresh_schemas()
+=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_3 UInt64')
+=> DBGInvoke __raft_insert_row(default, test, 4, 55, 0, 256)
+=> DBGInvoke __add_column_to_tidb_table(default, test, 'col_4 Nullable(UInt8)')
+# 256 will overflow when decoding using out-of-date schema (UInt8).
+=> select col_3 from default.test
+Received exception from server (version {#WORD}):
+Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 256 of type UInt8.
+# Read with specified bigger schema version will trigger schema sync.
+=> select col_3, col_4 from default.test " --schema_version "100
+┌─col_3─┬─col_4─┐
+│     0 │    \N │
+│     0 │    \N │
+│     0 │    \N │
+│     0 │    \N │
+│     0 │    \N │
+│   256 │    \N │
+└───────┴───────┘
+
+# Edge values not overflowing.
+=> DBGInvoke __try_flush_region(4)
+=> DBGInvoke __raft_insert_row(default, test, 4, 56, -9223372036854775807, 18446744073709551615, 1)
+=> DBGInvoke __raft_insert_row(default, test, 4, 57, 9223372036854775807, 18446744073709551615, 1)
+=> DBGInvoke __drop_column_from_tidb_table(default, test, col_3)
+=> select col_2, col_3, col_4 from default.test
+┌─col_2─┬─col_3─┬─col_4─┐
+│     1 │     0 │    \N │
+│     2 │     0 │    \N │
+└───────┴───────┴───────┘
+┌─col_2─┬─col_3─┬─col_4─┐
+│  -128 │     0 │    \N │
+│   128 │     0 │    \N │
+│ 65536 │     0 │    \N │
+└───────┴───────┴───────┘
+┌────────────────col_2─┬────────────────col_3─┬─col_4─┐
+│                    0 │                  256 │    \N │
+│ -9223372036854775807 │ 18446744073709551615 │     1 │
+└──────────────────────┴──────────────────────┴───────┘
+┌───────────────col_2─┬────────────────col_3─┬─col_4─┐
+│ 9223372036854775807 │ 18446744073709551615 │     1 │
+└─────────────────────┴──────────────────────┴───────┘
+=> select col_3, col_4 from default.test " --schema_version "100
+Received exception from server (version {#WORD}):
+Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_3.
+
+# Sync drop column and type change together by checking value overflow in CH when flushing.
+=> DBGInvoke __try_flush_region(4)
+=> DBGInvoke __modify_column_in_tidb_table(default, test, 'col_4 Nullable(UInt64)')
+=> DBGInvoke __raft_insert_row(default, test, 4, 58, 0, 256)
+=> DBGInvoke __drop_column_from_tidb_table(default, test, col_2)
+# -256 will overflow when decoding using out-of-date schema (UInt8).
+=> select col_4 from default.test
+Received exception from server (version {#WORD}):
+Code: 49. DB::Exception: Received from {#WORD} DB::Exception: Detected overflow for data 256 of type Nullable(UInt8).
+# Read with specified bigger schema version will trigger schema sync.
+=> select col_4 from default.test " --schema_version "100
+┌─col_4─┐
+│    \N │
+│    \N │
+│    \N │
+│    \N │
+│    \N │
+│    \N │
+│     1 │
+│     1 │
+│   256 │
+└───────┘
+=> select col_2 from default.test
+Received exception from server (version {#WORD}):
+Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier: col_2.
+
+# Clean up.
+=> DBGInvoke __drop_tidb_table(default, test)
+=> drop table if exists default.test
+=> DBGInvoke __enable_schema_sync_service('true')

tests/mutable-test/txn_schema/alter_on_write.test

@@ -94,7 +94,7 @@ Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier
 └───────┴───────┘
 
 # Not sync drop column for edge values not overflowing.
-=> DBGInvoke __raft_insert_row(default, test, 4, 55, -9223372036854775807, 18446744073709551615, 1)
+=> DBGInvoke __raft_insert_row(default, test, 4, 56, -9223372036854775807, 18446744073709551615, 1)
 => DBGInvoke __raft_insert_row(default, test, 4, 57, 9223372036854775807, 18446744073709551615, 1)
 => DBGInvoke __drop_column_from_tidb_table(default, test, col_3)
 => DBGInvoke __try_flush_region(4)
@@ -120,7 +120,7 @@ Code: 47. DB::Exception: Received from {#WORD} DB::Exception: Unknown identifier
 
 # Sync drop column and type change together by checking value overflow in CH when flushing.
 => DBGInvoke __modify_column_in_tidb_table(default, test, 'col_4 Nullable(UInt64)')
-=> DBGInvoke __raft_insert_row(default, test, 4, 58, 0, -256)
+=> DBGInvoke __raft_insert_row(default, test, 4, 58, 0, 256)
 => DBGInvoke __drop_column_from_tidb_table(default, test, col_2)
 => desc default.test
 ┌─name────────┬─type────────────┬─default_type─┬─default_expression─┐