From 5e205105f52048934d18ccace1b70a86a0ed08a5 Mon Sep 17 00:00:00 2001 From: JaySon Date: Thu, 9 Feb 2023 12:33:59 +0800 Subject: [PATCH] Remove useless IDAsPathUpgrader (#6782) ref pingcap/tiflash#6233 --- dbms/src/CMakeLists.txt | 1 - dbms/src/Common/CPUAffinityManager.cpp | 4 +- dbms/src/Common/CPUAffinityManager.h | 8 +- dbms/src/Common/Config/ConfigProcessor.h | 1 + dbms/src/Common/FileChecker.h | 2 + dbms/src/Common/LRUCache.h | 2 +- dbms/src/Common/MyTime.cpp | 2 +- dbms/src/Common/getNumberOfCPUCores.cpp | 3 +- dbms/src/Flash/Mpp/MPPHandler.h | 6 +- dbms/src/Interpreters/CMakeLists.txt | 25 - dbms/src/Interpreters/IDAsPathUpgrader.cpp | 929 ------------------ dbms/src/Interpreters/IDAsPathUpgrader.h | 202 ---- dbms/src/Interpreters/sortBlock.h | 2 +- dbms/src/Interpreters/tests/CMakeLists.txt | 55 -- dbms/src/Interpreters/tests/aggregate.cpp | 117 --- dbms/src/Interpreters/tests/create_query.cpp | 115 --- .../tests/gtest_id_as_path_upgrader.cpp | 210 ---- dbms/src/Interpreters/tests/hash_map.cpp | 319 ------ dbms/src/Interpreters/tests/hash_map3.cpp | 105 -- .../Interpreters/tests/hash_map_string.cpp | 505 ---------- .../Interpreters/tests/hash_map_string_2.cpp | 686 ------------- .../Interpreters/tests/hash_map_string_3.cpp | 525 ---------- .../tests/hash_map_string_small.cpp | 206 ---- .../tests/logical_expressions_optimizer.cpp | 310 ------ dbms/src/Interpreters/tests/select_query.cpp | 69 -- .../Interpreters/tests/two_level_hash_map.cpp | 149 --- dbms/src/Interpreters/tests/users.cpp | 299 ------ dbms/src/Server/Server.cpp | 28 +- .../Storages/DeltaMerge/DeltaIndexManager.h | 4 +- dbms/src/Storages/DeltaMerge/File/DMFile.cpp | 21 +- dbms/src/Storages/DeltaMerge/File/DMFile.h | 5 +- .../Storages/DeltaMerge/ReadThread/CPU.cpp | 5 +- dbms/src/Storages/DeltaMerge/ReadThread/CPU.h | 12 +- .../DeltaMerge/ReadThread/MergedTask.h | 10 +- .../ReadThread/SegmentReadTaskScheduler.cpp | 2 +- .../ReadThread/SegmentReadTaskScheduler.h | 2 +- .../DeltaMerge/ReadThread/SegmentReader.cpp | 10 +- .../DeltaMerge/ReadThread/SegmentReader.h | 11 +- dbms/src/Storages/DeltaMerge/SchemaUpdate.cpp | 6 +- dbms/src/Storages/IStorage.h | 2 +- dbms/src/Storages/MarkCache.h | 2 +- dbms/src/Storages/Page/V2/PageFile.cpp | 4 +- dbms/src/Storages/Page/V3/BlobStore.cpp | 4 +- dbms/src/Storages/Page/V3/PageDirectory.cpp | 3 +- .../Page/V3/Universal/UniversalWriteBatch.h | 21 +- dbms/src/Storages/Page/WriteBatch.h | 2 +- libs/libdaemon/src/BaseDaemon.cpp | 7 +- tests/_env.sh | 2 +- .../misc/timestamp_rough_set_filter.test | 3 + tests/delta-merge-test/run.sh | 6 +- tests/docker/_env.sh | 2 +- .../test%2Ddb/test%2Dtbl/stable/.gitignore | 0 .../data/test%2Ddb/test-tbl/data/.gitignore | 0 .../data/test%2Ddb/test-tbl/log/.gitignore | 0 .../data/test%2Ddb/test-tbl/meta/.gitignore | 0 .../data/test/test%2Dtbl/stable/.gitignore | 0 .../data0/data/test/test-tbl/data/.gitignore | 0 .../data0/data/test/test-tbl/log/.gitignore | 0 .../data0/data/test/test-tbl/meta/.gitignore | 0 .../data0/metadata/test%2Ddb.sql | 16 - .../data0/metadata/test%2Ddb/test%2Dtbl.sql | 20 - .../flash-1136-v3.1.0/data0/metadata/test.sql | 16 - .../data0/metadata/test/test%2Dtbl.sql | 20 - .../test%2Ddb/test%2Dtbl/stable/.gitignore | 0 .../data/test%2Ddb/test-tbl/data/.gitignore | 0 .../data/test%2Ddb/test-tbl/log/.gitignore | 0 .../data/test%2Ddb/test-tbl/meta/.gitignore | 0 .../data/test-db/test-tbl/stable/.gitignore | 0 .../data0/data/test/test-tbl/data/.gitignore | 0 .../data0/data/test/test-tbl/log/.gitignore | 0 .../data0/data/test/test-tbl/meta/.gitignore | 0 .../data/test/test-tbl/stable/.gitignore | 0 .../flash-1136/data0/metadata/test%2Ddb.sql | 16 - .../data0/metadata/test%2Ddb/test%2Dtbl.sql | 20 - .../flash-1136/data0/metadata/test.sql | 16 - .../data0/metadata/test/test%2Dtbl.sql | 20 - .../data/test-db/test-tbl/stable/.gitignore | 0 .../data/test/#hello-world/data/.gitignore | 0 .../data/test/#hello-world/log/.gitignore | 0 .../data/test/#hello-world/meta/.gitignore | 0 .../data/test/#hello-world/stable/.gitignore | 0 .../issue-941/data0/metadata/test.sql | 16 - .../data0/metadata/test/%23hello%2Dworld.sql | 20 - .../data/test/#hello-world/stable/.gitignore | 0 .../db/data/test/abc/data/.gitignore | 0 .../db/data/test/abc/log/.gitignore | 0 .../db/data/test/abc/meta/.gitignore | 0 .../db/data/test/abc/stable/.gitignore | 0 .../db/data/test/emp_bak/data/.gitignore | 0 .../db/data/test/emp_bak/log/.gitignore | 0 .../db/data/test/emp_bak/meta/.gitignore | 0 .../db/data/test/emp_bak/stable/.gitignore | 0 .../db/data/test/emp_bak_49/data/.gitignore | 0 .../db/data/test/emp_bak_49/log/.gitignore | 0 .../db/data/test/emp_bak_49/meta/.gitignore | 0 .../db/data/test/emp_bak_49/stable/.gitignore | 0 .../db/data/test/emp_bak_50/data/.gitignore | 0 .../db/data/test/emp_bak_50/log/.gitignore | 0 .../db/data/test/emp_bak_50/meta/.gitignore | 0 .../db/data/test/emp_bak_50/stable/.gitignore | 0 .../db/data/test/emp_bak_51/data/.gitignore | 0 .../db/data/test/emp_bak_51/log/.gitignore | 0 .../db/data/test/emp_bak_51/meta/.gitignore | 0 .../db/data/test/emp_bak_51/stable/.gitignore | 0 .../db/data/test/emp_bak_52/data/.gitignore | 0 .../db/data/test/emp_bak_52/log/.gitignore | 0 .../db/data/test/emp_bak_52/meta/.gitignore | 0 .../db/data/test/emp_bak_52/stable/.gitignore | 0 .../testdata/oncall-1651/db/metadata/test.sql | 16 - .../oncall-1651/db/metadata/test/abc.sql | 20 - .../oncall-1651/db/metadata/test/emp_bak.sql | 23 - .../db/metadata/test/emp_bak_49.sql | 23 - .../db/metadata/test/emp_bak_50.sql | 23 - .../db/metadata/test/emp_bak_51.sql | 23 - .../db/metadata/test/emp_bak_52.sql | 23 - 115 files changed, 96 insertions(+), 5266 deletions(-) delete mode 100644 dbms/src/Interpreters/CMakeLists.txt delete mode 100644 dbms/src/Interpreters/IDAsPathUpgrader.cpp delete mode 100644 dbms/src/Interpreters/IDAsPathUpgrader.h delete mode 100644 dbms/src/Interpreters/tests/CMakeLists.txt delete mode 100644 dbms/src/Interpreters/tests/aggregate.cpp delete mode 100644 dbms/src/Interpreters/tests/create_query.cpp delete mode 100644 dbms/src/Interpreters/tests/gtest_id_as_path_upgrader.cpp delete mode 100644 dbms/src/Interpreters/tests/hash_map.cpp delete mode 100644 dbms/src/Interpreters/tests/hash_map3.cpp delete mode 100644 dbms/src/Interpreters/tests/hash_map_string.cpp delete mode 100644 dbms/src/Interpreters/tests/hash_map_string_2.cpp delete mode 100644 dbms/src/Interpreters/tests/hash_map_string_3.cpp delete mode 100644 dbms/src/Interpreters/tests/hash_map_string_small.cpp delete mode 100644 dbms/src/Interpreters/tests/logical_expressions_optimizer.cpp delete mode 100644 dbms/src/Interpreters/tests/select_query.cpp delete mode 100644 dbms/src/Interpreters/tests/two_level_hash_map.cpp delete mode 100644 dbms/src/Interpreters/tests/users.cpp delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test%2Dtbl/stable/.gitignore delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test-tbl/data/.gitignore delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test-tbl/log/.gitignore delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test-tbl/meta/.gitignore delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/data/test/test%2Dtbl/stable/.gitignore delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/data/test/test-tbl/data/.gitignore delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/data/test/test-tbl/log/.gitignore delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/data/test/test-tbl/meta/.gitignore delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb.sql delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb/test%2Dtbl.sql delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/metadata/test.sql delete mode 100644 tests/testdata/flash-1136-v3.1.0/data0/metadata/test/test%2Dtbl.sql delete mode 100644 tests/testdata/flash-1136-v3.1.0/data1/data/test%2Ddb/test%2Dtbl/stable/.gitignore delete mode 100644 tests/testdata/flash-1136/data0/data/test%2Ddb/test-tbl/data/.gitignore delete mode 100644 tests/testdata/flash-1136/data0/data/test%2Ddb/test-tbl/log/.gitignore delete mode 100644 tests/testdata/flash-1136/data0/data/test%2Ddb/test-tbl/meta/.gitignore delete mode 100644 tests/testdata/flash-1136/data0/data/test-db/test-tbl/stable/.gitignore delete mode 100644 tests/testdata/flash-1136/data0/data/test/test-tbl/data/.gitignore delete mode 100644 tests/testdata/flash-1136/data0/data/test/test-tbl/log/.gitignore delete mode 100644 tests/testdata/flash-1136/data0/data/test/test-tbl/meta/.gitignore delete mode 100644 tests/testdata/flash-1136/data0/data/test/test-tbl/stable/.gitignore delete mode 100644 tests/testdata/flash-1136/data0/metadata/test%2Ddb.sql delete mode 100644 tests/testdata/flash-1136/data0/metadata/test%2Ddb/test%2Dtbl.sql delete mode 100644 tests/testdata/flash-1136/data0/metadata/test.sql delete mode 100644 tests/testdata/flash-1136/data0/metadata/test/test%2Dtbl.sql delete mode 100644 tests/testdata/flash-1136/data1/data/test-db/test-tbl/stable/.gitignore delete mode 100644 tests/testdata/issue-941/data0/data/test/#hello-world/data/.gitignore delete mode 100644 tests/testdata/issue-941/data0/data/test/#hello-world/log/.gitignore delete mode 100644 tests/testdata/issue-941/data0/data/test/#hello-world/meta/.gitignore delete mode 100644 tests/testdata/issue-941/data0/data/test/#hello-world/stable/.gitignore delete mode 100644 tests/testdata/issue-941/data0/metadata/test.sql delete mode 100644 tests/testdata/issue-941/data0/metadata/test/%23hello%2Dworld.sql delete mode 100644 tests/testdata/issue-941/data1/data/test/#hello-world/stable/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/abc/data/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/abc/log/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/abc/meta/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/abc/stable/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak/data/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak/log/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak/meta/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak/stable/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_49/data/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_49/log/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_49/meta/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_49/stable/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_50/data/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_50/log/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_50/meta/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_50/stable/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_51/data/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_51/log/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_51/meta/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_51/stable/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_52/data/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_52/log/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_52/meta/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/data/test/emp_bak_52/stable/.gitignore delete mode 100644 tests/testdata/oncall-1651/db/metadata/test.sql delete mode 100644 tests/testdata/oncall-1651/db/metadata/test/abc.sql delete mode 100644 tests/testdata/oncall-1651/db/metadata/test/emp_bak.sql delete mode 100644 tests/testdata/oncall-1651/db/metadata/test/emp_bak_49.sql delete mode 100644 tests/testdata/oncall-1651/db/metadata/test/emp_bak_50.sql delete mode 100644 tests/testdata/oncall-1651/db/metadata/test/emp_bak_51.sql delete mode 100644 tests/testdata/oncall-1651/db/metadata/test/emp_bak_52.sql diff --git a/dbms/src/CMakeLists.txt b/dbms/src/CMakeLists.txt index 058c15d7cc5..9c8fe816ba0 100644 --- a/dbms/src/CMakeLists.txt +++ b/dbms/src/CMakeLists.txt @@ -23,7 +23,6 @@ add_subdirectory (Storages) add_subdirectory (Parsers) add_subdirectory (IO) add_subdirectory (Functions) -add_subdirectory (Interpreters) add_subdirectory (AggregateFunctions) add_subdirectory (Server) add_subdirectory (Client) diff --git a/dbms/src/Common/CPUAffinityManager.cpp b/dbms/src/Common/CPUAffinityManager.cpp index 44676903321..ca326e23fd7 100644 --- a/dbms/src/Common/CPUAffinityManager.cpp +++ b/dbms/src/Common/CPUAffinityManager.cpp @@ -24,9 +24,9 @@ #include #include +#include #include #include -#include #include #include #include @@ -75,7 +75,7 @@ CPUAffinityManager & CPUAffinityManager::getInstance() CPUAffinityManager::CPUAffinityManager() : query_cpu_percent(0) , cpu_cores(0) - , log(&Poco::Logger::get("CPUAffinityManager")) + , log(Logger::get()) {} #ifdef __linux__ diff --git a/dbms/src/Common/CPUAffinityManager.h b/dbms/src/Common/CPUAffinityManager.h index 239604f6213..8cac799a90c 100644 --- a/dbms/src/Common/CPUAffinityManager.h +++ b/dbms/src/Common/CPUAffinityManager.h @@ -24,7 +24,6 @@ namespace Poco { -class Logger; namespace Util { class LayeredConfiguration; @@ -33,10 +32,13 @@ class LayeredConfiguration; namespace DB { +class Logger; +using LoggerPtr = std::shared_ptr; namespace tests { class CPUAffinityManagerTest_CPUAffinityManager_Test; -} +} // namespace tests + struct CPUAffinityConfig { CPUAffinityConfig() @@ -132,7 +134,7 @@ class CPUAffinityManager MAYBE_UNUSED_MEMBER int cpu_cores; std::vector query_threads; - Poco::Logger * log; + LoggerPtr log; CPUAffinityManager(); // Disable copy and move diff --git a/dbms/src/Common/Config/ConfigProcessor.h b/dbms/src/Common/Config/ConfigProcessor.h index db5e4fca9bb..7618f58e915 100644 --- a/dbms/src/Common/Config/ConfigProcessor.h +++ b/dbms/src/Common/Config/ConfigProcessor.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/dbms/src/Common/FileChecker.h b/dbms/src/Common/FileChecker.h index c0f606a3a54..15c0ccccb25 100644 --- a/dbms/src/Common/FileChecker.h +++ b/dbms/src/Common/FileChecker.h @@ -15,8 +15,10 @@ #pragma once #include +#include #include +#include #include diff --git a/dbms/src/Common/LRUCache.h b/dbms/src/Common/LRUCache.h index d2a817d2f51..cea64994c24 100644 --- a/dbms/src/Common/LRUCache.h +++ b/dbms/src/Common/LRUCache.h @@ -315,7 +315,7 @@ class LRUCache { // If queue.insert() throws exception, cells and queue will be in inconsistent. cells.erase(it); - tryLogCurrentException(Logger::get("LRUCache"), "queue.insert throw exception"); + tryLogCurrentException(Logger::get(), "queue.insert throw exception"); throw; } } diff --git a/dbms/src/Common/MyTime.cpp b/dbms/src/Common/MyTime.cpp index be3bcc19c4a..e01d27b6581 100644 --- a/dbms/src/Common/MyTime.cpp +++ b/dbms/src/Common/MyTime.cpp @@ -2647,7 +2647,7 @@ std::optional MyDateTimeParser::parseAsPackedUInt(const StringRef & str_ if (!f(ctx, my_time)) { #ifndef NDEBUG - LOG_TRACE(&Poco::Logger::get("MyDateTimeParser"), + LOG_TRACE(Logger::get(), "parse error, [str={}] [format={}] [parse_pos={}]", ctx.view.toString(), format, diff --git a/dbms/src/Common/getNumberOfCPUCores.cpp b/dbms/src/Common/getNumberOfCPUCores.cpp index fd615d6a471..2d955368c63 100644 --- a/dbms/src/Common/getNumberOfCPUCores.cpp +++ b/dbms/src/Common/getNumberOfCPUCores.cpp @@ -62,9 +62,8 @@ void computeAndSetNumberOfPhysicalCPUCores(UInt16 number_of_logical_cpu_cores_, auto hardware_logical_cpu_cores = std::thread::hardware_concurrency(); UInt16 physical_cpu_cores = number_of_logical_cpu_cores_ / (hardware_logical_cpu_cores / number_of_hardware_physical_cores); CPUCores::number_of_physical_cpu_cores = physical_cpu_cores > 0 ? physical_cpu_cores : 1; - auto log = DB::Logger::get("CPUCores"); LOG_INFO( - log, + DB::Logger::get(), "logical cpu cores: {}, hardware logical cpu cores: {}, hardware physical cpu cores: {}, physical cpu cores: {}, number_of_physical_cpu_cores: {}", number_of_logical_cpu_cores_, hardware_logical_cpu_cores, diff --git a/dbms/src/Flash/Mpp/MPPHandler.h b/dbms/src/Flash/Mpp/MPPHandler.h index 8041d02d444..79e548546f6 100644 --- a/dbms/src/Flash/Mpp/MPPHandler.h +++ b/dbms/src/Flash/Mpp/MPPHandler.h @@ -24,12 +24,12 @@ class MPPHandler { const mpp::DispatchTaskRequest & task_request; - Poco::Logger * log; + LoggerPtr log; public: - MPPHandler(const mpp::DispatchTaskRequest & task_request_) + explicit MPPHandler(const mpp::DispatchTaskRequest & task_request_) : task_request(task_request_) - , log(&Poco::Logger::get("MPPHandler")) + , log(Logger::get()) {} grpc::Status execute(const ContextPtr & context, mpp::DispatchTaskResponse * response); void handleError(const MPPTaskPtr & task, String error); diff --git a/dbms/src/Interpreters/CMakeLists.txt b/dbms/src/Interpreters/CMakeLists.txt deleted file mode 100644 index f6f7fb6c660..00000000000 --- a/dbms/src/Interpreters/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -list(GET Poco_INCLUDE_DIRS 0 Poco_Foundation_INCLUDE_DIR) -list(GET Poco_INCLUDE_DIRS 1 Poco_Util_INCLUDE_DIR) - -if (NOT DOUBLE_CONVERSION_INCLUDE_DIR) - get_target_property(DOUBLE_CONVERSION_INCLUDE_DIR ${DOUBLE_CONVERSION_LIBRARIES} INTERFACE_INCLUDE_DIRECTORIES) -endif () - -if (ENABLE_TESTS) - add_subdirectory (tests EXCLUDE_FROM_ALL) -endif () diff --git a/dbms/src/Interpreters/IDAsPathUpgrader.cpp b/dbms/src/Interpreters/IDAsPathUpgrader.cpp deleted file mode 100644 index e253dfdb66d..00000000000 --- a/dbms/src/Interpreters/IDAsPathUpgrader.cpp +++ /dev/null @@ -1,929 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ -extern const int BAD_ARGUMENTS; -extern const int LOGICAL_ERROR; -extern const int FILE_DOESNT_EXIST; -extern const int SYNTAX_ERROR; -} // namespace ErrorCodes - -static constexpr auto SYSTEM_DATABASE = "system"; - -namespace -{ -std::shared_ptr getDatabaseEngine(const FileProviderPtr & file_provider, const String & filename) -{ - String query; - if (Poco::File(filename).exists()) - { - ReadBufferFromFileProvider in(file_provider, filename, EncryptionPath(filename, ""), 1024); - readStringUntilEOF(query, in); - } - else - { - // only directory exists, "default" database, return "Ordinary" engine by default. - return std::static_pointer_cast(makeASTFunction("Ordinary")); - } - - ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "in file " + filename, 0); - ASTCreateQuery & ast_create_query = typeid_cast(*ast); - auto * storage = ast_create_query.storage; - if (storage == nullptr || storage->engine == nullptr || storage->engine->name.empty()) - { - throw Exception("Can not get database engine for file: " + filename, ErrorCodes::LOGICAL_ERROR); - } - - return std::static_pointer_cast(storage->engine->clone()); -} - -// Get from `table_metadata_file` -std::pair getTableInfo(const FileProviderPtr & file_provider, const String & table_metadata_file) -{ - String definition; - if (Poco::File(table_metadata_file).exists()) - { - ReadBufferFromFileProvider in(file_provider, table_metadata_file, EncryptionPath(table_metadata_file, ""), 1024); - readStringUntilEOF(definition, in); - } - else - { - throw Exception("Can not open table schema file: " + table_metadata_file, ErrorCodes::LOGICAL_ERROR); - } - - ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, definition.data(), definition.data() + definition.size(), "in file " + table_metadata_file, 0); - ASTCreateQuery & ast_create_query = typeid_cast(*ast); - auto * storage = ast_create_query.storage; - if (storage == nullptr || storage->engine == nullptr || storage->engine->name.empty()) - { - throw Exception("Can not get table engine for file: " + table_metadata_file, ErrorCodes::LOGICAL_ERROR); - } - - TiDB::TableInfo info; - ASTFunction * engine = storage->engine; - const auto * args = typeid_cast(engine->arguments.get()); - if (args == nullptr) - throw Exception("Can not cast table engine arguments", ErrorCodes::BAD_ARGUMENTS); - - const ASTLiteral * table_info_ast = nullptr; - if (engine->name == MutableSupport::delta_tree_storage_name) - { - if (args->children.size() >= 2) - { - table_info_ast = typeid_cast(args->children[1].get()); - } - } - else if (engine->name == MutableSupport::txn_storage_name) - { - if (args->children.size() >= 3) - { - table_info_ast = typeid_cast(args->children[2].get()); - } - } - else - { - throw Exception("Unknown storage engine: " + engine->name, ErrorCodes::LOGICAL_ERROR); - } - - if (table_info_ast && table_info_ast->value.getType() == Field::Types::String) - { - const auto table_info_json = safeGet(table_info_ast->value); - if (!table_info_json.empty()) - { - info.deserialize(table_info_json); - return {ast_create_query.table, info}; - } - } - - throw Exception("Can not get TableInfo for file: " + table_metadata_file, ErrorCodes::BAD_ARGUMENTS); -} - -void renamePath(const String & old_path, const String & new_path, Poco::Logger * log, bool must_success) -{ - if (auto file = Poco::File{old_path}; file.exists()) - { - file.renameTo(new_path); - } - else - { - std::string err_msg = fmt::format(R"(Path "{}" is missing.)", old_path); - if (must_success) - throw Exception(err_msg); - else - LOG_WARNING(log, err_msg); - } -} - -void writeTableDefinitionToFile( - const FileProviderPtr & file_provider, - const String & table_meta_path, - const ASTPtr & query, - bool fsync_metadata) -{ - String table_meta_tmp_path = table_meta_path + ".tmp"; - { - String statement = getTableDefinitionFromCreateQuery(query); - - /// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown. - WriteBufferFromFileProvider out(file_provider, table_meta_tmp_path, EncryptionPath(table_meta_tmp_path, ""), true, nullptr, statement.size(), O_WRONLY | O_CREAT | O_EXCL); - writeString(statement, out); - out.next(); - if (fsync_metadata) - out.sync(); - out.close(); - } - file_provider->renameFile( - table_meta_tmp_path, - EncryptionPath(table_meta_tmp_path, ""), - table_meta_path, - EncryptionPath(table_meta_path, ""), - true); -} - -void writeDatabaseDefinitionToFile( - const FileProviderPtr & file_provider, - const String & database_meta_path, - const ASTPtr & query, - bool fsync_metadata) -{ - String db_meta_tmp_path = database_meta_path + ".tmp"; - { - String statement = getDatabaseDefinitionFromCreateQuery(query); - - /// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown. - WriteBufferFromFileProvider out(file_provider, db_meta_tmp_path, EncryptionPath(db_meta_tmp_path, ""), true, nullptr, statement.size(), O_WRONLY | O_CREAT | O_EXCL); - writeString(statement, out); - out.next(); - if (fsync_metadata) - out.sync(); - out.close(); - } - file_provider->renameFile( - db_meta_tmp_path, - EncryptionPath(db_meta_tmp_path, ""), - database_meta_path, - EncryptionPath(database_meta_path, ""), - true); -} - -ASTPtr parseCreateDatabaseAST(const String & statement) -{ - ParserCreateQuery parser; - const char * pos = statement.data(); - std::string error_msg; - auto ast = tryParseQuery(parser, - pos, - pos + statement.size(), - error_msg, - /*hilite=*/false, - String("in ") + __PRETTY_FUNCTION__, - /*allow_multi_statements=*/false, - 0); - if (!ast) - throw Exception(error_msg, ErrorCodes::SYNTAX_ERROR); - return ast; -} - -// By default, only remove directory if it is empy -void tryRemoveDirectory(const String & directory, Poco::Logger * log, bool recursive = false) -{ - if (auto dir = Poco::File(directory); dir.exists() && dir.isDirectory()) - { - try - { - dir.remove(/*recursive=*/recursive); - } - catch (Poco::DirectoryNotEmptyException &) - { - // just ignore and keep that directory if it is not empty - LOG_WARNING(log, "Can not remove directory: {}, it is not empty", directory); - } - } -} - -// This function will tidy up path and compare if them are the same one. -// For example "/tmp/data/a.sql" is equal to "/tmp//data//a.sql" -inline bool isSamePath(const String & lhs, const String & rhs) -{ - return Poco::Path{lhs}.absolute().toString() == Poco::Path{rhs}.absolute().toString(); -} - -} // namespace - - -// ================================================ -// TableDiskInfo -// ================================================ - -String IDAsPathUpgrader::TableDiskInfo::name() const -{ - // Name in table_info may not be updated, use the name in `ATTACH TABLE ...`. - // The name in table_info will be updated in later schema sync. - return old_name; -} -String IDAsPathUpgrader::TableDiskInfo::newName() const -{ - return mapper->mapTableName(*tidb_table_info); -} -const TiDB::TableInfo & IDAsPathUpgrader::TableDiskInfo::getInfo() const -{ - return *tidb_table_info; -} - -// "metadata/${db_name}/${tbl_name}.sql" -String IDAsPathUpgrader::TableDiskInfo::getMetaFilePath(const String & root_path, const DatabaseDiskInfo & db) const -{ - return db.getMetaDirectory(root_path) + escapeForFileName(name()) + ".sql"; -} -// "data/${db_name}/${tbl_name}/" -String IDAsPathUpgrader::TableDiskInfo::getDataDirectory( - const String & root_path, - const DatabaseDiskInfo & db, - bool escape_db, - bool escape_tbl) const -{ - String res = db.getDataDirectory(root_path, escape_db); - if (escape_tbl) - res += escapeForFileName(name()); - else - res += name(); - return res + "/"; -} -// "extra_data/${db_name}/${tbl_name}/" -String IDAsPathUpgrader::TableDiskInfo::getExtraDirectory( - const String & root_path, - const DatabaseDiskInfo & db, - bool escape_db, - bool escape_tbl) const -{ - String res = db.getExtraDirectory(root_path, escape_db); - if (escape_tbl) - res += escapeForFileName(name()); - else - res += name(); - return res + "/"; -} - -// "metadata/db_${db_id}/t_${id}.sql" -String IDAsPathUpgrader::TableDiskInfo::getNewMetaFilePath(const String & root_path, const DatabaseDiskInfo & db) const -{ - return db.getNewMetaDirectory(root_path) + escapeForFileName(newName()) + ".sql"; -} -// "data/t_${id}/" -String IDAsPathUpgrader::TableDiskInfo::getNewDataDirectory(const String & root_path, const DatabaseDiskInfo & db) const -{ - return db.getNewDataDirectory(root_path) + escapeForFileName(newName()) + "/"; -} -// "extra_data/t_${id}" -String IDAsPathUpgrader::TableDiskInfo::getNewExtraDirectory(const String & root_path, const DatabaseDiskInfo & db) const -{ - return db.getNewExtraDirectory(root_path) + escapeForFileName(newName()) + "/"; -} - -// ================================================ -// DatabaseDiskInfo -// ================================================ - -void IDAsPathUpgrader::DatabaseDiskInfo::setDBInfo(TiDB::DBInfoPtr info_) -{ - tidb_db_info = info_; -} - -const TiDB::DBInfo & IDAsPathUpgrader::DatabaseDiskInfo::getInfo() const -{ - if (!hasValidTiDBInfo()) - throw Exception("Try to get database info of not inited database: " + name); - return *tidb_db_info; -} - -String IDAsPathUpgrader::DatabaseDiskInfo::newName() const -{ - return mapper->mapDatabaseName(getInfo()); -} - -String IDAsPathUpgrader::DatabaseDiskInfo::getTiDBSerializeInfo() const -{ - if (!hasValidTiDBInfo()) - throw Exception("Try to serialize database info of not inited database: " + name); - return tidb_db_info->serialize(); -} - -// "metadata/${db_name}.sql" -String IDAsPathUpgrader::DatabaseDiskInfo::doGetMetaFilePath(const String & root_path, bool tmp) const -{ - String meta_dir = doGetMetaDirectory(root_path, tmp); - return (endsWith(meta_dir, "/") ? meta_dir.substr(0, meta_dir.size() - 1) : meta_dir) + ".sql"; -} -// "metadata/${db_name}/" -String IDAsPathUpgrader::DatabaseDiskInfo::doGetMetaDirectory(const String & root_path, bool tmp) const -{ - return root_path + (endsWith(root_path, "/") ? "" : "/") + "metadata/" + escapeForFileName(name + (tmp ? TMP_SUFFIX : "")) + "/"; -} -// "data/${db_name}/" -String IDAsPathUpgrader::DatabaseDiskInfo::doGetDataDirectory(const String & root_path, bool escape, bool tmp) const -{ - // Old data path don't do escape for path - if (escape) - return root_path + (endsWith(root_path, "/") ? "" : "/") + "data/" + escapeForFileName(name + (tmp ? TMP_SUFFIX : "")) + "/"; - else - { - // Old extra data path (in PathPool) don't escape for path. - return root_path + (endsWith(root_path, "/") ? "" : "/") + "data/" + name + (tmp ? TMP_SUFFIX : "") + "/"; - } -} -// "extra_data/${db_name}/" -String IDAsPathUpgrader::DatabaseDiskInfo::doGetExtraDirectory(const String & extra_root, bool escape, bool tmp) const -{ - if (escape) - return extra_root + (endsWith(extra_root, "/") ? "" : "/") + escapeForFileName(name + (tmp ? TMP_SUFFIX : "")) + "/"; - else - { - // Old extra data path (in PathPool) don't escape for path. - return extra_root + (endsWith(extra_root, "/") ? "" : "/") + name + (tmp ? TMP_SUFFIX : "") + "/"; - } -} - -// "metadata/db_${id}.sql" -String IDAsPathUpgrader::DatabaseDiskInfo::getNewMetaFilePath(const String & root_path) const -{ - String meta_dir = getNewMetaDirectory(root_path); - return (endsWith(meta_dir, "/") ? meta_dir.substr(0, meta_dir.size() - 1) : meta_dir) + ".sql"; -} -// "metadata/db_${id}/" -String IDAsPathUpgrader::DatabaseDiskInfo::getNewMetaDirectory(const String & root_path) const -{ - return root_path + (endsWith(root_path, "/") ? "" : "/") + "/metadata/" + escapeForFileName(newName()) + "/"; -} -// "data/" -String IDAsPathUpgrader::DatabaseDiskInfo::getNewDataDirectory(const String & root_path) -{ - return root_path + "/data/"; -} -// "extra_data/" -String IDAsPathUpgrader::DatabaseDiskInfo::getNewExtraDirectory(const String & extra_root) -{ - return extra_root + "/"; -} - - -void IDAsPathUpgrader::DatabaseDiskInfo::renameToTmpDirectories(const Context & ctx, Poco::Logger * log) -{ - if (moved_to_tmp) - return; - - auto root_path = ctx.getPath(); - // Rename database meta file if exist - renamePath(doGetMetaFilePath(root_path, false), doGetMetaFilePath(root_path, true), log, false); - // Rename database meta dir - renamePath(doGetMetaDirectory(root_path, false), doGetMetaDirectory(root_path, true), log, true); - - // Rename database data dir - renamePath( // - doGetDataDirectory(root_path, /*escape*/ true, /*tmp*/ false), - doGetDataDirectory(root_path, /*escape*/ true, /*tmp*/ true), - log, - true); - - // Rename database data dir for multi-paths - auto root_pool = ctx.getPathPool(); - for (const auto & extra_path : root_pool.listPaths()) - renamePath( // - doGetExtraDirectory(extra_path, /*escape*/ true, /*tmp*/ false), // - doGetExtraDirectory(extra_path, /*escape*/ true, /*tmp*/ true), - log, - false); - - moved_to_tmp = true; -} - - -// ================================================ -// IDAsPathUpgrader -// ================================================ - -IDAsPathUpgrader::IDAsPathUpgrader(Context & global_ctx_, bool is_mock_, std::unordered_set reserved_databases_) - : global_context(global_ctx_) - , root_path{global_context.getPath()} - , is_mock(is_mock_) - , mapper(is_mock ? std::make_shared() // - : std::make_shared()) - , reserved_databases{std::move(reserved_databases_)} - , log{&Poco::Logger::get("IDAsPathUpgrader")} -{} - -bool IDAsPathUpgrader::needUpgrade() -{ - const auto metadata_path = global_context.getPath() + "/metadata"; - - // For old version, we have database directories and its `.sql` file - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator it(metadata_path); it != dir_end; ++it) - { - if (!it->isDirectory()) - continue; - - /// For '.svn', '.gitignore' directory and similar. - if (it.name().at(0) == '.') - continue; - - if (it.name() == SYSTEM_DATABASE) - continue; - - String db_name = unescapeForFileName(it.name()); - databases.emplace(db_name, DatabaseDiskInfo{db_name, mapper}); - } - - bool has_old_db_engine = false; - for (auto && [db_name, db_info] : databases) - { - (void)db_name; - const String database_metadata_file = db_info.getMetaFilePath(root_path); - auto engine = getDatabaseEngine(global_context.getFileProvider(), database_metadata_file); - db_info.engine = engine->name; - if (db_info.engine != "TiFlash") - { - has_old_db_engine = true; - LOG_INFO(log, "Find old style of database engine, doing upgrade [path={}] [engine={}]", database_metadata_file, db_info.engine); - } - } - - return has_old_db_engine; -} - -std::vector IDAsPathUpgrader::fetchInfosFromTiDB() const -{ - // Fetch DBs info from TiDB/TiKV - // Note: Not get table info from TiDB, just rename according to TableID in persisted TableInfo - for (size_t i = 0; i < 60; i++) // retry for 3 mins - { - try - { - auto schema_syncer = global_context.getTMTContext().getSchemaSyncer(); - return schema_syncer->fetchAllDBs(); - } - catch (Poco::Exception & e) - { - const int wait_seconds = 3; - LOG_ERROR( - log, - "Upgrade failed because fetch schema error: {}\nWe will sleep for {} seconds and try again.", - e.displayText(), - wait_seconds); - ::sleep(wait_seconds); - } - } - throw Exception("Upgrade failed because fetch schema error."); -} - -static void dropAbsentDatabase( - Context & context, - const String & db_name, - const IDAsPathUpgrader::DatabaseDiskInfo & db_info, - Poco::Logger * log) -{ - if (db_info.hasValidTiDBInfo()) - throw Exception("Invalid call for dropAbsentDatabase for database " + db_name + " with info: " + db_info.getTiDBSerializeInfo()); - - /// tryRemoveDirectory with recursive=true to clean up - - const auto root_path = context.getPath(); - // Remove old metadata dir - const String old_meta_dir = db_info.getMetaDirectory(root_path); - tryRemoveDirectory(old_meta_dir, log, true); - // Remove old metadata file - const String old_meta_file = db_info.getMetaFilePath(root_path); - if (auto file = Poco::File(old_meta_file); file.exists()) - file.remove(); - else - LOG_WARNING(log, "Can not remove database meta file: {}", old_meta_file); - // Remove old data dir - const String old_data_dir = db_info.getDataDirectory(root_path); - tryRemoveDirectory(old_data_dir, log, true); - // not escaped dir created by old PathPool - const String old_data_dir_not_escaped = db_info.getDataDirectory(root_path, false); - tryRemoveDirectory(old_data_dir_not_escaped, log, true); - - const auto & data_extra_paths = context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - tryRemoveDirectory(db_info.getExtraDirectory(extra_root_path), log, true); - tryRemoveDirectory(db_info.getExtraDirectory(extra_root_path, false), log, true); - } -} - -void IDAsPathUpgrader::linkDatabaseTableInfos(const std::vector & all_databases) -{ - for (const auto & db : all_databases) - { - if (auto iter = databases.find(db->name); iter != databases.end()) - { - iter->second.setDBInfo(db); - } - } - - // list all table in old style. - for (auto iter = databases.begin(); iter != databases.end(); /*empty*/) - { - const auto & db_name = iter->first; - auto & db_info = iter->second; - if (!db_info.hasValidTiDBInfo()) - { - // If we can't find it in TiDB, maybe it already dropped. - if (reserved_databases.count(db_name) > 0) - { - // For mock test or develop environment, we may reserve some database - // for convenience. Keep them as what they are. Print warnings and - // ignore it in later upgrade. - LOG_WARNING(log, "Database {} is reserved, ignored in upgrade.", db_name); - } - else - { - // If we keep them as "Ordinary", when user actually create database with - // same name, next time TiFlash restart and will try to do "upgrade" on - // those legacy data, and it will mess everything up. - // Drop them. - dropAbsentDatabase(global_context, db_name, db_info, log); - } - iter = databases.erase(iter); - continue; - } - - if (db_info.engine == "TiFlash") - { - ++iter; - continue; - } - - const String db_meta_dir = db_info.getMetaDirectory(root_path); - std::vector file_names = DatabaseLoading::listSQLFilenames(db_meta_dir, log); - for (const auto & table_filename : file_names) - { - String table_meta_file = db_meta_dir + "/" + table_filename; - // Name in table_info may not be updated, use the name in `ATTACH TABLE ...`. - auto [old_name, table_info] = getTableInfo(global_context.getFileProvider(), table_meta_file); - db_info.tables.emplace_back( // - TableDiskInfo{old_name, std::make_shared(table_info), mapper}); - } - ++iter; - } -} - -void IDAsPathUpgrader::fixNotEscapedDirectories() -{ - for (const auto & [db_name, db_info] : databases) - { - const auto db_name_escaped = escapeForFileName(db_name); - - // database's meta file, meta dir (created by old DatabaseOrdinary) is escaped. - // only need to create data path - if (db_name != db_name_escaped) - { - LOG_INFO(log, "database `{}` fixing name escape to `{}`", db_name, db_name_escaped); - // Create directory for escaped database - auto escaped_db_data_dir = db_info.getDataDirectory(root_path, /*escape=*/true); - if (Poco::File dir(escaped_db_data_dir); !dir.exists()) - dir.createDirectory(); - - const auto & data_extra_paths = global_context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - auto escaped_extra_dir = db_info.getExtraDirectory(extra_root_path, /*escape=*/true); - if (Poco::File dir(escaped_extra_dir); !dir.exists()) - dir.createDirectory(); - } - } - - /// Fix not escaped name for table - for (const auto & table : db_info.tables) - { - const auto table_name_escaped = escapeForFileName(table.name()); - if (db_name_escaped == db_name && table_name_escaped == table.name()) - continue; - - LOG_INFO( - log, - "table `{}`.`{}` fixing name escape to `{}`.`{}`", - db_name, - table.name(), - db_name_escaped, - table_name_escaped); - // Table's metadata don't need to fix. - - // Fix data path. It was create by DatabaseOrdinary and StorageDeltaMerge, - // database name is escaped but table name not. - auto not_escaped_path = table.getDataDirectory(root_path, db_info, /*escape_db*/ true, /*escape_tbl*/ false); - auto escaped_path = table.getDataDirectory(root_path, db_info, /*escape_db*/ true, /*escape_tbl*/ true); - if (auto file = Poco::File{not_escaped_path}; file.exists()) - { - if (auto escaped_dir = Poco::File{escaped_path}; !escaped_dir.exists()) - escaped_dir.createDirectory(); - renamePath(not_escaped_path + "/meta", escaped_path + "/meta", log, true); - renamePath(not_escaped_path + "/data", escaped_path + "/data", log, true); - renamePath(not_escaped_path + "/log", escaped_path + "/log", log, true); - // For the cases that database's name did not need to be escaped but table's name did. - renamePath(not_escaped_path + "/stable", escaped_path + "/stable", log, false); - tryRemoveDirectory(not_escaped_path, log); - } - auto db_tbl_not_escaped_path = not_escaped_path; - if (db_name != db_name_escaped) - { - // For the cases that database's name need to be escaped. - // Stable dir was created by old PathPool, database name and table name were not escaped. - db_tbl_not_escaped_path = table.getDataDirectory(root_path, db_info, false, false); - auto not_escaped_stable = db_tbl_not_escaped_path + "/stable"; - auto escaped_stable = table.getDataDirectory(root_path, db_info, true, true) + "/stable"; - if (auto file = Poco::File{not_escaped_stable}; file.exists()) - renamePath(not_escaped_stable, escaped_stable, log, true); - } - - // Fix extra path. - const auto & data_extra_paths = global_context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - // It was created by old PathPool, both database name and table name are not escaped. - auto not_escaped_extra_path = table.getExtraDirectory(extra_root_path, db_info, /*escape_db*/ false, /*escape_tbl*/ false); - if (isSamePath(not_escaped_extra_path, db_tbl_not_escaped_path)) - continue; - auto escaped_extra_path = table.getExtraDirectory(extra_root_path, db_info, /*escape_db*/ true, /*escape_tbl*/ true); - renamePath(not_escaped_extra_path, escaped_extra_path, log, false); - } - LOG_INFO( - log, - "table `{}`.`{}` fixing name escape to `{}`.`{}` done.", - db_name, - table.name(), - db_name_escaped, - table_name_escaped); - } - - if (db_name != db_name_escaped) - { - // clean not escaped database dir created by old PathPool - const String not_escaped_data_dir = db_info.getDataDirectory(root_path, /*escape*/ false); - tryRemoveDirectory(not_escaped_data_dir, log, true); - const auto & data_extra_paths = global_context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - auto not_escaped_extra_data_dir = db_info.getExtraDirectory(extra_root_path, /*escape*/ false); - if (isSamePath(not_escaped_data_dir, not_escaped_extra_data_dir)) - continue; - tryRemoveDirectory(not_escaped_extra_data_dir, log); - } - } - LOG_INFO(log, "database `{}` fixing name escape to `{}` done.", db_name, db_name_escaped); - } -} - -void IDAsPathUpgrader::resolveConflictDirectories() -{ - std::unordered_set conflict_databases; - for (const auto & [db_name, db_info] : databases) - { - // In theory, user can create database naming "t_xx" and there is cyclic renaming between table and database. - // First detect if there is any database may have cyclic rename with table. - for (const auto & table : db_info.tables) - { - const auto new_tbl_name = table.newName(); - if (auto iter = databases.find(new_tbl_name); iter != databases.end()) - { - conflict_databases.insert(iter->first); - LOG_INFO( - log, - "Detect cyclic renaming between table `{}`.`{}`(new name:{}) and database `{}`", - db_name, - table.name(), - new_tbl_name, - iter->first); - } - } - - // In theory, user can create two database naming "db_xx" and there is cyclic renaming. - // We need to break that cyclic. - const auto new_database_name = db_info.newName(); - if (auto iter = databases.find(new_database_name); iter != databases.end()) - { - conflict_databases.insert(iter->first); - LOG_INFO( - log, - "Detect cyclic renaming between database `{}`(new name:{}) and database `{}`", - db_name, - new_database_name, - iter->first); - } - } - LOG_INFO(log, "Detect {} cyclic renaming", conflict_databases.size()); - for (const auto & db_name : conflict_databases) - { - auto iter = databases.find(db_name); - auto & db_info = iter->second; - LOG_INFO(log, "Move {} to tmp directories..", db_name); - db_info.renameToTmpDirectories(global_context, log); - } -} - -void IDAsPathUpgrader::doRename() -{ - for (const auto & [db_name, db_info] : databases) - { - renameDatabase(db_name, db_info); - } -} - -void IDAsPathUpgrader::renameDatabase(const String & db_name, const DatabaseDiskInfo & db_info) -{ - const auto mapped_db_name = db_info.newName(); - - { - // Create directory for target database - auto new_db_meta_dir = db_info.getNewMetaDirectory(root_path); - Poco::File(new_db_meta_dir).createDirectory(); - } - - // Rename all tables of this database - for (const auto & table : db_info.tables) - { - renameTable(db_name, db_info, mapped_db_name, table); - } - - // Then rename database - LOG_INFO(log, "database `{}` to `{}` renaming", db_name, mapped_db_name); - { - // Recreate metadata file for database - const String new_meta_file = db_info.getNewMetaFilePath(root_path); - const String statement = "ATTACH DATABASE `" + mapped_db_name + "` ENGINE=TiFlash('" + db_info.getTiDBSerializeInfo() + "', 1)\n"; - auto ast = parseCreateDatabaseAST(statement); - const auto & settings = global_context.getSettingsRef(); - writeDatabaseDefinitionToFile(global_context.getFileProvider(), new_meta_file, ast, settings.fsync_metadata); - } - - { - // Remove old metadata dir - const String old_meta_dir = db_info.getMetaDirectory(root_path); - tryRemoveDirectory(old_meta_dir, log); - // Remove old metadata file - const String old_meta_file = db_info.getMetaFilePath(root_path); - if (auto file = Poco::File(old_meta_file); file.exists()) - file.remove(); - else - LOG_WARNING(log, "Can not remove database meta file: {}", old_meta_file); - // Remove old data dir - const String old_data_dir = db_info.getDataDirectory(root_path); - tryRemoveDirectory(old_data_dir, log); - const auto & data_extra_paths = global_context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - tryRemoveDirectory(db_info.getExtraDirectory(extra_root_path), log); - } - } - LOG_INFO(log, "database `{}` to `{}` rename done.", db_name, mapped_db_name); -} - -void IDAsPathUpgrader::renameTable( - const String & db_name, - const DatabaseDiskInfo & db_info, - const String & mapped_db_name, - const TableDiskInfo & table) -{ - const auto mapped_table_name = table.newName(); - LOG_INFO( - log, - "table `{}`.`{}` to `{}`.`{}` renaming", - db_name, - table.name(), - mapped_db_name, - mapped_table_name); - - String old_tbl_data_path; - { - // Former data path use ${path}/data/${database}/${table}/ as data path. - // Rename it to ${path}/data/${mapped_table_name}. - old_tbl_data_path = table.getDataDirectory(root_path, db_info); - renamePath(old_tbl_data_path, table.getNewDataDirectory(root_path, db_info), log, true); - } - - { - // Rename data path for multi disk - auto data_extra_paths = global_context.getPathPool(); - for (const auto & extra_root_path : data_extra_paths.listPaths()) - { - auto old_tbl_extra_data_path = table.getExtraDirectory(extra_root_path, db_info); - if (isSamePath(old_tbl_extra_data_path, old_tbl_data_path)) - continue; - renamePath(old_tbl_extra_data_path, table.getNewExtraDirectory(extra_root_path, db_info), log, false); - } - } - - // Recreate metadata file - { - auto old_tbl_meta_file = table.getMetaFilePath(root_path, db_info); - auto ast = DatabaseLoading::getQueryFromMetadata(global_context, old_tbl_meta_file, /*throw_on_error=*/true); - if (!ast) - throw Exception("There is no metadata file for table " + table.name() + ", expected file: " + old_tbl_meta_file, - ErrorCodes::FILE_DOESNT_EXIST); - - ASTCreateQuery & ast_create_query = typeid_cast(*ast); - ast_create_query.table = mapped_table_name; - ASTStorage * storage_ast = ast_create_query.storage; - TiDB::TableInfo table_info = table.getInfo(); // get a copy - if (table_info.is_partition_table) - { - LOG_INFO( - log, - "partition table `{}`.`{}` to `{}`.`{}` update table info", - db_name, - table.name(), - mapped_db_name, - mapped_table_name); - // Old partition name is "${table_name}_${physical_id}" while new name is "t_${physical_id}" - // If it is a partition table, we need to update TiDB::TableInfo::name - do - { - if (!storage_ast || !storage_ast->engine) - break; - auto * args = typeid_cast(storage_ast->engine->arguments.get()); - if (!args) - break; - - table_info.name = mapper->mapPartitionName(table_info); - std::shared_ptr literal = std::make_shared(Field(table_info.serialize())); - if (args->children.size() == 1) - args->children.emplace_back(literal); - else if (args->children.size() >= 2) - args->children.at(1) = literal; - } while (false); - } - - const String new_tbl_meta_file = table.getNewMetaFilePath(root_path, db_info); - const auto & settings = global_context.getSettingsRef(); - writeTableDefinitionToFile(global_context.getFileProvider(), new_tbl_meta_file, ast, settings.fsync_metadata); - - // Remove old metadata file - if (auto file = Poco::File(old_tbl_meta_file); file.exists()) - file.remove(); - } - - LOG_INFO( - log, - "table `{}`.`{}` to `{}`.`{}` rename done.", - db_name, - table.name(), - mapped_db_name, - mapped_table_name); -} - -void IDAsPathUpgrader::doUpgrade() -{ - auto all_databases = fetchInfosFromTiDB(); - linkDatabaseTableInfos(all_databases); - fixNotEscapedDirectories(); - // Check if destination db / tbl file exists and resolve conflict - resolveConflictDirectories(); - // Rename - doRename(); -} - -} // namespace DB diff --git a/dbms/src/Interpreters/IDAsPathUpgrader.h b/dbms/src/Interpreters/IDAsPathUpgrader.h deleted file mode 100644 index 38dc37536aa..00000000000 --- a/dbms/src/Interpreters/IDAsPathUpgrader.h +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include -#include -#include - -namespace Poco -{ -class Logger; -} - -namespace TiDB -{ -struct TableInfo; -using TableInfoPtr = std::shared_ptr; - -struct DBInfo; -using DBInfoPtr = std::shared_ptr; -} // namespace TiDB - -namespace DB -{ -class Context; -class PathPool; -struct SchemaNameMapper; - -class IDAsPathUpgrader -{ -public: - struct DatabaseDiskInfo; - - struct TableDiskInfo - { - public: - String name() const; - String newName() const; - - TableDiskInfo(String old_name_, TiDB::TableInfoPtr info_, std::shared_ptr mapper_) - : old_name(old_name_) - , tidb_table_info(std::move(info_)) - , mapper(std::move(mapper_)) - {} - - private: - String old_name; - TiDB::TableInfoPtr tidb_table_info; - std::shared_ptr mapper; - - public: - // "metadata/${db_name}/${tbl_name}.sql" - String getMetaFilePath(const String & root_path, const DatabaseDiskInfo & db) const; - // "data/${db_name}/${tbl_name}/" - String getDataDirectory(const String & root_path, const DatabaseDiskInfo & db, bool escape_db = true, bool escape_tbl = true) const; - // "extra_data/${db_name}/${tbl_name}/" - String getExtraDirectory( - const String & root_path, - const DatabaseDiskInfo & db, - bool escape_db = true, - bool escape_tbl = true) const; - - // "metadata/db_${db_id}/t_${id}.sql" - String getNewMetaFilePath(const String & root_path, const DatabaseDiskInfo & db) const; - // "data/t_${id}/" - String getNewDataDirectory(const String & root_path, const DatabaseDiskInfo & db) const; - // "extra_data/t_${id}" - String getNewExtraDirectory(const String & root_path, const DatabaseDiskInfo & db) const; - - const TiDB::TableInfo & getInfo() const; - }; - - struct DatabaseDiskInfo - { - public: - static constexpr auto TMP_SUFFIX = "_flash_upgrade"; - - String engine; - std::vector tables; - - private: - String name; - std::shared_ptr mapper; - bool moved_to_tmp = false; - TiDB::DBInfoPtr tidb_db_info = nullptr; - - const TiDB::DBInfo & getInfo() const; - - public: - DatabaseDiskInfo(String name_, std::shared_ptr mapper_) - : name(std::move(name_)) - , mapper(std::move(mapper_)) - {} - - void setDBInfo(TiDB::DBInfoPtr info_); - - bool hasValidTiDBInfo() const { return tidb_db_info != nullptr; } - - String newName() const; - - String getTiDBSerializeInfo() const; - - // "metadata/${db_name}.sql" - String getMetaFilePath(const String & root_path) const { return doGetMetaFilePath(root_path, moved_to_tmp); } - // "metadata/${db_name}/" - String getMetaDirectory(const String & root_path) const { return doGetMetaDirectory(root_path, moved_to_tmp); } - // "data/${db_name}/" - String getDataDirectory(const String & root_path, bool escape = true) const - { - return doGetDataDirectory(root_path, escape, moved_to_tmp); - } - // "extra_data/${db_name}/". db_name is not escaped. - String getExtraDirectory(const String & extra_root, bool escape = true) const - { - return doGetExtraDirectory(extra_root, escape, moved_to_tmp); - } - - void renameToTmpDirectories(const Context & ctx, Poco::Logger * log); - - // "metadata/db_${id}.sql" - String getNewMetaFilePath(const String & root_path) const; - // "metadata/db_${id}/" - String getNewMetaDirectory(const String & root_path) const; - // "data/" - static String getNewDataDirectory(const String & root_path); - // "extra_data/" - static String getNewExtraDirectory(const String & extra_root); - - private: - // "metadata/${db_name}.sql" - String doGetMetaFilePath(const String & root_path, bool tmp) const; - // "metadata/${db_name}/" - String doGetMetaDirectory(const String & root_path, bool tmp) const; - // "data/${db_name}/" - String doGetDataDirectory(const String & root_path, bool escape, bool tmp) const; - // "extra_data/${db_name}/" - String doGetExtraDirectory(const String & extra_root, bool escape, bool tmp) const; - }; - -public: - /// Upgrader - // If some database can not find in TiDB, they will be dropped - // if theirs name is not in reserved_databases - IDAsPathUpgrader(Context & global_ctx_, bool is_mock_, std::unordered_set reserved_databases_); - - bool needUpgrade(); - - void doUpgrade(); - -private: - std::vector fetchInfosFromTiDB() const; - - void linkDatabaseTableInfos(const std::vector & all_databases); - - // Some path created by old PathPool, its database / table name is not escaped, - // normalized those names first. - void fixNotEscapedDirectories(); - - void resolveConflictDirectories(); - - void doRename(); - - void renameDatabase(const String & db_name, const DatabaseDiskInfo & db_info); - - void renameTable( - const String & db_name, - const DatabaseDiskInfo & db_info, - const String & mapped_db_name, - const TableDiskInfo & table_info); - -private: - Context & global_context; - - const String root_path; - - std::map databases; - - const bool is_mock = false; - - std::shared_ptr mapper; - - const std::unordered_set reserved_databases; - - Poco::Logger * log; -}; - -} // namespace DB diff --git a/dbms/src/Interpreters/sortBlock.h b/dbms/src/Interpreters/sortBlock.h index 80098066c72..a768920dab0 100644 --- a/dbms/src/Interpreters/sortBlock.h +++ b/dbms/src/Interpreters/sortBlock.h @@ -24,7 +24,7 @@ namespace DB void sortBlock(Block & block, const SortDescription & description, size_t limit = 0); -/** Used only in StorageMergeTree to sort the data with INSERT. +/** Used only in StorageDeltaMerge to sort the data with INSERT. * Sorting is stable. This is important for keeping the order of rows in the CollapsingMergeTree engine * - because based on the order of rows it is determined whether to delete or leave groups of rows when collapsing. * Collations are not supported. Partial sorting is not supported. diff --git a/dbms/src/Interpreters/tests/CMakeLists.txt b/dbms/src/Interpreters/tests/CMakeLists.txt deleted file mode 100644 index f310837b3f0..00000000000 --- a/dbms/src/Interpreters/tests/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_executable (create_query create_query.cpp) -target_link_libraries (create_query dbms) - -add_executable (select_query select_query.cpp) -target_link_libraries (select_query clickhouse_storages_system dbms) - -add_executable (aggregate aggregate.cpp) -target_link_libraries (aggregate dbms) - -add_executable (hash_map hash_map.cpp) -target_include_directories (hash_map BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map dbms) - -add_executable (hash_map3 hash_map3.cpp) -target_link_libraries (hash_map3 dbms ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES}) - -add_executable (hash_map_string hash_map_string.cpp) -target_include_directories (hash_map_string BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_string dbms) - -add_executable (hash_map_string_2 hash_map_string_2.cpp) -target_link_libraries (hash_map_string_2 dbms) - -add_executable (hash_map_string_3 hash_map_string_3.cpp) -target_link_libraries (hash_map_string_3 dbms ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES}) -target_include_directories (hash_map_string_3 BEFORE PRIVATE ${TiFlash_SOURCE_DIR}/contrib/libfarmhash) -target_include_directories (hash_map_string_3 BEFORE PRIVATE ${TiFlash_SOURCE_DIR}/contrib/libmetrohash/src) - -add_executable (hash_map_string_small hash_map_string_small.cpp) -target_include_directories (hash_map_string_small BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_string_small dbms) - -add_executable (two_level_hash_map two_level_hash_map.cpp) -target_include_directories (two_level_hash_map BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) -target_link_libraries (two_level_hash_map dbms) - -add_executable (logical_expressions_optimizer logical_expressions_optimizer.cpp) -target_link_libraries (logical_expressions_optimizer dbms) - -add_executable (users users.cpp) -target_link_libraries (users dbms ${Boost_FILESYSTEM_LIBRARY} dbms) diff --git a/dbms/src/Interpreters/tests/aggregate.cpp b/dbms/src/Interpreters/tests/aggregate.cpp deleted file mode 100644 index 27491f4afbb..00000000000 --- a/dbms/src/Interpreters/tests/aggregate.cpp +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - - -int main(int argc, char ** argv) -{ - using namespace DB; - - try - { - size_t n = argc == 2 ? atoi(argv[1]) : 10; - - Block block; - - { - ColumnWithTypeAndName column; - column.name = "x"; - column.type = std::make_shared(); - auto col = ColumnInt16::create(); - auto & vec_x = col->getData(); - - vec_x.resize(n); - for (size_t i = 0; i < n; ++i) - vec_x[i] = i % 9; - - column.column = std::move(col); - block.insert(column); - } - - const char * strings[] = {"abc", "def", "abcd", "defg", "ac"}; - - { - ColumnWithTypeAndName column; - column.name = "s1"; - column.type = std::make_shared(); - auto col = ColumnString::create(); - - for (size_t i = 0; i < n; ++i) - col->insert(std::string(strings[i % 5])); - - column.column = std::move(col); - block.insert(column); - } - - { - ColumnWithTypeAndName column; - column.name = "s2"; - column.type = std::make_shared(); - auto col = ColumnString::create(); - - for (size_t i = 0; i < n; ++i) - col->insert(std::string(strings[i % 3])); - - column.column = std::move(col); - block.insert(column); - } - - BlockInputStreamPtr stream = std::make_shared(block); - AggregatedDataVariants aggregated_data_variants; - - AggregateFunctionFactory factory; - - AggregateDescriptions aggregate_descriptions(1); - - DataTypes empty_list_of_types; - aggregate_descriptions[0].function = factory.get("count", empty_list_of_types); - - Aggregator::Params params(stream->getHeader(), {0, 1}, aggregate_descriptions, false); - - Aggregator aggregator(params); - - { - Stopwatch stopwatch; - stopwatch.start(); - - KeyManagerPtr key_manager = std::make_shared(false); - FileProviderPtr file_provider = std::make_shared(key_manager, false); - aggregator.execute(stream, aggregated_data_variants, file_provider); - - stopwatch.stop(); - std::cout << std::fixed << std::setprecision(2) - << "Elapsed " << stopwatch.elapsedSeconds() << " sec." - << ", " << n / stopwatch.elapsedSeconds() << " rows/sec." - << std::endl; - } - } - catch (const Exception & e) - { - std::cerr << e.displayText() << std::endl; - } - - return 0; -} \ No newline at end of file diff --git a/dbms/src/Interpreters/tests/create_query.cpp b/dbms/src/Interpreters/tests/create_query.cpp deleted file mode 100644 index 03f787e1547..00000000000 --- a/dbms/src/Interpreters/tests/create_query.cpp +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include -#include -#include - -#include - -#include -#include - - -using namespace DB; - -int main(int, char **) -try -{ - std::string input = "CREATE TABLE IF NOT EXISTS hits (\n" - "WatchID UInt64,\n" - "JavaEnable UInt8,\n" - "Title String,\n" - "EventTime DateTime,\n" - "CounterID UInt32,\n" - "ClientIP UInt32,\n" - "RegionID UInt32,\n" - "UniqID UInt64,\n" - "CounterClass UInt8,\n" - "OS UInt8,\n" - "UserAgent UInt8,\n" - "URL String,\n" - "Referer String,\n" - "ResolutionWidth UInt16,\n" - "ResolutionHeight UInt16,\n" - "ResolutionDepth UInt8,\n" - "FlashMajor UInt8,\n" - "FlashMinor UInt8,\n" - "FlashMinor2 String,\n" - "NetMajor UInt8,\n" - "NetMinor UInt8,\n" - "UserAgentMajor UInt16,\n" - "UserAgentMinor FixedString(2),\n" - "CookieEnable UInt8,\n" - "JavascriptEnable UInt8,\n" - "IsMobile UInt8,\n" - "MobilePhone UInt8,\n" - "MobilePhoneModel String,\n" - "Params String,\n" - "IPNetworkID UInt32,\n" - "TraficSourceID Int8,\n" - "SearchEngineID UInt16,\n" - "SearchPhrase String,\n" - "AdvEngineID UInt8,\n" - "IsArtifical UInt8,\n" - "WindowClientWidth UInt16,\n" - "WindowClientHeight UInt16,\n" - "ClientTimeZone Int16,\n" - "ClientEventTime DateTime,\n" - "SilverlightVersion1 UInt8,\n" - "SilverlightVersion2 UInt8,\n" - "SilverlightVersion3 UInt32,\n" - "SilverlightVersion4 UInt16,\n" - "PageCharset String,\n" - "CodeVersion UInt32,\n" - "IsLink UInt8,\n" - "IsDownload UInt8,\n" - "IsNotBounce UInt8,\n" - "FUniqID UInt64,\n" - "OriginalURL String,\n" - "HID UInt32,\n" - "IsOldCounter UInt8,\n" - "IsEvent UInt8,\n" - "IsParameter UInt8,\n" - "DontCountHits UInt8,\n" - "WithHash UInt8\n" - ") ENGINE = Log"; - - ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); - - Context context = Context::createGlobal(); - - context.setPath("./"); - auto database = std::make_shared("test", "./metadata/test/", context); - context.addDatabase("test", database); - database->loadTables(context, nullptr, false); - context.setCurrentDatabase("test"); - - InterpreterCreateQuery interpreter(ast, context); - interpreter.execute(); - - return 0; -} -catch (const Exception & e) -{ - std::cerr << e.what() << ", " << e.displayText() << std::endl - << std::endl - << "Stack trace:" << std::endl - << e.getStackTrace().toString(); - return 1; -} diff --git a/dbms/src/Interpreters/tests/gtest_id_as_path_upgrader.cpp b/dbms/src/Interpreters/tests/gtest_id_as_path_upgrader.cpp deleted file mode 100644 index 1ff1d8febb7..00000000000 --- a/dbms/src/Interpreters/tests/gtest_id_as_path_upgrader.cpp +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB::tests -{ - -class IDAsPathUpgrader_test : public ::testing::Test -{ -public: - void SetUp() override - { - try - { - registerStorages(); - } - catch (DB::Exception &) - { - // Maybe another test has already registed, ignore exception here. - } - } - - void TearDown() override - { - auto ctx = TiFlashTestEnv::getContext(); - auto databases = ctx.getDatabases(); - for (auto database : databases) - { - database.second->shutdown(); - ctx.detachDatabase(database.first); - } - } -}; - -TEST_F(IDAsPathUpgrader_test, ONCALL_1651) -try -{ - CHECK_TESTS_WITH_DATA_ENABLED; - - // prepare a "test" database for upgrader - MockTiDB::instance().newDataBase("test"); // id == 2 - - // Generated by running these SQL on cluster version v3.1.0 - // > create table test.aaa(pk int); - // > rename table test.aaa TO test.abc; - // > CREATE TABLE test.employees(id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, - // fname VARCHAR(25) NOT NULL, - // lname VARCHAR(25) NOT NULL, - // store_id INT NOT NULL, - // department_id INT NOT NULL) - // PARTITION BY RANGE(id)(PARTITION p0 VALUES LESS THAN(5), - // PARTITION p1 VALUES LESS THAN(10), - // PARTITION p2 VALUES LESS THAN(15), - // PARTITION p3 VALUES LESS THAN MAXVALUE); - // > RENAME TABLE test.employees TO test.emp; - // > RENAME TABLE test.emp TO test.emp_bak; - const auto test_path = TiFlashTestEnv::findTestDataPath("oncall-1651"); - auto ctx = TiFlashTestEnv::getContext(DB::Settings(), test_path); - - IDAsPathUpgrader upgrader(ctx, false, {}); - ASSERT_TRUE(upgrader.needUpgrade()); - upgrader.doUpgrade(); - - { - // After upgrade, next time we don't need it. - IDAsPathUpgrader checker_after_upgrade(ctx, false, {}); - ASSERT_FALSE(checker_after_upgrade.needUpgrade()); - } - - // load metadata should not throw any exception - loadMetadata(ctx); - - ASSERT_TRUE(ctx.isDatabaseExist("db_2")); // "test" - auto & storages = ctx.getTMTContext().getStorages(); - ASSERT_NE(storages.get(45), nullptr); // `test`.`abc` - ASSERT_NE(storages.get(48), nullptr); // `test`.`emp_bak` - ASSERT_NE(storages.get(49), nullptr); - ASSERT_NE(storages.get(50), nullptr); - ASSERT_NE(storages.get(51), nullptr); - ASSERT_NE(storages.get(52), nullptr); -} -CATCH - -TEST_F(IDAsPathUpgrader_test, FLASH_1136_4_0_0_rc) -try -{ - CHECK_TESTS_WITH_DATA_ENABLED; - - // prepare a "test" database for upgrader - MockTiDB::instance().newDataBase("test"); // id == 2 - MockTiDB::instance().newDataBase("test-db"); // id == 3 - - // Generated by running these SQL on cluster version v4.0.0-rc - // > create database `test-db`; - // > create table `test-tbl` (pk int); - // > cerate table `test`.`test-tbl` (pk int); - const auto test_path = TiFlashTestEnv::findTestDataPath("flash-1136"); - auto ctx = TiFlashTestEnv::getContext(DB::Settings(), test_path); - - IDAsPathUpgrader upgrader(ctx, false, {}); - ASSERT_TRUE(upgrader.needUpgrade()); - upgrader.doUpgrade(); - - { - // After upgrade, next time we don't need it. - IDAsPathUpgrader checker_after_upgrade(ctx, false, {}); - ASSERT_FALSE(checker_after_upgrade.needUpgrade()); - } - - // load metadata should not throw any exception - loadMetadata(ctx); - - ASSERT_TRUE(ctx.isDatabaseExist("db_2")); - ASSERT_TRUE(ctx.isDatabaseExist("db_3")); - auto & storages = ctx.getTMTContext().getStorages(); - ASSERT_NE(storages.get(66), nullptr); // `test-db`.`test-tbl` - ASSERT_NE(storages.get(666), nullptr); // `test`.`test-tbl` -} -CATCH - -TEST_F(IDAsPathUpgrader_test, FLASH_1136_3_1_0) -try -{ - CHECK_TESTS_WITH_DATA_ENABLED; - - // prepare a "test" database for upgrader - MockTiDB::instance().newDataBase("test"); // id == 2 - MockTiDB::instance().newDataBase("test-db"); // id == 3 - - // Generated by running these SQL on cluster version v4.0.0-rc - // > create database `test-db`; - // > create table `test-tbl` (pk int); - // > cerate table `test`.`test-tbl` (pk int); - const auto test_path = TiFlashTestEnv::findTestDataPath("flash-1136-v3.1.0"); - auto ctx = TiFlashTestEnv::getContext(DB::Settings(), test_path); - - IDAsPathUpgrader upgrader(ctx, false, {}); - ASSERT_TRUE(upgrader.needUpgrade()); - upgrader.doUpgrade(); - - { - // After upgrade, next time we don't need it. - IDAsPathUpgrader checker_after_upgrade(ctx, false, {}); - ASSERT_FALSE(checker_after_upgrade.needUpgrade()); - } - - // load metadata should not throw any exception - loadMetadata(ctx); - - ASSERT_TRUE(ctx.isDatabaseExist("db_2")); - ASSERT_TRUE(ctx.isDatabaseExist("db_3")); - auto & storages = ctx.getTMTContext().getStorages(); - ASSERT_NE(storages.get(66), nullptr); // `test-db`.`test-tbl` - ASSERT_NE(storages.get(666), nullptr); // `test`.`test-tbl` -} -CATCH - -TEST_F(IDAsPathUpgrader_test, ISSUE_941) -try -{ - CHECK_TESTS_WITH_DATA_ENABLED; - - // prepare a "test" database for upgrader - MockTiDB::instance().newDataBase("test"); // id == 2 - - // Generated by running these SQL on cluster version v4.0.0-rc - // > create database `test`; - // > create table `#hello-world` (pk int); - const auto test_path = TiFlashTestEnv::findTestDataPath("issue-941"); - auto ctx = TiFlashTestEnv::getContext(DB::Settings(), test_path); - - IDAsPathUpgrader upgrader(ctx, false, {}); - ASSERT_TRUE(upgrader.needUpgrade()); - upgrader.doUpgrade(); - - { - // After upgrade, next time we don't need it. - IDAsPathUpgrader checker_after_upgrade(ctx, false, {}); - ASSERT_FALSE(checker_after_upgrade.needUpgrade()); - } - - // load metadata should not throw any exception - loadMetadata(ctx); - - ASSERT_TRUE(ctx.isDatabaseExist("db_2")); - auto & storages = ctx.getTMTContext().getStorages(); - ASSERT_NE(storages.get(45), nullptr); // `test`.`#hello-world` -} -CATCH - -} // namespace DB::tests diff --git a/dbms/src/Interpreters/tests/hash_map.cpp b/dbms/src/Interpreters/tests/hash_map.cpp deleted file mode 100644 index d75f578367f..00000000000 --- a/dbms/src/Interpreters/tests/hash_map.cpp +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include - -#include -#include - -#include -/* -#define DBMS_HASH_MAP_COUNT_COLLISIONS -*/ -#include -#include -#include -#include -#include -#include -#include -#include - - -/** The test checks the speed of hash tables, simulating their use for aggregation. - * The first argument specifies the number of elements to be inserted. - * The second argument can be a number from 1 to 4 - the number of the data structure being tested. - * This is important, because if you run all the tests one by one, the results will be incorrect. - * (Due to the peculiarities of the work of the allocator, the first test takes advantage.) - * - * Depending on USE_AUTO_ARRAY, one of the structures is selected as the value. - * USE_AUTO_ARRAY = 0 - uses std::vector (hard-copy structure, sizeof = 24 bytes). - * USE_AUTO_ARRAY = 1 - uses AutoArray (a structure specially designed for such cases, sizeof = 8 bytes). - * - * That is, the test also allows you to compare AutoArray and std::vector. - * - * If USE_AUTO_ARRAY = 0, then HashMap confidently overtakes all. - * If USE_AUTO_ARRAY = 1, then HashMap is slightly less serious (20%) ahead of google::dense_hash_map. - * - * When using HashMap, AutoArray has a rather serious (40%) advantage over std::vector. - * And when using other hash tables, AutoArray even more seriously overtakes std::vector - * (up to three and a half times in the case of std::unordered_map and google::sparse_hash_map). - * - * HashMap, unlike google::dense_hash_map, much more depends on the quality of the hash function. - * - * PS. Measure everything yourself, otherwise I'm almost confused. - * - * PPS. Now the aggregation does not use an array of aggregate functions as values. - * States of aggregate functions were separated from the interface to manipulate them, and put in the pool. - * But in this test, there was something similar to the old scenario of using hash tables in the aggregation. - */ - -#define USE_AUTO_ARRAY 0 - - -struct AlternativeHash -{ - size_t operator() (UInt64 x) const - { - x ^= x >> 23; - x *= 0x2127599bf4325c37ULL; - x ^= x >> 47; - - return x; - } -}; - - -#if defined(__x86_64__) - -struct CRC32Hash_ -{ - size_t operator() (UInt64 x) const - { - UInt64 crc = -1ULL; - asm("crc32q %[x], %[crc]\n" : [crc] "+r" (crc) : [x] "rm" (x)); - return crc; - } -}; - -#endif - - -int main(int argc, char ** argv) -{ - using namespace DB; - - using Key = UInt64; - -#if USE_AUTO_ARRAY - using Value = AutoArray; -#else - using Value = std::vector; -#endif - - size_t n = argc < 2 ? 10000000 : atoi(argv[1]); - //size_t m = atoi(argv[2]); - - AggregateFunctionFactory factory; - DataTypes data_types_empty; - DataTypes data_types_uint64; - data_types_uint64.push_back(std::make_shared()); - - std::vector data(n); - Value value; - - AggregateFunctionPtr func_count = factory.get("count", data_types_empty); - AggregateFunctionPtr func_avg = factory.get("avg", data_types_uint64); - AggregateFunctionPtr func_uniq = factory.get("uniq", data_types_uint64); - - #define INIT \ - { \ - value.resize(3); \ - \ - value[0] = func_count.get();\ - value[1] = func_avg.get(); \ - value[2] = func_uniq.get(); \ - } - - INIT; - -#ifndef USE_AUTO_ARRAY - #undef INIT - #define INIT -#endif - - Row row(1); - row[0] = UInt64(0); - - std::cerr << "sizeof(Key) = " << sizeof(Key) << ", sizeof(Value) = " << sizeof(Value) << std::endl; - - { - Stopwatch watch; - /* for (size_t i = 0; i < n; ++i) - data[i] = rand() % m; - - for (size_t i = 0; i < n; i += 10) - data[i] = 0;*/ - - ReadBufferFromFile in1("UniqID.bin"); - CompressedReadBuffer in2(in1); - - in2.readStrict(reinterpret_cast(&data[0]), sizeof(data[0]) * n); - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "Vector. Size: " << n - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - if (argc < 3 || atoi(argv[2]) == 1) - { - Stopwatch watch; - - HashMap map; - HashMap::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(data[i], it, inserted); - if (inserted) - { - new(&it->second) Value(std::move(value)); - INIT; - } - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; - } - - if (argc < 3 || atoi(argv[2]) == 2) - { - Stopwatch watch; - - using Map = HashMap; - Map map; - Map::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(data[i], it, inserted); - if (inserted) - { - new(&it->second) Value(std::move(value)); - INIT; - } - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap, AlternativeHash. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; - } - -#if defined(__x86_64__) - if (argc < 3 || atoi(argv[2]) == 3) - { - Stopwatch watch; - - using Map = HashMap; - Map map; - Map::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(data[i], it, inserted); - if (inserted) - { - new(&it->second) Value(std::move(value)); - INIT; - } - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap, CRC32Hash. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; - } -#endif - - if (argc < 3 || atoi(argv[2]) == 4) - { - Stopwatch watch; - - std::unordered_map> map; - std::unordered_map>::iterator it; - for (size_t i = 0; i < n; ++i) - { - it = map.insert(std::make_pair(data[i], std::move(value))).first; - INIT; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "std::unordered_map. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - if (argc < 3 || atoi(argv[2]) == 5) - { - Stopwatch watch; - - google::dense_hash_map> map; - google::dense_hash_map>::iterator it; - map.set_empty_key(-1ULL); - for (size_t i = 0; i < n; ++i) - { - it = map.insert(std::make_pair(data[i], std::move(value))).first; - INIT; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "google::dense_hash_map. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - if (argc < 3 || atoi(argv[2]) == 6) - { - Stopwatch watch; - - google::sparse_hash_map> map; - google::sparse_hash_map>::iterator it; - for (size_t i = 0; i < n; ++i) - { - map.insert(std::make_pair(data[i], std::move(value))); - INIT; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "google::sparse_hash_map. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - return 0; -} diff --git a/dbms/src/Interpreters/tests/hash_map3.cpp b/dbms/src/Interpreters/tests/hash_map3.cpp deleted file mode 100644 index daadb8301f8..00000000000 --- a/dbms/src/Interpreters/tests/hash_map3.cpp +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#define DBMS_HASH_MAP_DEBUG_RESIZES -#define DBMS_HASH_MAP_COUNT_COLLISIONS - - -#include - -#include - -#include - -#include -#include - -#include - -#include - -#include - - -template -< - typename Key, - typename Mapped, - typename Hash = DefaultHash, - typename Grower = HashTableGrower<>, - typename Allocator = HashTableAllocator -> -class HashMapWithDump : public HashMap -{ -public: - void dump() const - { - for (size_t i = 0; i < this->grower.bufSize(); ++i) - { - if (this->buf[i].isZero(*this)) - std::cerr << "[ ]"; - else - std::cerr << '[' << this->buf[i].getValue().first.data << ", " << this->buf[i].getValue().second << ']'; - } - std::cerr << std::endl; - } -}; - - -struct SimpleHash -{ - size_t operator() (UInt64 x) const { return x; } - size_t operator() (StringRef x) const { return DB::parse(x.data); } -}; - -struct Grower : public HashTableGrower<2> -{ - void increaseSize() - { - ++size_degree; - } -}; - -int main(int, char **) -{ - using Map = HashMapWithDump< - StringRef, - UInt64, - SimpleHash, - Grower, - HashTableAllocatorWithStackMemory<4 * 24>>; - - Map map; - - map.dump(); - std::cerr << "size: " << map.size() << std::endl; - map[StringRef("1", 1)] = 1; - map.dump(); - std::cerr << "size: " << map.size() << std::endl; - map[StringRef("9", 1)] = 1; - map.dump(); - std::cerr << "size: " << map.size() << std::endl; - std::cerr << "Collisions: " << map.getCollisions() << std::endl; - map[StringRef("3", 1)] = 2; - map.dump(); - std::cerr << "size: " << map.size() << std::endl; - std::cerr << "Collisions: " << map.getCollisions() << std::endl; - - for (auto x : map) - std::cerr << x.first.toString() << " -> " << x.second << std::endl; - - return 0; -} diff --git a/dbms/src/Interpreters/tests/hash_map_string.cpp b/dbms/src/Interpreters/tests/hash_map_string.cpp deleted file mode 100644 index af7717112a6..00000000000 --- a/dbms/src/Interpreters/tests/hash_map_string.cpp +++ /dev/null @@ -1,505 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include - -#include -#include - -#include - -//#define DBMS_HASH_MAP_COUNT_COLLISIONS -#define DBMS_HASH_MAP_DEBUG_RESIZES - -#include -#include -#include -#include -#include -#include -#include - - -struct CompactStringRef -{ - union - { - const char * data_mixed = nullptr; - struct - { - char dummy[6]; - UInt16 size; - }; - }; - - CompactStringRef(const char * data_, size_t size_) - { - data_mixed = data_; - size = size_; - } - - CompactStringRef(const unsigned char * data_, size_t size_) : CompactStringRef(reinterpret_cast(data_), size_) {} - explicit CompactStringRef(const std::string & s) : CompactStringRef(s.data(), s.size()) {} - CompactStringRef() {} - - const char * data() const { return reinterpret_cast(reinterpret_cast(data_mixed) & 0x0000FFFFFFFFFFFFULL); } - - std::string toString() const { return std::string(data(), size); } -}; - -inline bool operator==(CompactStringRef lhs, CompactStringRef rhs) -{ - if (lhs.size != rhs.size) - return false; - - const char * lhs_data = lhs.data(); - const char * rhs_data = rhs.data(); - for (size_t pos = lhs.size - 1; pos < lhs.size; --pos) - if (lhs_data[pos] != rhs_data[pos]) - return false; - - return true; -} - -namespace ZeroTraits -{ - template <> - inline bool check(CompactStringRef x) { return nullptr == x.data_mixed; } - - template <> - inline void set(CompactStringRef & x) { x.data_mixed = nullptr; } -}; - -template <> -struct DefaultHash -{ - size_t operator() (CompactStringRef x) const - { - return CityHash_v1_0_2::CityHash64(x.data(), x.size); - } -}; - - -#define mix(h) ({ \ - (h) ^= (h) >> 23; \ - (h) *= 0x2127599bf4325c37ULL; \ - (h) ^= (h) >> 47; }) - -struct FastHash64 -{ - size_t operator() (CompactStringRef x) const - { - const char * buf = x.data(); - size_t len = x.size; - - const UInt64 m = 0x880355f21e6d1965ULL; - const UInt64 *pos = reinterpret_cast(buf); - const UInt64 *end = pos + (len / 8); - const unsigned char *pos2; - UInt64 h = len * m; - UInt64 v; - - while (pos != end) - { - v = *pos++; - h ^= mix(v); - h *= m; - } - - pos2 = reinterpret_cast(pos); - v = 0; - - switch (len & 7) - { - case 7: v ^= static_cast(pos2[6]) << 48; [[fallthrough]]; - case 6: v ^= static_cast(pos2[5]) << 40; [[fallthrough]]; - case 5: v ^= static_cast(pos2[4]) << 32; [[fallthrough]]; - case 4: v ^= static_cast(pos2[3]) << 24; [[fallthrough]]; - case 3: v ^= static_cast(pos2[2]) << 16; [[fallthrough]]; - case 2: v ^= static_cast(pos2[1]) << 8; [[fallthrough]]; - case 1: v ^= static_cast(pos2[0]); - h ^= mix(v); - h *= m; - } - - return mix(h); - } -}; - - -struct CrapWow -{ - size_t operator() (CompactStringRef x) const - { -#if __x86_64__ - const char * key = x.data(); - size_t len = x.size; - size_t seed = 0; - - const UInt64 m = 0x95b47aa3355ba1a1, n = 0x8a970be7488fda55; - - UInt64 hash; - // 3 = m, 4 = n - // r12 = h, r13 = k, ecx = seed, r12 = key - asm( - "leaq (%%rcx,%4), %%r13\n" - "movq %%rdx, %%r14\n" - "movq %%rcx, %%r15\n" - "movq %%rcx, %%r12\n" - "addq %%rax, %%r13\n" - "andq $0xfffffffffffffff0, %%rcx\n" - "jz QW%=\n" - "addq %%rcx, %%r14\n\n" - "negq %%rcx\n" - "XW%=:\n" - "movq %4, %%rax\n" - "mulq (%%r14,%%rcx)\n" - "xorq %%rax, %%r12\n" - "xorq %%rdx, %%r13\n" - "movq %3, %%rax\n" - "mulq 8(%%r14,%%rcx)\n" - "xorq %%rdx, %%r12\n" - "xorq %%rax, %%r13\n" - "addq $16, %%rcx\n" - "jnz XW%=\n" - "QW%=:\n" - "movq %%r15, %%rcx\n" - "andq $8, %%r15\n" - "jz B%=\n" - "movq %4, %%rax\n" - "mulq (%%r14)\n" - "addq $8, %%r14\n" - "xorq %%rax, %%r12\n" - "xorq %%rdx, %%r13\n" - "B%=:\n" - "andq $7, %%rcx\n" - "jz F%=\n" - "movq $1, %%rdx\n" - "shlq $3, %%rcx\n" - "movq %3, %%rax\n" - "shlq %%cl, %%rdx\n" - "addq $-1, %%rdx\n" - "andq (%%r14), %%rdx\n" - "mulq %%rdx\n" - "xorq %%rdx, %%r12\n" - "xorq %%rax, %%r13\n" - "F%=:\n" - "leaq (%%r13,%4), %%rax\n" - "xorq %%r12, %%rax\n" - "mulq %4\n" - "xorq %%rdx, %%rax\n" - "xorq %%r12, %%rax\n" - "xorq %%r13, %%rax\n" - : "=a"(hash), "=c"(key), "=d"(key) - : "r"(m), "r"(n), "a"(seed), "c"(len), "d"(key) - : "%r12", "%r13", "%r14", "%r15", "cc" - ); - return hash; -#else - return 0; -#endif - } -}; - - -struct SimpleHash -{ - size_t operator() (CompactStringRef x) const - { - const char * pos = x.data(); - size_t size = x.size; - - const char * end = pos + size; - - size_t res = 0; - - if (size == 0) - return 0; - - if (size < 8) - { - memcpy(reinterpret_cast(&res), pos, size); - return intHash64(res); - } - - while (pos + 8 < end) - { - UInt64 word = *reinterpret_cast(pos); - res = intHash64(word ^ res); - - pos += 8; - } - - UInt64 word = *reinterpret_cast(end - 8); - res = intHash64(word ^ res); - - return res; - } -}; - - -using Key = CompactStringRef; -using Value = UInt64; - - -struct Grower : public HashTableGrower<> -{ - /// The state of this structure is enough to get the buffer size of the hash table. - - /// Defines the initial size of the hash table. - static const size_t initial_size_degree = 16; - Grower() { size_degree = initial_size_degree; } - - size_t max_fill = (1ULL << initial_size_degree) * 0.9; - - /// The size of the hash table in the cells. - size_t bufSize() const { return 1ULL << size_degree; } - - size_t maxFill() const { return max_fill /*1 << (size_degree - 1)*/; } - size_t mask() const { return bufSize() - 1; } - - /// From the hash value, get the cell number in the hash table. - size_t place(size_t x) const { return x & mask(); } - - /// The next cell in the collision resolution chain. - size_t next(size_t pos) const { ++pos; return pos & mask(); } - - /// Whether the hash table is sufficiently full. You need to increase the size of the hash table, or remove something unnecessary from it. - bool overflow(size_t elems) const { return elems > maxFill(); } - - /// Increase the size of the hash table. - void increaseSize() - { - size_degree += size_degree >= 23 ? 1 : 2; - max_fill = (1ULL << size_degree) * 0.9; - } - - /// Set the buffer size by the number of elements in the hash table. Used when deserializing a hash table. - void set(size_t /*num_elems*/) - { - throw Poco::Exception(__PRETTY_FUNCTION__); - } -}; - - -int main(int argc, char ** argv) -{ - if (argc < 3) - { - std::cerr << "Usage: program n m\n"; - return 1; - } - - size_t n = atoi(argv[1]); - size_t m = atoi(argv[2]); - - DB::Arena pool; - std::vector data(n); - - std::cerr << "sizeof(Key) = " << sizeof(Key) << ", sizeof(Value) = " << sizeof(Value) << std::endl; - - { - Stopwatch watch; - DB::ReadBufferFromFileDescriptor in1(STDIN_FILENO); - DB::CompressedReadBuffer in2(in1); - - std::string tmp; - for (size_t i = 0; i < n && !in2.eof(); ++i) - { - DB::readStringBinary(tmp, in2); - data[i] = Key(pool.insert(tmp.data(), tmp.size()), tmp.size()); - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "Vector. Size: " << n - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - if (!m || m == 1) - { - Stopwatch watch; - - //using Map = HashMap; - - /// Saving the hash accelerates the resize by about 2 times, and the overall performance by 6-8%. - using Map = HashMapWithSavedHash, Grower>; - - Map map; - Map::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(data[i], it, inserted); - if (inserted) - it->second = 0; - ++it->second; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap (CityHash64). Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; - } - - if (!m || m == 2) - { - Stopwatch watch; - - using Map = HashMapWithSavedHash; - - Map map; - Map::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(data[i], it, inserted); - if (inserted) - it->second = 0; - ++it->second; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap (FastHash64). Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; - } - - if (!m || m == 3) - { - Stopwatch watch; - - using Map = HashMapWithSavedHash; - - Map map; - Map::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(data[i], it, inserted); - if (inserted) - it->second = 0; - ++it->second; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap (CrapWow). Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; - } - - if (!m || m == 4) - { - Stopwatch watch; - - using Map = HashMapWithSavedHash; - - Map map; - Map::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(data[i], it, inserted); - if (inserted) - it->second = 0; - ++it->second; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap (SimpleHash). Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; - } - - if (!m || m == 5) - { - Stopwatch watch; - - std::unordered_map> map; - for (size_t i = 0; i < n; ++i) - ++map[data[i]]; - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "std::unordered_map. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - if (!m || m == 6) - { - Stopwatch watch; - - google::dense_hash_map> map; - map.set_empty_key(Key("\0", 1)); - for (size_t i = 0; i < n; ++i) - ++map[data[i]]; - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "google::dense_hash_map. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - if (!m || m == 7) - { - Stopwatch watch; - - google::sparse_hash_map> map; - for (size_t i = 0; i < n; ++i) - ++map[data[i]]; - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "google::sparse_hash_map. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - return 0; -} diff --git a/dbms/src/Interpreters/tests/hash_map_string_2.cpp b/dbms/src/Interpreters/tests/hash_map_string_2.cpp deleted file mode 100644 index e073084aa8c..00000000000 --- a/dbms/src/Interpreters/tests/hash_map_string_2.cpp +++ /dev/null @@ -1,686 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include - -//#define DBMS_HASH_MAP_COUNT_COLLISIONS -#define DBMS_HASH_MAP_DEBUG_RESIZES - -#include -#include -#include -#include -#include -#include -#include - -#if __SSE4_1__ - #include -#endif - - -/** Do this: -for file in MobilePhoneModel PageCharset Params URLDomain UTMSource Referer URL Title; do - for size in 30000 100000 300000 1000000 5000000; do - echo - BEST_METHOD=0 - BEST_RESULT=0 - for method in {1..12}; do - echo -ne $file $size $method ''; - TOTAL_ELEMS=0 - for i in {0..1000}; do - TOTAL_ELEMS=$(( $TOTAL_ELEMS + $size )) - if [[ $TOTAL_ELEMS -gt 25000000 ]]; then break; fi - ./hash_map_string_2 $size $method < ${file}.bin 2>&1 | - grep HashMap | grep -oE '[0-9\.]+ elem'; - done | awk -W interactive '{ if ($1 > x) { x = $1 }; printf(".") } END { print x }' | tee /tmp/hash_map_string_2_res; - CUR_RESULT=$(cat /tmp/hash_map_string_2_res | tr -d '.') - if [[ $CUR_RESULT -gt $BEST_RESULT ]]; then - BEST_METHOD=$method - BEST_RESULT=$CUR_RESULT - fi; - done; - echo Best: $BEST_METHOD - $BEST_RESULT - done; -done -*/ - - -#define DefineStringRef(STRUCT) \ -\ -struct STRUCT : public StringRef {}; \ -\ -namespace ZeroTraits \ -{ \ - template <> \ - inline bool check(STRUCT x) { return 0 == x.size; } \ - \ - template <> \ - inline void set(STRUCT & x) { x.size = 0; } \ -}; \ - \ -template <> \ -struct DefaultHash \ -{ \ - size_t operator() (STRUCT x) const \ - { \ - return CityHash_v1_0_2::CityHash64(x.data, x.size); \ - } \ -}; - -DefineStringRef(StringRef_Compare1_Ptrs) -DefineStringRef(StringRef_Compare1_Index) -DefineStringRef(StringRef_CompareMemcmp) -DefineStringRef(StringRef_Compare8_1_byUInt64) -DefineStringRef(StringRef_Compare16_1_byMemcmp) -DefineStringRef(StringRef_Compare16_1_byUInt64_logicAnd) -DefineStringRef(StringRef_Compare16_1_byUInt64_bitAnd) - -#if __SSE4_1__ -DefineStringRef(StringRef_Compare16_1_byIntSSE) -DefineStringRef(StringRef_Compare16_1_byFloatSSE) -DefineStringRef(StringRef_Compare16_1_bySSE4) -DefineStringRef(StringRef_Compare16_1_bySSE4_wide) -DefineStringRef(StringRef_Compare16_1_bySSE_wide) -#endif - -DefineStringRef(StringRef_CompareAlwaysTrue) -DefineStringRef(StringRef_CompareAlmostAlwaysTrue) - - -inline bool operator==(StringRef_Compare1_Ptrs lhs, StringRef_Compare1_Ptrs rhs) -{ - if (lhs.size != rhs.size) - return false; - - if (lhs.size == 0) - return true; - - const char * pos1 = lhs.data; - const char * pos2 = rhs.data; - - const char * end1 = pos1 + lhs.size; - - while (pos1 < end1) - { - if (*pos1 != *pos2) - return false; - - ++pos1; - ++pos2; - } - - return true; -} - -inline bool operator==(StringRef_Compare1_Index lhs, StringRef_Compare1_Index rhs) -{ - if (lhs.size != rhs.size) - return false; - - if (lhs.size == 0) - return true; - - for (size_t i = 0; i < lhs.size; ++i) - if (lhs.data[i] != rhs.data[i]) - return false; - - return true; -} - -inline bool operator==(StringRef_CompareMemcmp lhs, StringRef_CompareMemcmp rhs) -{ - if (lhs.size != rhs.size) - return false; - - if (lhs.size == 0) - return true; - - return 0 == memcmp(lhs.data, rhs.data, lhs.size); -} - - -inline bool operator==(StringRef_Compare8_1_byUInt64 lhs, StringRef_Compare8_1_byUInt64 rhs) -{ - if (lhs.size != rhs.size) - return false; - - if (lhs.size == 0) - return true; - - const char * p1 = lhs.data; - const char * p2 = rhs.data; - size_t size = lhs.size; - - const char * p1_end = p1 + size; - const char * p1_end_8 = p1 + size / 8 * 8; - - while (p1 < p1_end_8) - { - if (reinterpret_cast(p1)[0] != reinterpret_cast(p2)[0]) - return false; - - p1 += 8; - p2 += 8; - } - - while (p1 < p1_end) - { - if (*p1 != *p2) - return false; - - ++p1; - ++p2; - } - - return true; -} - - - -inline bool compare_byMemcmp(const char * p1, const char * p2) -{ - return 0 == memcmp(p1, p2, 16); -} - -inline bool compare_byUInt64_logicAnd(const char * p1, const char * p2) -{ - return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0] - && reinterpret_cast(p1)[1] == reinterpret_cast(p2)[1]; -} - -inline bool compare_byUInt64_bitAnd(const char * p1, const char * p2) -{ - return (reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]) - & (reinterpret_cast(p1)[1] == reinterpret_cast(p2)[1]); -} - -#if __SSE4_1__ - -inline bool compare_byIntSSE(const char * p1, const char * p2) -{ - return 0xFFFF == _mm_movemask_epi8(_mm_cmpeq_epi8( - _mm_loadu_si128(reinterpret_cast(p1)), - _mm_loadu_si128(reinterpret_cast(p2)))); -} - -inline bool compare_byFloatSSE(const char * p1, const char * p2) -{ - return !_mm_movemask_ps(_mm_cmpneq_ps( /// Looks like incorrect while comparing subnormal floats. - _mm_loadu_ps(reinterpret_cast(p1)), - _mm_loadu_ps(reinterpret_cast(p2)))); -} - -#endif - - -template -inline bool memequal(const char * p1, const char * p2, size_t size) -{ -// const char * p1_end = p1 + size; - const char * p1_end_16 = p1 + size / 16 * 16; - - while (p1 < p1_end_16) - { - if (!compare(p1, p2)) - return false; - - p1 += 16; - p2 += 16; - } - -/* while (p1 < p1_end) - { - if (*p1 != *p2) - return false; - - ++p1; - ++p2; - }*/ - - switch (size % 16) - { - case 15: if (p1[14] != p2[14]) return false; [[fallthrough]]; - case 14: if (p1[13] != p2[13]) return false; [[fallthrough]]; - case 13: if (p1[12] != p2[12]) return false; [[fallthrough]]; - case 12: if (reinterpret_cast(p1)[2] == reinterpret_cast(p2)[2]) goto l8; else return false; - case 11: if (p1[10] != p2[10]) return false; [[fallthrough]]; - case 10: if (p1[9] != p2[9]) return false; [[fallthrough]]; - case 9: if (p1[8] != p2[8]) return false; - l8: [[fallthrough]]; - case 8: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 7: if (p1[6] != p2[6]) return false; [[fallthrough]]; - case 6: if (p1[5] != p2[5]) return false; [[fallthrough]]; - case 5: if (p1[4] != p2[4]) return false; [[fallthrough]]; - case 4: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 3: if (p1[2] != p2[2]) return false; [[fallthrough]]; - case 2: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 1: if (p1[0] != p2[0]) return false; [[fallthrough]]; - case 0: break; - } - - return true; -} - - -#if __SSE4_1__ - -inline bool memequal_sse41(const char * p1, const char * p2, size_t size) -{ -// const char * p1_end = p1 + size; - const char * p1_end_16 = p1 + size / 16 * 16; - - __m128i zero16 = _mm_setzero_si128(); - - while (p1 < p1_end_16) - { - if (!_mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(reinterpret_cast(p1)), - _mm_loadu_si128(reinterpret_cast(p2))))) - return false; - - p1 += 16; - p2 += 16; - } - -/* while (p1 < p1_end) - { - if (*p1 != *p2) - return false; - - ++p1; - ++p2; - }*/ - - switch (size % 16) - { - case 15: if (p1[14] != p2[14]) return false; [[fallthrough]]; - case 14: if (p1[13] != p2[13]) return false; [[fallthrough]]; - case 13: if (p1[12] != p2[12]) return false; [[fallthrough]]; - case 12: if (reinterpret_cast(p1)[2] == reinterpret_cast(p2)[2]) goto l8; else return false; - case 11: if (p1[10] != p2[10]) return false; [[fallthrough]]; - case 10: if (p1[9] != p2[9]) return false; [[fallthrough]]; - case 9: if (p1[8] != p2[8]) return false; - l8: [[fallthrough]]; - case 8: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 7: if (p1[6] != p2[6]) return false; [[fallthrough]]; - case 6: if (p1[5] != p2[5]) return false; [[fallthrough]]; - case 5: if (p1[4] != p2[4]) return false; [[fallthrough]]; - case 4: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 3: if (p1[2] != p2[2]) return false; [[fallthrough]]; - case 2: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 1: if (p1[0] != p2[0]) return false; [[fallthrough]]; - case 0: break; - } - - return true; -} - - -inline bool memequal_sse41_wide(const char * p1, const char * p2, size_t size) -{ - __m128i zero16 = _mm_setzero_si128(); -// const char * p1_end = p1 + size; - - while (size >= 64) - { - if (_mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(&reinterpret_cast(p1)[0]), - _mm_loadu_si128(&reinterpret_cast(p2)[0]))) - && _mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(&reinterpret_cast(p1)[1]), - _mm_loadu_si128(&reinterpret_cast(p2)[1]))) - && _mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(&reinterpret_cast(p1)[2]), - _mm_loadu_si128(&reinterpret_cast(p2)[2]))) - && _mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(&reinterpret_cast(p1)[3]), - _mm_loadu_si128(&reinterpret_cast(p2)[3])))) - { - p1 += 64; - p2 += 64; - size -= 64; - } - else - return false; - } - - switch ((size % 64) / 16) - { - case 3: - if (!_mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(&reinterpret_cast(p1)[2]), - _mm_loadu_si128(&reinterpret_cast(p2)[2])))) - return false; - [[fallthrough]]; - case 2: - if (!_mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(&reinterpret_cast(p1)[1]), - _mm_loadu_si128(&reinterpret_cast(p2)[1])))) - return false; - [[fallthrough]]; - case 1: - if (!_mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(&reinterpret_cast(p1)[0]), - _mm_loadu_si128(&reinterpret_cast(p2)[0])))) - return false; - } - - p1 += (size % 64) / 16 * 16; - p2 += (size % 64) / 16 * 16; - -/* - - if (size >= 32) - { - if (_mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(&reinterpret_cast(p1)[0]), - _mm_loadu_si128(&reinterpret_cast(p2)[0]))) - & _mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(&reinterpret_cast(p1)[1]), - _mm_loadu_si128(&reinterpret_cast(p2)[1])))) - { - p1 += 32; - p2 += 32; - size -= 32; - } - else - return false; - } - - if (size >= 16) - { - if (_mm_testc_si128( - zero16, - _mm_xor_si128( - _mm_loadu_si128(&reinterpret_cast(p1)[0]), - _mm_loadu_si128(&reinterpret_cast(p2)[0])))) - { - p1 += 16; - p2 += 16; - size -= 16; - } - else - return false; - }*/ - - switch (size % 16) - { - case 15: if (p1[14] != p2[14]) return false; [[fallthrough]]; - case 14: if (p1[13] != p2[13]) return false; [[fallthrough]]; - case 13: if (p1[12] != p2[12]) return false; [[fallthrough]]; - case 12: if (reinterpret_cast(p1)[2] == reinterpret_cast(p2)[2]) goto l8; else return false; - case 11: if (p1[10] != p2[10]) return false; [[fallthrough]]; - case 10: if (p1[9] != p2[9]) return false; [[fallthrough]]; - case 9: if (p1[8] != p2[8]) return false; - l8: [[fallthrough]]; - case 8: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 7: if (p1[6] != p2[6]) return false; [[fallthrough]]; - case 6: if (p1[5] != p2[5]) return false; [[fallthrough]]; - case 5: if (p1[4] != p2[4]) return false; [[fallthrough]]; - case 4: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 3: if (p1[2] != p2[2]) return false; [[fallthrough]]; - case 2: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 1: if (p1[0] != p2[0]) return false; [[fallthrough]]; - case 0: break; - } - - return true; -} - - -inline bool memequal_sse_wide(const char * p1, const char * p2, size_t size) -{ - while (size >= 64) - { - if ( compare_byIntSSE(p1, p2) - && compare_byIntSSE(p1 + 16, p2 + 16) - && compare_byIntSSE(p1 + 32, p2 + 32) - && compare_byIntSSE(p1 + 48, p2 + 48)) - { - p1 += 64; - p2 += 64; - size -= 64; - } - else - return false; - } - - switch ((size % 64) / 16) - { - case 3: if (!compare_byIntSSE(p1 + 32, p2 + 32)) return false; [[fallthrough]]; - case 2: if (!compare_byIntSSE(p1 + 16, p2 + 16)) return false; [[fallthrough]]; - case 1: if (!compare_byIntSSE(p1 , p2 )) return false; - } - - p1 += (size % 64) / 16 * 16; - p2 += (size % 64) / 16 * 16; - - switch (size % 16) - { - case 15: if (p1[14] != p2[14]) return false; [[fallthrough]]; - case 14: if (p1[13] != p2[13]) return false; [[fallthrough]]; - case 13: if (p1[12] != p2[12]) return false; [[fallthrough]]; - case 12: if (reinterpret_cast(p1)[2] == reinterpret_cast(p2)[2]) goto l8; else return false; - case 11: if (p1[10] != p2[10]) return false; [[fallthrough]]; - case 10: if (p1[9] != p2[9]) return false; [[fallthrough]]; - case 9: if (p1[8] != p2[8]) return false; - l8: [[fallthrough]]; - case 8: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 7: if (p1[6] != p2[6]) return false; [[fallthrough]]; - case 6: if (p1[5] != p2[5]) return false; [[fallthrough]]; - case 5: if (p1[4] != p2[4]) return false; [[fallthrough]]; - case 4: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 3: if (p1[2] != p2[2]) return false; [[fallthrough]]; - case 2: return reinterpret_cast(p1)[0] == reinterpret_cast(p2)[0]; - case 1: if (p1[0] != p2[0]) return false; [[fallthrough]]; - case 0: break; - } - - return true; -} - -#endif - - -#define Op(METHOD) \ -inline bool operator==(StringRef_Compare16_1_ ## METHOD lhs, StringRef_Compare16_1_ ## METHOD rhs) \ -{ \ - if (lhs.size != rhs.size) \ - return false; \ -\ - if (lhs.size == 0) \ - return true; \ -\ - return memequal(lhs.data, rhs.data, lhs.size); \ -} - -Op(byMemcmp) -Op(byUInt64_logicAnd) -Op(byUInt64_bitAnd) - -#if __SSE4_1__ - -Op(byIntSSE) -Op(byFloatSSE) - - -inline bool operator==(StringRef_Compare16_1_bySSE4 lhs, StringRef_Compare16_1_bySSE4 rhs) -{ - if (lhs.size != rhs.size) - return false; - - if (lhs.size == 0) - return true; - - return memequal_sse41(lhs.data, rhs.data, lhs.size); -} - -inline bool operator==(StringRef_Compare16_1_bySSE4_wide lhs, StringRef_Compare16_1_bySSE4_wide rhs) -{ - if (lhs.size != rhs.size) - return false; - - if (lhs.size == 0) - return true; - - return memequal_sse41_wide(lhs.data, rhs.data, lhs.size); -} - -inline bool operator==(StringRef_Compare16_1_bySSE_wide lhs, StringRef_Compare16_1_bySSE_wide rhs) -{ - if (lhs.size != rhs.size) - return false; - - if (lhs.size == 0) - return true; - - return memequal_sse_wide(lhs.data, rhs.data, lhs.size); -} - -#endif - - -inline bool operator==(StringRef_CompareAlwaysTrue, StringRef_CompareAlwaysTrue) -{ - return true; -} - -inline bool operator==(StringRef_CompareAlmostAlwaysTrue lhs, StringRef_CompareAlmostAlwaysTrue rhs) -{ - if (lhs.size != rhs.size) - return false; - - return true; -} - - - -using Value = UInt64; - - -template -void NO_INLINE bench(const std::vector & data, const char * name) -{ - Stopwatch watch; - - using Map = HashMapWithSavedHash>; - - Map map; - typename Map::iterator it; - bool inserted; - - for (size_t i = 0, size = data.size(); i < size; ++i) - { - map.emplace(static_cast(data[i]), it, inserted); - if (inserted) - it->second = 0; - ++it->second; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap (" << name << "). Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << data.size() / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; -} - - -int main(int argc, char ** argv) -{ - if (argc < 3) - { - std::cerr << "Usage: program n m\n"; - return 1; - } - - size_t n = atoi(argv[1]); - size_t m = atoi(argv[2]); - - DB::Arena pool; - std::vector data(n); - - std::cerr << "sizeof(Key) = " << sizeof(StringRef) << ", sizeof(Value) = " << sizeof(Value) << std::endl; - - { - Stopwatch watch; - DB::ReadBufferFromFileDescriptor in1(STDIN_FILENO); - DB::CompressedReadBuffer in2(in1); - - std::string tmp; - for (size_t i = 0; i < n && !in2.eof(); ++i) - { - DB::readStringBinary(tmp, in2); - data[i] = StringRef(pool.insert(tmp.data(), tmp.size()), tmp.size()); - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "Vector. Size: " << n - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - if (!m || m == 1) bench (data, "StringRef_Compare1_Ptrs"); - if (!m || m == 2) bench (data, "StringRef_Compare1_Index"); - if (!m || m == 3) bench (data, "StringRef_CompareMemcmp"); - if (!m || m == 4) bench (data, "StringRef_Compare8_1_byUInt64"); - if (!m || m == 5) bench (data, "StringRef_Compare16_1_byMemcmp"); - if (!m || m == 6) bench(data, "StringRef_Compare16_1_byUInt64_logicAnd"); - if (!m || m == 7) bench (data, "StringRef_Compare16_1_byUInt64_bitAnd"); -#if __SSE4_1__ - if (!m || m == 8) bench (data, "StringRef_Compare16_1_byIntSSE"); - if (!m || m == 9) bench (data, "StringRef_Compare16_1_byFloatSSE"); - if (!m || m == 10) bench (data, "StringRef_Compare16_1_bySSE4"); - if (!m || m == 11) bench (data, "StringRef_Compare16_1_bySSE4_wide"); - if (!m || m == 12) bench (data, "StringRef_Compare16_1_bySSE_wide"); -#endif - if (!m || m == 100) bench (data, "StringRef_CompareAlwaysTrue"); - if (!m || m == 101) bench (data, "StringRef_CompareAlmostAlwaysTrue"); - - /// 10 > 8, 9 - /// 1, 2, 5 - bad - - - return 0; -} diff --git a/dbms/src/Interpreters/tests/hash_map_string_3.cpp b/dbms/src/Interpreters/tests/hash_map_string_3.cpp deleted file mode 100644 index 1d6be35a5b1..00000000000 --- a/dbms/src/Interpreters/tests/hash_map_string_3.cpp +++ /dev/null @@ -1,525 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include - -#include -#include - -#define DBMS_HASH_MAP_COUNT_COLLISIONS -#define DBMS_HASH_MAP_DEBUG_RESIZES - -#include -#include -#include -#include -#include -#include -#include - -#if __SSE4_1__ - #include -#endif - - -/** Do this: -for file in MobilePhoneModel PageCharset Params URLDomain UTMSource Referer URL Title; do - for size in 30000 100000 300000 1000000 5000000; do - echo - BEST_METHOD=0 - BEST_RESULT=0 - for method in {1..11}; do - echo -ne $file $size $method ''; - TOTAL_ELEMS=0 - for i in {0..1000}; do - TOTAL_ELEMS=$(( $TOTAL_ELEMS + $size )) - if [[ $TOTAL_ELEMS -gt 25000000 ]]; then break; fi - ./hash_map_string_3 $size $method < ${file}.bin 2>&1 | - grep HashMap | grep -oE '[0-9\.]+ elem'; - done | awk -W interactive '{ if ($1 > x) { x = $1 }; printf(".") } END { print x }' | tee /tmp/hash_map_string_3_res; - CUR_RESULT=$(cat /tmp/hash_map_string_3_res | tr -d '.') - if [[ $CUR_RESULT -gt $BEST_RESULT ]]; then - BEST_METHOD=$method - BEST_RESULT=$CUR_RESULT - fi; - done; - echo Best: $BEST_METHOD - $BEST_RESULT - done; -done -*/ - - -#define DefineStringRef(STRUCT) \ -\ -struct STRUCT : public StringRef {}; \ -\ -namespace ZeroTraits \ -{ \ - template <> \ - inline bool check(STRUCT x) { return nullptr == x.data; } \ - \ - template <> \ - inline void set(STRUCT & x) { x.data = nullptr; } \ -}; \ - \ -template <> \ -struct DefaultHash \ -{ \ - size_t operator() (STRUCT x) const \ - { \ - return CityHash_v1_0_2::CityHash64(x.data, x.size); \ - } \ -}; - - -DefineStringRef(StringRef_CompareMemcmp) -DefineStringRef(StringRef_CompareAlwaysTrue) - - -inline bool operator==(StringRef_CompareMemcmp lhs, StringRef_CompareMemcmp rhs) -{ - if (lhs.size != rhs.size) - return false; - - if (lhs.size == 0) - return true; - - return 0 == memcmp(lhs.data, rhs.data, lhs.size); -} - -inline bool operator==(StringRef_CompareAlwaysTrue, StringRef_CompareAlwaysTrue) -{ - return true; -} - - -#define mix(h) ({ \ - (h) ^= (h) >> 23; \ - (h) *= 0x2127599bf4325c37ULL; \ - (h) ^= (h) >> 47; }) - -struct FastHash64 -{ - size_t operator() (StringRef x) const - { - const char * buf = x.data; - size_t len = x.size; - - const uint64_t m = 0x880355f21e6d1965ULL; - const uint64_t *pos = reinterpret_cast(buf); - const uint64_t *end = pos + (len / 8); - const unsigned char *pos2; - uint64_t h = len * m; - uint64_t v; - - while (pos != end) - { - v = *pos++; - h ^= mix(v); - h *= m; - } - - pos2 = reinterpret_cast(pos); - v = 0; - - switch (len & 7) - { - case 7: v ^= static_cast(pos2[6]) << 48; [[fallthrough]]; - case 6: v ^= static_cast(pos2[5]) << 40; [[fallthrough]]; - case 5: v ^= static_cast(pos2[4]) << 32; [[fallthrough]]; - case 4: v ^= static_cast(pos2[3]) << 24; [[fallthrough]]; - case 3: v ^= static_cast(pos2[2]) << 16; [[fallthrough]]; - case 2: v ^= static_cast(pos2[1]) << 8; [[fallthrough]]; - case 1: v ^= static_cast(pos2[0]); - h ^= mix(v); - h *= m; - } - - return mix(h); - } -}; - - -struct FNV1a -{ - size_t operator() (StringRef x) const - { - size_t res = 0xcbf29ce484222325ULL; - - const char * pos = x.data; - const char * end = x.data + x.size; - - for (; pos < end; ++pos) - { - res *= 1099511628211ULL; - res ^= *pos; - } - - return res; - } -}; - - -#if __SSE4_1__ - -struct CrapWow -{ - size_t operator() (StringRef x) const - { - const char * key = x.data; - size_t len = x.size; - size_t seed = 0; - - const UInt64 m = 0x95b47aa3355ba1a1, n = 0x8a970be7488fda55; - UInt64 hash; - // 3 = m, 4 = n - // r12 = h, r13 = k, ecx = seed, r12 = key - asm( - "leaq (%%rcx,%4), %%r13\n" - "movq %%rdx, %%r14\n" - "movq %%rcx, %%r15\n" - "movq %%rcx, %%r12\n" - "addq %%rax, %%r13\n" - "andq $0xfffffffffffffff0, %%rcx\n" - "jz QW%=\n" - "addq %%rcx, %%r14\n\n" - "negq %%rcx\n" - "XW%=:\n" - "movq %4, %%rax\n" - "mulq (%%r14,%%rcx)\n" - "xorq %%rax, %%r12\n" - "xorq %%rdx, %%r13\n" - "movq %3, %%rax\n" - "mulq 8(%%r14,%%rcx)\n" - "xorq %%rdx, %%r12\n" - "xorq %%rax, %%r13\n" - "addq $16, %%rcx\n" - "jnz XW%=\n" - "QW%=:\n" - "movq %%r15, %%rcx\n" - "andq $8, %%r15\n" - "jz B%=\n" - "movq %4, %%rax\n" - "mulq (%%r14)\n" - "addq $8, %%r14\n" - "xorq %%rax, %%r12\n" - "xorq %%rdx, %%r13\n" - "B%=:\n" - "andq $7, %%rcx\n" - "jz F%=\n" - "movq $1, %%rdx\n" - "shlq $3, %%rcx\n" - "movq %3, %%rax\n" - "shlq %%cl, %%rdx\n" - "addq $-1, %%rdx\n" - "andq (%%r14), %%rdx\n" - "mulq %%rdx\n" - "xorq %%rdx, %%r12\n" - "xorq %%rax, %%r13\n" - "F%=:\n" - "leaq (%%r13,%4), %%rax\n" - "xorq %%r12, %%rax\n" - "mulq %4\n" - "xorq %%rdx, %%rax\n" - "xorq %%r12, %%rax\n" - "xorq %%r13, %%rax\n" - : "=a"(hash), "=c"(key), "=d"(key) - : "r"(m), "r"(n), "a"(seed), "c"(len), "d"(key) - : "%r12", "%r13", "%r14", "%r15", "cc" - ); - return hash; - } -}; - -#endif - - -struct SimpleHash -{ - size_t operator() (StringRef x) const - { - const char * pos = x.data; - size_t size = x.size; - - const char * end = pos + size; - - size_t res = 0; - - if (size == 0) - return 0; - - if (size < 8) - { -#if __SSE4_1__ - return hashLessThan8(x.data, x.size); -#endif - } - - while (pos + 8 < end) - { - uint64_t word = *reinterpret_cast(pos); - res = intHash64(word ^ res); - - pos += 8; - } - - uint64_t word = *reinterpret_cast(end - 8); - res = intHash64(word ^ res); - - return res; - } -}; - - -struct VerySimpleHash -{ - size_t operator() (StringRef x) const - { - const char * pos = x.data; - size_t size = x.size; - - const char * end = pos + size; - - size_t res = 0; - - if (size == 0) - return 0; - - if (size < 8) - { -#if __SSE4_1__ - return hashLessThan8(x.data, x.size); -#endif - } - - while (pos + 8 < end) - { - res ^= reinterpret_cast(pos)[0]; - res ^= res >> 33; - res *= 0xff51afd7ed558ccdULL; - - pos += 8; - } - - res ^= *reinterpret_cast(end - 8); - res ^= res >> 33; - res *= 0xc4ceb9fe1a85ec53ULL; - res ^= res >> 33; - - return res; - } -}; - - -struct FarmHash64 -{ - size_t operator() (StringRef x) const - { - return farmhash::Hash64(x.data, x.size); - } -}; - - -template -struct MetroHash64 -{ - size_t operator() (StringRef x) const - { - union - { - uint64_t u64; - std::uint8_t u8[sizeof(u64)]; - }; - - metrohash64(reinterpret_cast(x.data), x.size, 0, u8); - - return u64; - } -}; - - -#if __SSE4_1__ - -/*struct CRC32Hash -{ - size_t operator() (StringRef x) const - { - const char * pos = x.data; - size_t size = x.size; - - if (size == 0) - return 0; - - if (size < 8) - { - return hashLessThan8(x.data, x.size); - } - - const char * end = pos + size; - size_t res = -1ULL; - - do - { - uint64_t word = *reinterpret_cast(pos); - res = _mm_crc32_u64(res, word); - - pos += 8; - } while (pos + 8 < end); - - uint64_t word = *reinterpret_cast(end - 8); - res = _mm_crc32_u64(res, word); - - return res; - } -};*/ - - -struct CRC32ILPHash -{ - size_t operator() (StringRef x) const - { - const char * pos = x.data; - size_t size = x.size; - - if (size == 0) - return 0; - - if (size < 16) - { - return hashLessThan16(x.data, x.size); - } - - const char * end = pos + size; - const char * end_16 = pos + size / 16 * 16; - size_t res0 = -1ULL; - size_t res1 = -1ULL; - - do - { - uint64_t word0 = reinterpret_cast(pos)[0]; - uint64_t word1 = reinterpret_cast(pos)[1]; - res0 = _mm_crc32_u64(res0, word0); - res1 = _mm_crc32_u64(res1, word1); - - pos += 16; - } while (pos < end_16); - - uint64_t word0 = *reinterpret_cast(end - 8); - uint64_t word1 = *reinterpret_cast(end - 16); - - /* return HashLen16(Rotate(word0 - word1, 43) + Rotate(res0, 30) + res1, - word0 + Rotate(word1 ^ k3, 20) - res0 + size);*/ - - res0 = _mm_crc32_u64(res0, word0); - res1 = _mm_crc32_u64(res1, word1); - - return hashLen16(res0, res1); - } -}; - -#endif - - -using Value = uint64_t; - - -template -void NO_INLINE bench(const std::vector & data, const char * name) -{ - Stopwatch watch; - - using Map = HashMapWithSavedHash; - - Map map; - typename Map::iterator it; - bool inserted; - - for (size_t i = 0, size = data.size(); i < size; ++i) - { - map.emplace(static_cast(data[i]), it, inserted); - if (inserted) - it->second = 0; - ++it->second; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap (" << name << "). Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << data.size() / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; -} - - -int main(int argc, char ** argv) -{ - if (argc < 3) - { - std::cerr << "Usage: program n m\n"; - return 1; - } - - size_t n = atoi(argv[1]); - size_t m = atoi(argv[2]); - - DB::Arena pool; - std::vector data(n); - - std::cerr << "sizeof(Key) = " << sizeof(StringRef) << ", sizeof(Value) = " << sizeof(Value) << std::endl; - - { - Stopwatch watch; - DB::ReadBufferFromFileDescriptor in1(STDIN_FILENO); - DB::CompressedReadBuffer in2(in1); - - std::string tmp; - for (size_t i = 0; i < n && !in2.eof(); ++i) - { - DB::readStringBinary(tmp, in2); - data[i] = StringRef(pool.insert(tmp.data(), tmp.size()), tmp.size()); - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "Vector. Size: " << n - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - if (!m || m == 1) bench(data, "StringRef_CityHash64"); - if (!m || m == 2) bench (data, "StringRef_FastHash64"); - if (!m || m == 3) bench (data, "StringRef_SimpleHash"); - if (!m || m == 4) bench (data, "StringRef_FNV1a"); - -#if __SSE4_1__ - if (!m || m == 5) bench (data, "StringRef_CrapWow"); - if (!m || m == 6) bench (data, "StringRef_CRC32Hash"); - if (!m || m == 7) bench (data, "StringRef_CRC32ILPHash"); -#endif - - if (!m || m == 8) bench (data, "StringRef_VerySimpleHash"); - if (!m || m == 9) bench (data, "StringRef_FarmHash64"); - if (!m || m == 10) bench>(data, "StringRef_MetroHash64_1"); - if (!m || m == 11) bench>(data, "StringRef_MetroHash64_2"); - - return 0; -} diff --git a/dbms/src/Interpreters/tests/hash_map_string_small.cpp b/dbms/src/Interpreters/tests/hash_map_string_small.cpp deleted file mode 100644 index 7c7059d6381..00000000000 --- a/dbms/src/Interpreters/tests/hash_map_string_small.cpp +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include - -#include -#include - -#include - -//#define DBMS_HASH_MAP_COUNT_COLLISIONS -#define DBMS_HASH_MAP_DEBUG_RESIZES - -#include -#include -#include -#include -#include -#include -#include - - -struct SmallStringRef -{ - UInt32 size; - - union - { - const char * data_big; - char data_small[12]; - }; - - bool isSmall() const { return size <= 12; } - - const char * data() const - { - return isSmall() ? data_small : data_big; - } - - SmallStringRef(const char * data_, size_t size_) - { - size = size_; - - if (isSmall()) - memcpy(data_small, data_, size_); - else - data_big = data_; - } - - SmallStringRef(const unsigned char * data_, size_t size_) : SmallStringRef(reinterpret_cast(data_), size_) {} - explicit SmallStringRef(const std::string & s) : SmallStringRef(s.data(), s.size()) {} - SmallStringRef() {} - - std::string toString() const { return std::string(data(), size); } -}; - - -inline bool operator==(SmallStringRef lhs, SmallStringRef rhs) -{ - if (lhs.size != rhs.size) - return false; - - if (lhs.size == 0) - return true; - -#if __SSE2__ - return memequalSSE2Wide(lhs.data(), rhs.data(), lhs.size); -#else - return false; -#endif -} - - -namespace ZeroTraits -{ - template <> - inline bool check(SmallStringRef x) { return x.size == 0; } - - template <> - inline void set(SmallStringRef & x) { x.size = 0; } -}; - -template <> -struct DefaultHash -{ - size_t operator() (SmallStringRef x) const - { - return DefaultHash()(StringRef(x.data(), x.size)); - } -}; - - -using Value = UInt64; - - -int main(int argc, char ** argv) -{ - if (argc < 3) - { - std::cerr << "Usage: program n m\n"; - return 1; - } - - size_t n = atoi(argv[1]); - size_t m = atoi(argv[2]); - - DB::Arena pool; - std::vector data(n); - - std::cerr << "sizeof(Key) = " << sizeof(SmallStringRef) << ", sizeof(Value) = " << sizeof(Value) << std::endl; - - { - Stopwatch watch; - DB::ReadBufferFromFileDescriptor in1(STDIN_FILENO); - DB::CompressedReadBuffer in2(in1); - - std::string tmp; - for (size_t i = 0; i < n && !in2.eof(); ++i) - { - DB::readStringBinary(tmp, in2); - data[i] = StringRef(pool.insert(tmp.data(), tmp.size()), tmp.size()); - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "Vector. Size: " << n - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - if (!m || m == 1) - { - Stopwatch watch; - - using Map = HashMapWithSavedHash; - - Map map; - Map::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(data[i], it, inserted); - if (inserted) - it->second = 0; - ++it->second; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap (StringRef). Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; - } - - if (!m || m == 2) - { - Stopwatch watch; - - using Map = HashMapWithSavedHash; - - Map map; - Map::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(SmallStringRef(data[i].data, data[i].size), it, inserted); - if (inserted) - it->second = 0; - ++it->second; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap (SmallStringRef). Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" -#ifdef DBMS_HASH_MAP_COUNT_COLLISIONS - << ", collisions: " << map.getCollisions() -#endif - << std::endl; - } - - return 0; -} diff --git a/dbms/src/Interpreters/tests/logical_expressions_optimizer.cpp b/dbms/src/Interpreters/tests/logical_expressions_optimizer.cpp deleted file mode 100644 index 6da69abade0..00000000000 --- a/dbms/src/Interpreters/tests/logical_expressions_optimizer.cpp +++ /dev/null @@ -1,310 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace -{ - -struct TestEntry -{ - std::string input; - std::string expected_output; - UInt64 limit; -}; - -using TestEntries = std::vector; -using TestResult = std::pair; - -void run(); -void performTests(const TestEntries & entries); -TestResult check(const TestEntry & entry); -bool parse(DB::ASTPtr & ast, const std::string & query); -bool equals(const DB::ASTPtr & lhs, const DB::ASTPtr & rhs); -void reorder(DB::IAST * ast); - - -void run() -{ - /// NOTE: Queries are not always realistic, but we are only interested in the syntax. - TestEntries entries = - { - { - "SELECT 1", - "SELECT 1", - 3 - }, - - // WHERE - - { - "SELECT name, value FROM report WHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report WHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - 4 - }, - - { - "SELECT name, value FROM report WHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report WHERE name IN ('Alice', 'Bob', 'Carol')", - 3 - }, - - { - "SELECT name, value FROM report WHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report WHERE name IN ('Alice', 'Bob', 'Carol')", - 2 - }, - - { - "SELECT name, value FROM report WHERE (name = 'Alice') OR (value = 1000) OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report WHERE (value = 1000) OR name IN ('Alice', 'Bob', 'Carol')", - 2 - }, - - { - "SELECT name, value FROM report WHERE (name = 'Alice') OR (value = 1000) OR (name = 'Bob') OR (name = 'Carol') OR (value = 2000)", - "SELECT name, value FROM report WHERE name IN ('Alice', 'Bob', 'Carol') OR value IN (1000, 2000)", - 2 - }, - - { - "SELECT value FROM report WHERE ((value + 1) = 1000) OR ((2 * value) = 2000) OR ((2 * value) = 4000) OR ((value + 1) = 3000)", - "SELECT value FROM report WHERE ((value + 1) IN (1000, 3000)) OR ((2 * value) IN (2000, 4000))", - 2 - }, - - { - "SELECT name, value FROM report WHERE ((name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')) AND ((value = 1000) OR (value = 2000))", - "SELECT name, value FROM report WHERE name IN ('Alice', 'Bob', 'Carol') AND ((value = 1000) OR (value = 2000))", - 3 - }, - - // PREWHERE - - { - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - 4 - }, - - { - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report PREWHERE name IN ('Alice', 'Bob', 'Carol')", - 3 - }, - - { - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report PREWHERE name IN ('Alice', 'Bob', 'Carol')", - 2 - }, - - { - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (value = 1000) OR (name = 'Bob') OR (name = 'Carol')", - "SELECT name, value FROM report PREWHERE (value = 1000) OR name IN ('Alice', 'Bob', 'Carol')", - 2 - }, - - { - "SELECT name, value FROM report PREWHERE (name = 'Alice') OR (value = 1000) OR (name = 'Bob') OR (name = 'Carol') OR (value = 2000)", - "SELECT name, value FROM report PREWHERE name IN ('Alice', 'Bob', 'Carol') OR value IN (1000, 2000)", - 2 - }, - - { - "SELECT value FROM report PREWHERE ((value + 1) = 1000) OR ((2 * value) = 2000) OR ((2 * value) = 4000) OR ((value + 1) = 3000)", - "SELECT value FROM report PREWHERE (value + 1) IN (1000, 3000) OR (2 * value) IN (2000, 4000)", - 2 - }, - - // HAVING - - { - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING number = 1", - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING number = 1", - 2 - }, - - { - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING (number = 1) OR (number = 2)", - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING number IN (1, 2)", - 2 - }, - - { - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING (number = 1) OR (number = 2)", - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING (number = 1) OR (number = 2)", - 3 - }, - - { - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING ((number + 1) = 1) OR ((number + 1) = 2) OR ((number + 3) = 7)", - "SELECT number, count() FROM (SELECT * FROM system.numbers LIMIT 10) GROUP BY number HAVING ((number + 3) = 7) OR (number + 1) IN (1, 2)", - 2 - }, - - // PREWHERE + WHERE + HAVING - - { - "SELECT number, count(), 1 AS T, 2 AS U FROM (SELECT * FROM system.numbers LIMIT 10) PREWHERE (U = 1) OR (U = 2) " - "WHERE (T = 1) OR (T = 2) GROUP BY number HAVING (number = 1) OR (number = 2)", - "SELECT number, count(), 1 AS T, 2 AS U FROM (SELECT * FROM system.numbers LIMIT 10) PREWHERE U IN (1, 2) " - "WHERE T IN (1, 2) GROUP BY number HAVING number IN (1, 2)", - 2 - }, - - { - "SELECT number, count(), 1 AS T, 2 AS U FROM (SELECT * FROM system.numbers LIMIT 10) PREWHERE (U = 1) OR (U = 2) OR (U = 3) " - "WHERE (T = 1) OR (T = 2) GROUP BY number HAVING (number = 1) OR (number = 2)", - "SELECT number, count(), 1 AS T, 2 AS U FROM (SELECT * FROM system.numbers LIMIT 10) PREWHERE U IN (1, 2, 3) " - "WHERE (T = 1) OR (T = 2) GROUP BY number HAVING (number = 1) OR (number = 2)", - 3 - }, - - { - "SELECT x = 1 OR x=2 OR (x = 3 AS x3) AS y, 4 AS x", - "SELECT x IN (1, 2, 3) AS y, 4 AS x", - 2 - } - }; - - performTests(entries); -} - -void performTests(const TestEntries & entries) -{ - unsigned int count = 0; - unsigned int i = 1; - - for (const auto & entry : entries) - { - auto res = check(entry); - if (res.first) - { - ++count; - std::cout << "Test " << i << " passed.\n"; - } - else - std::cout << "Test " << i << " failed. Expected: " << entry.expected_output << ". Received: " << res.second << "\n"; - - ++i; - } - std::cout << count << " out of " << entries.size() << " test(s) passed.\n"; -} - -TestResult check(const TestEntry & entry) -{ - try - { - /// Parse and optimize the incoming query. - DB::ASTPtr ast_input; - if (!parse(ast_input, entry.input)) - return TestResult(false, "parse error"); - - auto select_query = typeid_cast(&*ast_input); - - DB::Settings settings; - settings.optimize_min_equality_disjunction_chain_length = entry.limit; - - DB::LogicalExpressionsOptimizer optimizer(select_query, settings); - optimizer.perform(); - - /// Parse the expected result. - DB::ASTPtr ast_expected; - if (!parse(ast_expected, entry.expected_output)) - return TestResult(false, "parse error"); - - /// Compare the optimized query and the expected result. - bool res = equals(ast_input, ast_expected); - std::string output = DB::queryToString(ast_input); - - return TestResult(res, output); - } - catch (DB::Exception & e) - { - return TestResult(false, e.displayText()); - } -} - -bool parse(DB::ASTPtr & ast, const std::string & query) -{ - DB::ParserSelectQuery parser; - std::string message; - auto begin = query.data(); - auto end = begin + query.size(); - ast = DB::tryParseQuery(parser, begin, end, message, false, "", false, 0); - return ast != nullptr; -} - -bool equals(const DB::ASTPtr & lhs, const DB::ASTPtr & rhs) -{ - DB::ASTPtr lhs_reordered = lhs->clone(); - reorder(&*lhs_reordered); - - DB::ASTPtr rhs_reordered = rhs->clone(); - reorder(&*rhs_reordered); - - return lhs_reordered->getTreeHash() == rhs_reordered->getTreeHash(); -} - -void reorderImpl(DB::IAST * ast) -{ - if (ast == nullptr) - return; - - auto & children = ast->children; - if (children.empty()) - return; - - for (auto & child : children) - reorderImpl(&*child); - - std::sort(children.begin(), children.end(), [](const DB::ASTPtr & lhs, const DB::ASTPtr & rhs) - { - return lhs->getTreeHash() < rhs->getTreeHash(); - }); -} - -void reorder(DB::IAST * ast) -{ - if (ast == nullptr) - return; - - auto select_query = typeid_cast(ast); - if (select_query == nullptr) - return; - - reorderImpl(select_query->where_expression.get()); - reorderImpl(select_query->prewhere_expression.get()); - reorderImpl(select_query->having_expression.get()); -} - -} - -int main() -{ - run(); - return 0; -} diff --git a/dbms/src/Interpreters/tests/select_query.cpp b/dbms/src/Interpreters/tests/select_query.cpp deleted file mode 100644 index 2d57d3bee89..00000000000 --- a/dbms/src/Interpreters/tests/select_query.cpp +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - - -using namespace DB; - -int main(int, char **) -try -{ - Poco::AutoPtr channel = new Poco::ConsoleChannel(std::cerr); - Poco::Logger::root().setChannel(channel); - Poco::Logger::root().setLevel("trace"); - - /// Pre-initialize the `DateLUT` so that the first initialization does not affect the measured execution speed. - DateLUT::instance(); - - Context context = Context::createGlobal(); - - context.setPath("./"); - - loadMetadata(context); - - DatabasePtr system = std::make_shared("system", "./metadata/system/", context); - context.addDatabase("system", system); - system->loadTables(context, nullptr, false); - attachSystemTablesLocal(*context.getDatabase("system")); - context.setCurrentDatabase("default"); - - ReadBufferFromFileDescriptor in(STDIN_FILENO); - WriteBufferFromFileDescriptor out(STDOUT_FILENO); - - executeQuery(in, out, /* allow_into_outfile = */ false, context, {}); - - return 0; -} -catch (const Exception & e) -{ - std::cerr << e.what() << ", " << e.displayText() << std::endl - << std::endl - << "Stack trace:" << std::endl - << e.getStackTrace().toString(); - return 1; -} diff --git a/dbms/src/Interpreters/tests/two_level_hash_map.cpp b/dbms/src/Interpreters/tests/two_level_hash_map.cpp deleted file mode 100644 index 5d7d67d71c2..00000000000 --- a/dbms/src/Interpreters/tests/two_level_hash_map.cpp +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include - -#include -#include - -//#define DBMS_HASH_MAP_DEBUG_RESIZES - -#include -#include - -#include -#include -#include -#include -#include - - -using Key = UInt64; -using Value = UInt64; - - -int main(int argc, char ** argv) -{ - if (argc < 2) - { - std::cerr << "Usage: program n\n"; - return 1; - } - - size_t n = atoi(argv[1]); - - std::vector data(n); - - std::cerr << "sizeof(Key) = " << sizeof(Key) << ", sizeof(Value) = " << sizeof(Value) << std::endl; - - { - Stopwatch watch; - DB::ReadBufferFromFileDescriptor in1(STDIN_FILENO); - DB::CompressedReadBuffer in2(in1); - - in2.readStrict(reinterpret_cast(&data[0]), sizeof(data[0]) * n); - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "Vector. Size: " << n - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - } - - { - Stopwatch watch; - - std::cerr << sizeof(HashMapCell>) << std::endl; - - using Map = TwoLevelHashTable>, DefaultHash, HashTableGrower<8>, HashTableAllocator>; - - Map map; - Map::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(data[i], it, inserted); - if (inserted) - it->second = 0; - ++it->second; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - - size_t sum_counts = 0; - size_t elems = 0; - for (const auto & kv : map) - { - sum_counts += kv.second; - ++elems; - } - - std::cerr << "sum_counts: " << sum_counts << ", elems: " << elems << std::endl; - } - - { - Stopwatch watch; - - using Map = TwoLevelHashTable>, DefaultHash, HashTableGrower<8>, HashTableAllocator>; - //using Map = HashMap; - - Map map; - Map::iterator it; - bool inserted; - - for (size_t i = 0; i < n; ++i) - { - map.emplace(i, it, inserted); - if (inserted) - it->second = 0; - ++it->second; - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "HashMap. Size: " << map.size() - << ", elapsed: " << watch.elapsedSeconds() - << " (" << n / watch.elapsedSeconds() << " elem/sec.)" - << std::endl; - - size_t sum_counts = 0; - size_t elems = 0; - for (const auto & kv : map) - { - sum_counts += kv.second; - ++elems; - - if (kv.first > n) - std::cerr << kv.first << std::endl; - } - - std::cerr << "sum_counts: " << sum_counts << ", elems: " << elems << std::endl; - - if (sum_counts != n) - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - } - - return 0; -} diff --git a/dbms/src/Interpreters/tests/users.cpp b/dbms/src/Interpreters/tests/users.cpp deleted file mode 100644 index 60df6af4d71..00000000000 --- a/dbms/src/Interpreters/tests/users.cpp +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace -{ - -namespace fs = boost::filesystem; - -struct TestEntry -{ - std::string user_name; - std::string database_name; - bool is_allowed; -}; - -using TestEntries = std::vector; - -struct TestDescriptor -{ - const char * config_content; - TestEntries entries; -}; - -using TestSet = std::vector; - -/// Tests description. - -TestSet test_set = -{ - { - "" - " " - " " - " " - " defaultdefault" - " " - " default" - " test" - " " - " " - " " - " defaultdefault" - " " - " " - " " - "", - - { - { "default", "default", true }, - { "default", "test", true }, - { "default", "stats", false }, - { "web", "default", true }, - { "web", "test", true }, - { "web", "stats", true }, - { "analytics", "default", false }, - { "analytics", "test", false }, - { "analytics", "stats", false } - } - }, - - { - "" - " " - " " - " " - " defaultdefault" - " " - " default" - " " - " " - " " - " defaultdefault" - " " - " " - " " - "", - - { - { "default", "default", true }, - { "default", "test", false }, - { "default", "stats", false }, - { "web", "default", true }, - { "web", "test", true }, - { "web", "stats", true }, - { "analytics", "default", false }, - { "analytics", "test", false }, - { "analytics", "stats", false } - } - }, - - { - "" - " " - " " - " " - " defaultdefault" - " " - " " - " " - " " - " defaultdefault" - " " - " " - " " - "", - - { - { "default", "default", true }, - { "default", "test", true }, - { "default", "stats", true }, - { "web", "default", true }, - { "web", "test", true }, - { "web", "stats", true }, - { "analytics", "default", false }, - { "analytics", "test", false }, - { "analytics", "stats", false } - } - }, - - { - "" - " " - " " - " " - " defaultdefault" - " " - " default" - " " - " " - " " - " defaultdefault" - " " - " test" - " " - " " - " " - " " - "", - - { - { "default", "default", true }, - { "default", "test", false }, - { "default", "stats", false }, - { "web", "default", false }, - { "web", "test", true }, - { "web", "stats", false }, - { "analytics", "default", false }, - { "analytics", "test", false }, - { "analytics", "stats", false } - } - } -}; - -std::string createTmpPath(const std::string & filename); -void createFile(const std::string & filename, const char * data); -void runOneTest(const TestDescriptor & test_descriptor); -auto runTestSet(const TestSet & test_set); - -std::string createTmpPath(const std::string & filename) -{ - char pattern[] = "/tmp/fileXXXXXX"; - char * dir = mkdtemp(pattern); - if (dir == nullptr) - throw std::runtime_error("Could not create directory"); - - return std::string(dir) + "/" + filename; -} - -void createFile(const std::string & filename, const char * data) -{ - std::ofstream ofs(filename.c_str()); - if (!ofs.is_open()) - throw std::runtime_error("Could not open file " + filename); - ofs << data; -} - -void runOneTest(const TestDescriptor & test_descriptor) -{ - const auto path_name = createTmpPath("users.xml"); - createFile(path_name, test_descriptor.config_content); - - ConfigurationPtr config; - - try - { - config = ConfigProcessor(path_name).loadConfig().configuration; - } - catch (const Poco::Exception & ex) - { - std::ostringstream os; - os << "Error: " << ex.what() << ": " << ex.displayText(); - throw std::runtime_error(os.str()); - } - - DB::SecurityManager security_manager; - - try - { - security_manager.loadFromConfig(*config); - } - catch (const Poco::Exception & ex) - { - std::ostringstream os; - os << "Error: " << ex.what() << ": " << ex.displayText(); - throw std::runtime_error(os.str()); - } - - for (const auto & entry : test_descriptor.entries) - { - bool res; - - try - { - res = security_manager.hasAccessToDatabase(entry.user_name, entry.database_name); - } - catch (const Poco::Exception &) - { - res = false; - } - - if (res != entry.is_allowed) - { - auto to_string = [](bool access){ return (access ? "'granted'" : "'denied'"); }; - std::ostringstream os; - os << "(user=" << entry.user_name << ", database=" << entry.database_name << "): "; - os << "Expected " << to_string(entry.is_allowed) << " but got " << to_string(res); - throw std::runtime_error(os.str()); - } - } - - fs::remove_all(fs::path(path_name).parent_path().string()); -} - -auto runTestSet(const TestSet & test_set) -{ - size_t test_num = 1; - size_t failure_count = 0; - - for (const auto & test_descriptor : test_set) - { - try - { - runOneTest(test_descriptor); - std::cout << "Test " << test_num << " passed\n"; - } - catch (const std::runtime_error & ex) - { - std::cerr << "Test " << test_num << " failed with reason: " << ex.what() << "\n"; - ++failure_count; - } - catch (...) - { - std::cerr << "Test " << test_num << " failed with unknown reason\n"; - ++failure_count; - } - - ++test_num; - } - - return std::make_tuple(test_set.size(), failure_count); -} - -} - -int main() -{ - size_t test_count; - size_t failure_count; - - std::tie(test_count, failure_count) = runTestSet(test_set); - - std::cout << (test_count - failure_count) << " test(s) passed out of " << test_count << "\n"; - - return (failure_count == 0) ? 0 : EXIT_FAILURE; -} diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 87f1b55d29f..5e71ee79684 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -49,7 +49,6 @@ #include #include #include -#include #include #include #include @@ -976,8 +975,8 @@ int Server::main(const std::vector & /*args*/) global_context->initializePageStorageMode(global_context->getPathPool(), STORAGE_FORMAT_CURRENT.page); // Use pd address to define which default_database we use by default. - // For mock test, we use "default". For deployed with pd/tidb/tikv use "system", which is always exist in TiFlash. - std::string default_database = config().getString("default_database", raft_config.pd_addrs.empty() ? "default" : "system"); + // For deployed with pd/tidb/tikv use "system", which is always exist in TiFlash. + std::string default_database = config().getString("default_database", "system"); Strings all_normal_path = storage_config.getAllNormalPaths(); const std::string path = all_normal_path[0]; global_context->setPath(path); @@ -1206,27 +1205,8 @@ int Server::main(const std::vector & /*args*/) global_context->initializeSharedBlockSchemas(); - { - // Note that this must do before initialize schema sync service. - do - { - // Check whether we need to upgrade directories hierarchy - // If some database can not find in TiDB, they will be dropped - // if theirs name is not in reserved_databases. - // Besides, database engine in reserved_databases just keep as - // what they are. - IDAsPathUpgrader upgrader( - *global_context, - /*is_mock=*/raft_config.pd_addrs.empty(), - /*reserved_databases=*/raft_config.ignore_databases); - if (!upgrader.needUpgrade()) - break; - upgrader.doUpgrade(); - } while (false); - - /// Then, load remaining databases - loadMetadata(*global_context); - } + // Load remaining databases + loadMetadata(*global_context); LOG_DEBUG(log, "Load metadata done."); if (!global_context->isDisaggregatedComputeMode()) diff --git a/dbms/src/Storages/DeltaMerge/DeltaIndexManager.h b/dbms/src/Storages/DeltaMerge/DeltaIndexManager.h index 48c8424ea13..60740424562 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaIndexManager.h +++ b/dbms/src/Storages/DeltaMerge/DeltaIndexManager.h @@ -50,7 +50,7 @@ class DeltaIndexManager size_t current_size = 0; const size_t max_size; - Poco::Logger * log; + LoggerPtr log; std::mutex mutex; @@ -60,7 +60,7 @@ class DeltaIndexManager public: explicit DeltaIndexManager(size_t max_size_) : max_size(max_size_) - , log(&Poco::Logger::get("DeltaIndexManager")) + , log(Logger::get()) {} /// Note that if isLimit() is false, than this method always return 0. diff --git a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp index c3dee70e21a..b27e5908cf6 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -102,13 +103,11 @@ String DMFile::ngcPath() const DMFilePtr DMFile::create(UInt64 file_id, const String & parent_path, DMConfigurationOpt configuration) { - Poco::Logger * log = &Poco::Logger::get("DMFile"); // On create, ref_id is the same as file_id. DMFilePtr new_dmfile(new DMFile(file_id, file_id, parent_path, Status::WRITABLE, - log, std::move(configuration))); auto path = new_dmfile->path(); @@ -116,7 +115,7 @@ DMFilePtr DMFile::create(UInt64 file_id, const String & parent_path, DMConfigura if (file.exists()) { file.remove(true); - LOG_WARNING(log, "Existing dmfile, removed: {}", path); + LOG_WARNING(Logger::get(), "Existing dmfile, removed: {}", path); } file.createDirectories(); @@ -146,8 +145,7 @@ DMFilePtr DMFile::restore( file_id, page_id, parent_path, - Status::READABLE, - &Poco::Logger::get("DMFile"))); + Status::READABLE)); if (!read_meta_mode.isNone()) { dmfile->readConfiguration(file_provider); @@ -395,7 +393,7 @@ void DMFile::readColumnStat(const FileProviderPtr & file_provider, const MetaPac } else { - log->warning(fmt::format("checksum for {} not found", name)); + LOG_WARNING(log, "checksum for {} not found", name); } } @@ -486,7 +484,7 @@ void DMFile::readPackProperty(const FileProviderPtr & file_provider, const MetaP } else { - log->warning(fmt::format("checksum for {} not found", name)); + LOG_WARNING(log, "checksum for {} not found", name); } } } @@ -533,7 +531,7 @@ void DMFile::finalizeForFolderMode(const FileProviderPtr & file_provider, const { if (STORAGE_FORMAT_CURRENT.dm_file >= DMFileFormat::V2 && !configuration) { - log->warning("checksum disabled due to lack of configuration"); + LOG_WARNING(log, "checksum disabled due to lack of configuration"); } writeMetadata(file_provider, write_limiter); if (unlikely(status != Status::WRITING)) @@ -569,7 +567,6 @@ std::set DMFile::listAllInPath( std::vector file_names; folder.list(file_names); std::set file_ids; - Poco::Logger * log = &Poco::Logger::get("DMFile"); auto try_parse_file_id = [](const String & name) -> std::optional { std::vector ss; @@ -597,7 +594,7 @@ std::set DMFile::listAllInPath( auto res = try_parse_file_id(name); if (!res) { - LOG_INFO(log, "Unrecognized temporary or dropped dmfile, ignored: {}", name); + LOG_INFO(Logger::get(), "Unrecognized temporary or dropped dmfile, ignored: {}", name); continue; } UInt64 file_id = *res; @@ -606,7 +603,7 @@ std::set DMFile::listAllInPath( const auto full_path = parent_path + "/" + name; if (Poco::File file(full_path); file.exists()) file.remove(true); - LOG_WARNING(log, "Existing temporary or dropped dmfile, removed: {}", full_path); + LOG_WARNING(Logger::get(), "Existing temporary or dropped dmfile, removed: {}", full_path); continue; } } @@ -619,7 +616,7 @@ std::set DMFile::listAllInPath( auto res = try_parse_file_id(name); if (!res) { - LOG_INFO(log, "Unrecognized DM file, ignored: {}", name); + LOG_INFO(Logger::get(), "Unrecognized DM file, ignored: {}", name); continue; } UInt64 file_id = *res; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFile.h b/dbms/src/Storages/DeltaMerge/File/DMFile.h index 06cf1f9502f..3a45597921a 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFile.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFile.h @@ -287,14 +287,13 @@ class DMFile : private boost::noncopyable UInt64 page_id_, String parent_path_, Status status_, - Poco::Logger * log_, DMConfigurationOpt configuration_ = std::nullopt) : file_id(file_id_) , page_id(page_id_) , parent_path(std::move(parent_path_)) , status(status_) , configuration(std::move(configuration_)) - , log(log_) + , log(Logger::get()) { } @@ -389,7 +388,7 @@ class DMFile : private boost::noncopyable SubFileStats sub_file_stats; - Poco::Logger * log; + LoggerPtr log; friend class DMFileWriter; friend class DMFileReader; diff --git a/dbms/src/Storages/DeltaMerge/ReadThread/CPU.cpp b/dbms/src/Storages/DeltaMerge/ReadThread/CPU.cpp index a946768795c..5f8684893e3 100644 --- a/dbms/src/Storages/DeltaMerge/ReadThread/CPU.cpp +++ b/dbms/src/Storages/DeltaMerge/ReadThread/CPU.cpp @@ -11,10 +11,11 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + #include +#include #include #include -#include #include #include @@ -93,7 +94,7 @@ std::vector> getLinuxNumaNodes() return numa_nodes; } -std::vector> getNumaNodes(Poco::Logger * log) +std::vector> getNumaNodes(const LoggerPtr & log) { #ifndef __APPLE__ // Apple macbooks does not support NUMA try diff --git a/dbms/src/Storages/DeltaMerge/ReadThread/CPU.h b/dbms/src/Storages/DeltaMerge/ReadThread/CPU.h index 2d1f564a43a..27439770ec2 100644 --- a/dbms/src/Storages/DeltaMerge/ReadThread/CPU.h +++ b/dbms/src/Storages/DeltaMerge/ReadThread/CPU.h @@ -13,12 +13,16 @@ // limitations under the License. #pragma once -#include - #include +namespace DB +{ +class Logger; +using LoggerPtr = std::shared_ptr; +} // namespace DB + namespace DB::DM { // `getNumaNodes` returns cpus of each Numa node. -std::vector> getNumaNodes(Poco::Logger * log); -} // namespace DB::DM \ No newline at end of file +std::vector> getNumaNodes(const LoggerPtr & log); +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/ReadThread/MergedTask.h b/dbms/src/Storages/DeltaMerge/ReadThread/MergedTask.h index 42e88f01f1e..128ef537cf5 100644 --- a/dbms/src/Storages/DeltaMerge/ReadThread/MergedTask.h +++ b/dbms/src/Storages/DeltaMerge/ReadThread/MergedTask.h @@ -47,7 +47,7 @@ class MergedTask , inited(false) , cur_idx(-1) , finished_count(0) - , log(&Poco::Logger::get("MergedTask")) + , log(Logger::get()) { passive_merged_segments.fetch_add(units.size() - 1, std::memory_order_relaxed); GET_METRIC(tiflash_storage_read_thread_gauge, type_merged_task).Increment(); @@ -142,7 +142,7 @@ class MergedTask bool inited; int cur_idx; size_t finished_count; - Poco::Logger * log; + LoggerPtr log; Stopwatch sw; inline static std::atomic passive_merged_segments{0}; }; @@ -156,7 +156,7 @@ class MergedTaskPool { public: MergedTaskPool() - : log(&Poco::Logger::get("MergedTaskPool")) + : log(Logger::get()) {} MergedTaskPtr pop(uint64_t pool_id); @@ -166,6 +166,6 @@ class MergedTaskPool private: std::mutex mtx; std::list merged_task_pool; - Poco::Logger * log; + LoggerPtr log; }; -} // namespace DB::DM \ No newline at end of file +} // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReadTaskScheduler.cpp b/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReadTaskScheduler.cpp index 99dbbc8caab..11d5e7da6a2 100644 --- a/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReadTaskScheduler.cpp +++ b/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReadTaskScheduler.cpp @@ -19,7 +19,7 @@ namespace DB::DM { SegmentReadTaskScheduler::SegmentReadTaskScheduler() : stop(false) - , log(&Poco::Logger::get("SegmentReadTaskScheduler")) + , log(Logger::get()) { sched_thread = std::thread(&SegmentReadTaskScheduler::schedLoop, this); } diff --git a/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReadTaskScheduler.h b/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReadTaskScheduler.h index c319275fd0f..5373e226609 100644 --- a/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReadTaskScheduler.h +++ b/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReadTaskScheduler.h @@ -78,6 +78,6 @@ class SegmentReadTaskScheduler std::atomic stop; std::thread sched_thread; - Poco::Logger * log; + LoggerPtr log; }; } // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReader.cpp b/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReader.cpp index 2b090e7913d..5981ec05eb9 100644 --- a/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReader.cpp +++ b/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReader.cpp @@ -11,6 +11,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + +#include #include #include #include @@ -29,7 +31,7 @@ class SegmentReader SegmentReader(WorkQueue & task_queue_, const std::vector & cpus_) : task_queue(task_queue_) , stop(false) - , log(&Poco::Logger::get(name)) + , log(Logger::get()) , cpus(cpus_) { t = std::thread(&SegmentReader::run, this); @@ -155,7 +157,7 @@ class SegmentReader WorkQueue & task_queue; std::atomic stop; - Poco::Logger * log; + LoggerPtr log; std::thread t; std::vector cpus; }; @@ -171,7 +173,7 @@ void SegmentReaderPool::addTask(MergedTaskPtr && task) } SegmentReaderPool::SegmentReaderPool(int thread_count, const std::vector & cpus) - : log(&Poco::Logger::get("SegmentReaderPool")) + : log(Logger::get()) { LOG_INFO(log, "Create start, thread_count={} cpus={}", thread_count, cpus); for (int i = 0; i < thread_count; i++) @@ -203,7 +205,7 @@ std::vector SegmentReaderPool::getReaderIds() const // ===== SegmentReaderPoolManager ===== // SegmentReaderPoolManager::SegmentReaderPoolManager() - : log(&Poco::Logger::get("SegmentReaderPoolManager")) + : log(Logger::get()) {} SegmentReaderPoolManager::~SegmentReaderPoolManager() = default; diff --git a/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReader.h b/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReader.h index 65d24865d3d..f57095fa623 100644 --- a/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReader.h +++ b/dbms/src/Storages/DeltaMerge/ReadThread/SegmentReader.h @@ -17,7 +17,12 @@ #include #include #include -#include + +namespace DB +{ +class Logger; +using LoggerPtr = std::shared_ptr; +} // namespace DB namespace DB::DM { @@ -43,7 +48,7 @@ class SegmentReaderPool WorkQueue task_queue; std::vector readers; - Poco::Logger * log; + LoggerPtr log; }; // SegmentReaderPoolManager is a NUMA-aware singleton that manages several SegmentReaderPool objects. @@ -74,7 +79,7 @@ class SegmentReaderPoolManager SegmentReaderPoolManager(); std::vector> reader_pools; std::unordered_set reader_ids; - Poco::Logger * log; + LoggerPtr log; }; } // namespace DB::DM diff --git a/dbms/src/Storages/DeltaMerge/SchemaUpdate.cpp b/dbms/src/Storages/DeltaMerge/SchemaUpdate.cpp index dd26d1b0637..52dbfe25fee 100644 --- a/dbms/src/Storages/DeltaMerge/SchemaUpdate.cpp +++ b/dbms/src/Storages/DeltaMerge/SchemaUpdate.cpp @@ -221,8 +221,6 @@ void applyAlter(ColumnDefines & table_columns, /// Caller should ensure the command is legal. /// eg. The column to modify/drop/rename must exist, the column to add must not exist, the new column name of rename must not exists. - Poco::Logger * log = &Poco::Logger::get("SchemaUpdate"); - if (command.type == AlterCommand::MODIFY_COLUMN) { // find column define and then apply modify @@ -248,7 +246,7 @@ void applyAlter(ColumnDefines & table_columns, { // Fall back to find column by name, this path should only call by tests. LOG_WARNING( - log, + Logger::get(), "Try to apply alter to column: {}, id: {}," " but not found by id, fall back locating col by name.", command.column_name, @@ -311,7 +309,7 @@ void applyAlter(ColumnDefines & table_columns, } else { - LOG_WARNING(log, "receive unknown alter command, type: {}", static_cast(command.type)); + LOG_WARNING(Logger::get(), "receive unknown alter command, type: {}", static_cast(command.type)); } } diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 1d3016e2853..87271b0e889 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -61,7 +61,7 @@ class IStorage : public std::enable_shared_from_this , public ITableDeclaration { public: - /// The main name of the table type (for example, StorageMergeTree). + /// The main name of the table type (for example, StorageDeltaMerge). virtual std::string getName() const = 0; /** The name of the table. diff --git a/dbms/src/Storages/MarkCache.h b/dbms/src/Storages/MarkCache.h index fe96636a046..c3167486181 100644 --- a/dbms/src/Storages/MarkCache.h +++ b/dbms/src/Storages/MarkCache.h @@ -42,7 +42,7 @@ struct MarksWeightFunction }; -/** Cache of 'marks' for StorageMergeTree. +/** Cache of 'marks' for StorageDeltaMerge. * Marks is an index structure that addresses ranges in column file, corresponding to ranges of primary key. */ class MarkCache : public LRUCache, MarksWeightFunction> diff --git a/dbms/src/Storages/Page/V2/PageFile.cpp b/dbms/src/Storages/Page/V2/PageFile.cpp index e99f96d7cc0..b6dfd57fde8 100644 --- a/dbms/src/Storages/Page/V2/PageFile.cpp +++ b/dbms/src/Storages/Page/V2/PageFile.cpp @@ -94,7 +94,7 @@ std::pair genWriteData( // meta_write_bytes += sizeof(WBSize) + sizeof(PageFormat::Version) + sizeof(WriteBatch::SequenceID); - for (auto & write : wb.getWrites()) + for (auto & write : wb.getMutWrites()) { meta_write_bytes += sizeof(IsPut); // We don't serialize `PUT_EXTERNAL` for V2, just convert it to `PUT` @@ -138,7 +138,7 @@ std::pair genWriteData( // PageUtil::put(meta_pos, wb.getSequence()); PageOffset page_data_file_off = page_file.getDataFileAppendPos(); - for (auto & write : wb.getWrites()) + for (auto & write : wb.getMutWrites()) { // We don't serialize `PUT_EXTERNAL` for V2, just convert it to `PUT` if (write.type == WriteBatchWriteType::PUT_EXTERNAL) diff --git a/dbms/src/Storages/Page/V3/BlobStore.cpp b/dbms/src/Storages/Page/V3/BlobStore.cpp index 8da5bcafd5b..4aa683d6bb1 100644 --- a/dbms/src/Storages/Page/V3/BlobStore.cpp +++ b/dbms/src/Storages/Page/V3/BlobStore.cpp @@ -168,7 +168,7 @@ typename BlobStore::PageEntriesEdit BlobStore::handleLargeWrite(typename Trait::WriteBatch & wb, const WriteLimiterPtr & write_limiter) { PageEntriesEdit edit; - for (auto & write : wb.getWrites()) + for (auto & write : wb.getMutWrites()) { switch (write.type) { @@ -315,7 +315,7 @@ BlobStore::write(typename Trait::WriteBatch & wb, const WriteLimiterPtr & size_t offset_in_allocated = 0; - for (auto & write : wb.getWrites()) + for (auto & write : wb.getMutWrites()) { switch (write.type) { diff --git a/dbms/src/Storages/Page/V3/PageDirectory.cpp b/dbms/src/Storages/Page/V3/PageDirectory.cpp index dfb9f1c740c..1bb6d5e0dce 100644 --- a/dbms/src/Storages/Page/V3/PageDirectory.cpp +++ b/dbms/src/Storages/Page/V3/PageDirectory.cpp @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -442,7 +443,7 @@ VersionedPageEntries::resolveToPageId(UInt64 seq, bool ignore_delete, Pag } else { - LOG_WARNING(&Poco::Logger::get("VersionedPageEntries"), "Can't resolve the EditRecordType {}", static_cast(type)); + LOG_WARNING(Logger::get(), "Can't resolve the EditRecordType {}", magic_enum::enum_name(type)); } return {ResolveResult::FAIL, Trait::PageIdTrait::getInvalidID(), PageVersion(0)}; diff --git a/dbms/src/Storages/Page/V3/Universal/UniversalWriteBatch.h b/dbms/src/Storages/Page/V3/Universal/UniversalWriteBatch.h index bfb09307e40..14fe6b891dc 100644 --- a/dbms/src/Storages/Page/V3/Universal/UniversalWriteBatch.h +++ b/dbms/src/Storages/Page/V3/Universal/UniversalWriteBatch.h @@ -61,17 +61,14 @@ class UniversalWriteBatch : private boost::noncopyable offsets.emplace_back(off, 0); off += data_sz; } - if (unlikely(!data_sizes.empty() && off != size)) - { - throw Exception(fmt::format( - "Try to put Page with fields, but page size and fields total size not match " - "[page_id={}] [num_fields={}] [page_size={}] [all_fields_size={}]", - page_id, - data_sizes.size(), - size, - off), - ErrorCodes::LOGICAL_ERROR); - } + + RUNTIME_CHECK_MSG(data_sizes.empty() || off == size, + "Try to put Page with fields, but page size and fields total size not match " + "[page_id={}] [num_fields={}] [page_size={}] [all_fields_size={}]", + page_id, + data_sizes.size(), + size, + off); Write w{WriteBatchWriteType::PUT, page_id, tag, read_buffer, size, "", std::move(offsets)}; total_data_size += size; @@ -113,7 +110,7 @@ class UniversalWriteBatch : private boost::noncopyable { return writes; } - Writes & getWrites() + Writes & getMutWrites() { return writes; } diff --git a/dbms/src/Storages/Page/WriteBatch.h b/dbms/src/Storages/Page/WriteBatch.h index cb6be591be3..209322ef68b 100644 --- a/dbms/src/Storages/Page/WriteBatch.h +++ b/dbms/src/Storages/Page/WriteBatch.h @@ -173,7 +173,7 @@ class WriteBatch : private boost::noncopyable { return writes; } - Writes & getWrites() + Writes & getMutWrites() { return writes; } diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp index 0e572b13573..bfe7cbc125d 100644 --- a/libs/libdaemon/src/BaseDaemon.cpp +++ b/libs/libdaemon/src/BaseDaemon.cpp @@ -1156,11 +1156,12 @@ void BaseDaemon::initialize(Application & self) void BaseDaemon::logRevision() const { - Logger::root().information("Welcome to TiFlash"); - Logger::root().information("Starting daemon with revision " + Poco::NumberFormatter::format(ClickHouseRevision::get())); + auto * log = &Logger::root(); + LOG_INFO(log, "Welcome to TiFlash"); + LOG_INFO(log, "Starting daemon with revision " + Poco::NumberFormatter::format(ClickHouseRevision::get())); std::stringstream ss; TiFlashBuildInfo::outputDetail(ss); - Logger::root().information("TiFlash build info: " + ss.str()); + LOG_INFO(log, "TiFlash build info: {}", ss.str()); } /// Used for exitOnTaskError() diff --git a/tests/_env.sh b/tests/_env.sh index 7803181c4a1..09e20cb5671 100644 --- a/tests/_env.sh +++ b/tests/_env.sh @@ -36,7 +36,7 @@ export storage_server="127.0.0.1" export storage_port="9000" # Default database for scripts -export storage_db="default" +export storage_db="system" # TiDB address export tidb_server="127.0.0.1" diff --git a/tests/delta-merge-test/query/misc/timestamp_rough_set_filter.test b/tests/delta-merge-test/query/misc/timestamp_rough_set_filter.test index b7a70d25fa6..a956f9a2775 100644 --- a/tests/delta-merge-test/query/misc/timestamp_rough_set_filter.test +++ b/tests/delta-merge-test/query/misc/timestamp_rough_set_filter.test @@ -18,11 +18,14 @@ => DBGInvoke __enable_schema_sync_service('true') => DBGInvoke __drop_tidb_table(default, test) +=> DBGInvoke __drop_tidb_db(default) => drop table if exists default.test +=> drop database if exists default => DBGInvoke __set_flush_threshold(1000000, 1000000) # Data. +=> DBGInvoke __mock_tidb_db(default) => DBGInvoke __mock_tidb_table(default, test, 'col_1 Int64, col_2 default \'asTiDBType|timestamp(5)\'') => DBGInvoke __refresh_schemas() => DBGInvoke __put_region(4, 0, 100, default, test) diff --git a/tests/delta-merge-test/run.sh b/tests/delta-merge-test/run.sh index 53e49d2d418..cd841ebd949 100755 --- a/tests/delta-merge-test/run.sh +++ b/tests/delta-merge-test/run.sh @@ -22,11 +22,7 @@ set -xe check_env -# We need to separate mock-test for dt and tmt, since this behavior -# is different in some tests -# * "tmt" engine ONLY support disable_bg_flush = false. -# * "dt" engine ONLY support disable_bg_flush = true. -# (only tics0 up) (for engine DetlaTree) +# (only tics0 up) docker-compose -f mock-test-dt.yaml down clean_data_log diff --git a/tests/docker/_env.sh b/tests/docker/_env.sh index 1b92af55bb9..abe39c3c699 100755 --- a/tests/docker/_env.sh +++ b/tests/docker/_env.sh @@ -26,7 +26,7 @@ export storage_server="127.0.0.1" export storage_port="9000" # Default database for scripts -export storage_db="default" +export storage_db="system" # TiDB address export tidb_server="tidb0" diff --git a/tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test%2Dtbl/stable/.gitignore b/tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test%2Dtbl/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test-tbl/data/.gitignore b/tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test-tbl/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test-tbl/log/.gitignore b/tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test-tbl/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test-tbl/meta/.gitignore b/tests/testdata/flash-1136-v3.1.0/data0/data/test%2Ddb/test-tbl/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136-v3.1.0/data0/data/test/test%2Dtbl/stable/.gitignore b/tests/testdata/flash-1136-v3.1.0/data0/data/test/test%2Dtbl/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136-v3.1.0/data0/data/test/test-tbl/data/.gitignore b/tests/testdata/flash-1136-v3.1.0/data0/data/test/test-tbl/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136-v3.1.0/data0/data/test/test-tbl/log/.gitignore b/tests/testdata/flash-1136-v3.1.0/data0/data/test/test-tbl/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136-v3.1.0/data0/data/test/test-tbl/meta/.gitignore b/tests/testdata/flash-1136-v3.1.0/data0/data/test/test-tbl/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb.sql b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb.sql deleted file mode 100644 index c97c56faf67..00000000000 --- a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH DATABASE `test-db` -ENGINE = Ordinary diff --git a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb/test%2Dtbl.sql b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb/test%2Dtbl.sql deleted file mode 100644 index 18be51be751..00000000000 --- a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb/test%2Dtbl.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE `test-tbl` -( - pk Nullable(Int32), - _tidb_rowid Int64 -) -ENGINE = DeltaMerge(_tidb_rowid, '{"cols":[{"comment":"","default":null,"id":1,"name":{"L":"pk","O":"pk"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":66,"name":{"L":"test-tbl","O":"test-tbl"},"partition":null,"pk_is_handle":false,"schema_version":36,"state":5,"update_timestamp":417293448885567496}') diff --git a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test.sql b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test.sql deleted file mode 100644 index 897b444f60b..00000000000 --- a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH DATABASE `test` -ENGINE = Ordinary diff --git a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test/test%2Dtbl.sql b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test/test%2Dtbl.sql deleted file mode 100644 index f6e5c321a91..00000000000 --- a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test/test%2Dtbl.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE `test-tbl` -( - pk Nullable(Int32), - _tidb_rowid Int64 -) -ENGINE = DeltaMerge(_tidb_rowid, '{"cols":[{"comment":"","default":null,"id":1,"name":{"L":"pk","O":"pk"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":666,"name":{"L":"test-tbl","O":"test-tbl"},"partition":null,"pk_is_handle":false,"schema_version":36,"state":5,"update_timestamp":417293448885567496}') diff --git a/tests/testdata/flash-1136-v3.1.0/data1/data/test%2Ddb/test%2Dtbl/stable/.gitignore b/tests/testdata/flash-1136-v3.1.0/data1/data/test%2Ddb/test%2Dtbl/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136/data0/data/test%2Ddb/test-tbl/data/.gitignore b/tests/testdata/flash-1136/data0/data/test%2Ddb/test-tbl/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136/data0/data/test%2Ddb/test-tbl/log/.gitignore b/tests/testdata/flash-1136/data0/data/test%2Ddb/test-tbl/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136/data0/data/test%2Ddb/test-tbl/meta/.gitignore b/tests/testdata/flash-1136/data0/data/test%2Ddb/test-tbl/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136/data0/data/test-db/test-tbl/stable/.gitignore b/tests/testdata/flash-1136/data0/data/test-db/test-tbl/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136/data0/data/test/test-tbl/data/.gitignore b/tests/testdata/flash-1136/data0/data/test/test-tbl/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136/data0/data/test/test-tbl/log/.gitignore b/tests/testdata/flash-1136/data0/data/test/test-tbl/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136/data0/data/test/test-tbl/meta/.gitignore b/tests/testdata/flash-1136/data0/data/test/test-tbl/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136/data0/data/test/test-tbl/stable/.gitignore b/tests/testdata/flash-1136/data0/data/test/test-tbl/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/flash-1136/data0/metadata/test%2Ddb.sql b/tests/testdata/flash-1136/data0/metadata/test%2Ddb.sql deleted file mode 100644 index c97c56faf67..00000000000 --- a/tests/testdata/flash-1136/data0/metadata/test%2Ddb.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH DATABASE `test-db` -ENGINE = Ordinary diff --git a/tests/testdata/flash-1136/data0/metadata/test%2Ddb/test%2Dtbl.sql b/tests/testdata/flash-1136/data0/metadata/test%2Ddb/test%2Dtbl.sql deleted file mode 100644 index 18be51be751..00000000000 --- a/tests/testdata/flash-1136/data0/metadata/test%2Ddb/test%2Dtbl.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE `test-tbl` -( - pk Nullable(Int32), - _tidb_rowid Int64 -) -ENGINE = DeltaMerge(_tidb_rowid, '{"cols":[{"comment":"","default":null,"id":1,"name":{"L":"pk","O":"pk"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":66,"name":{"L":"test-tbl","O":"test-tbl"},"partition":null,"pk_is_handle":false,"schema_version":36,"state":5,"update_timestamp":417293448885567496}') diff --git a/tests/testdata/flash-1136/data0/metadata/test.sql b/tests/testdata/flash-1136/data0/metadata/test.sql deleted file mode 100644 index 897b444f60b..00000000000 --- a/tests/testdata/flash-1136/data0/metadata/test.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH DATABASE `test` -ENGINE = Ordinary diff --git a/tests/testdata/flash-1136/data0/metadata/test/test%2Dtbl.sql b/tests/testdata/flash-1136/data0/metadata/test/test%2Dtbl.sql deleted file mode 100644 index f6e5c321a91..00000000000 --- a/tests/testdata/flash-1136/data0/metadata/test/test%2Dtbl.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE `test-tbl` -( - pk Nullable(Int32), - _tidb_rowid Int64 -) -ENGINE = DeltaMerge(_tidb_rowid, '{"cols":[{"comment":"","default":null,"id":1,"name":{"L":"pk","O":"pk"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":666,"name":{"L":"test-tbl","O":"test-tbl"},"partition":null,"pk_is_handle":false,"schema_version":36,"state":5,"update_timestamp":417293448885567496}') diff --git a/tests/testdata/flash-1136/data1/data/test-db/test-tbl/stable/.gitignore b/tests/testdata/flash-1136/data1/data/test-db/test-tbl/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/issue-941/data0/data/test/#hello-world/data/.gitignore b/tests/testdata/issue-941/data0/data/test/#hello-world/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/issue-941/data0/data/test/#hello-world/log/.gitignore b/tests/testdata/issue-941/data0/data/test/#hello-world/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/issue-941/data0/data/test/#hello-world/meta/.gitignore b/tests/testdata/issue-941/data0/data/test/#hello-world/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/issue-941/data0/data/test/#hello-world/stable/.gitignore b/tests/testdata/issue-941/data0/data/test/#hello-world/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/issue-941/data0/metadata/test.sql b/tests/testdata/issue-941/data0/metadata/test.sql deleted file mode 100644 index 897b444f60b..00000000000 --- a/tests/testdata/issue-941/data0/metadata/test.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH DATABASE `test` -ENGINE = Ordinary diff --git a/tests/testdata/issue-941/data0/metadata/test/%23hello%2Dworld.sql b/tests/testdata/issue-941/data0/metadata/test/%23hello%2Dworld.sql deleted file mode 100644 index f93ab0a91b4..00000000000 --- a/tests/testdata/issue-941/data0/metadata/test/%23hello%2Dworld.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE `#hello-world` -( - t Nullable(Int32), - _tidb_rowid Int64 -) -ENGINE = DeltaMerge(_tidb_rowid, '{"cols":[{"comment":"","default":null,"id":1,"name":{"L":"t","O":"t"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":45,"name":{"L":"#hello-world","O":"#hello-world"},"partition":null,"pk_is_handle":false,"schema_version":23,"state":5,"update_timestamp":418360557052624904}') diff --git a/tests/testdata/issue-941/data1/data/test/#hello-world/stable/.gitignore b/tests/testdata/issue-941/data1/data/test/#hello-world/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/abc/data/.gitignore b/tests/testdata/oncall-1651/db/data/test/abc/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/abc/log/.gitignore b/tests/testdata/oncall-1651/db/data/test/abc/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/abc/meta/.gitignore b/tests/testdata/oncall-1651/db/data/test/abc/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/abc/stable/.gitignore b/tests/testdata/oncall-1651/db/data/test/abc/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak/data/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak/log/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak/meta/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak/stable/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_49/data/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_49/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_49/log/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_49/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_49/meta/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_49/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_49/stable/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_49/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_50/data/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_50/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_50/log/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_50/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_50/meta/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_50/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_50/stable/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_50/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_51/data/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_51/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_51/log/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_51/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_51/meta/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_51/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_51/stable/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_51/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_52/data/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_52/data/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_52/log/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_52/log/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_52/meta/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_52/meta/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/data/test/emp_bak_52/stable/.gitignore b/tests/testdata/oncall-1651/db/data/test/emp_bak_52/stable/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/testdata/oncall-1651/db/metadata/test.sql b/tests/testdata/oncall-1651/db/metadata/test.sql deleted file mode 100644 index a92a7db72c5..00000000000 --- a/tests/testdata/oncall-1651/db/metadata/test.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH DATABASE test -ENGINE = Ordinary diff --git a/tests/testdata/oncall-1651/db/metadata/test/abc.sql b/tests/testdata/oncall-1651/db/metadata/test/abc.sql deleted file mode 100644 index 9ea2b02b274..00000000000 --- a/tests/testdata/oncall-1651/db/metadata/test/abc.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE abc -( - pk Nullable(Int32), - _tidb_rowid Int64 -) -ENGINE = DeltaMerge(_tidb_rowid, '{"cols":[{"comment":"","default":null,"id":1,"name":{"L":"pk","O":"pk"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":45,"name":{"L":"aaa","O":"aaa"},"partition":null,"pk_is_handle":false,"schema_version":23,"state":5,"update_timestamp":417160204721061891}') diff --git a/tests/testdata/oncall-1651/db/metadata/test/emp_bak.sql b/tests/testdata/oncall-1651/db/metadata/test/emp_bak.sql deleted file mode 100644 index 4be86529f02..00000000000 --- a/tests/testdata/oncall-1651/db/metadata/test/emp_bak.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE emp_bak -( - id Int32, - fname String, - lname String, - store_id Int32, - department_id Int32 -) -ENGINE = DeltaMerge(id, '{"belonging_table_id":-1,"cols":[{"comment":"","default":null,"id":1,"name":{"L":"id","O":"id"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":515,"Flen":11,"Tp":3}},{"comment":"","default":null,"id":2,"name":{"L":"fname","O":"fname"},"offset":1,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":25,"Tp":15}},{"comment":"","default":null,"id":3,"name":{"L":"lname","O":"lname"},"offset":2,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":25,"Tp":15}},{"comment":"","default":null,"id":4,"name":{"L":"store_id","O":"store_id"},"offset":3,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":11,"Tp":3}},{"comment":"","default":null,"id":5,"name":{"L":"department_id","O":"department_id"},"offset":4,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":11,"Tp":3}}],"comment":"","id":48,"name":{"L":"emp","O":"emp"},"partition":{"definitions":[{"comment":"","id":49,"name":{"L":"p0","O":"p0"}},{"comment":"","id":50,"name":{"L":"p1","O":"p1"}},{"comment":"","id":51,"name":{"L":"p2","O":"p2"}},{"comment":"","id":52,"name":{"L":"p3","O":"p3"}}],"enable":true,"expr":"`id`","num":4,"type":1},"pk_is_handle":true,"schema_version":31,"state":5,"update_timestamp":417160573256204297}') diff --git a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_49.sql b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_49.sql deleted file mode 100644 index 315b7f24731..00000000000 --- a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_49.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE emp_bak_49 -( - id Int32, - fname String, - lname String, - store_id Int32, - department_id Int32 -) -ENGINE = DeltaMerge(id, '{"belonging_table_id":48,"cols":[{"comment":"","default":null,"id":1,"name":{"L":"id","O":"id"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":515,"Flen":11,"Tp":3}},{"comment":"","default":null,"id":2,"name":{"L":"fname","O":"fname"},"offset":1,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":25,"Tp":15}},{"comment":"","default":null,"id":3,"name":{"L":"lname","O":"lname"},"offset":2,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":25,"Tp":15}},{"comment":"","default":null,"id":4,"name":{"L":"store_id","O":"store_id"},"offset":3,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":11,"Tp":3}},{"comment":"","default":null,"id":5,"name":{"L":"department_id","O":"department_id"},"offset":4,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":11,"Tp":3}}],"comment":"","id":49,"is_partition_sub_table":true,"name":{"L":"employees_49","O":"employees_49"},"partition":{"definitions":[{"comment":"","id":49,"name":{"L":"p0","O":"p0"}},{"comment":"","id":50,"name":{"L":"p1","O":"p1"}},{"comment":"","id":51,"name":{"L":"p2","O":"p2"}},{"comment":"","id":52,"name":{"L":"p3","O":"p3"}}],"enable":true,"expr":"`id`","num":4,"type":1},"pk_is_handle":true,"schema_version":25,"state":5,"update_timestamp":417160265315647498}') diff --git a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_50.sql b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_50.sql deleted file mode 100644 index 055be9e4d91..00000000000 --- a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_50.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE emp_bak_50 -( - id Int32, - fname String, - lname String, - store_id Int32, - department_id Int32 -) -ENGINE = DeltaMerge(id, '{"belonging_table_id":48,"cols":[{"comment":"","default":null,"id":1,"name":{"L":"id","O":"id"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":515,"Flen":11,"Tp":3}},{"comment":"","default":null,"id":2,"name":{"L":"fname","O":"fname"},"offset":1,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":25,"Tp":15}},{"comment":"","default":null,"id":3,"name":{"L":"lname","O":"lname"},"offset":2,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":25,"Tp":15}},{"comment":"","default":null,"id":4,"name":{"L":"store_id","O":"store_id"},"offset":3,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":11,"Tp":3}},{"comment":"","default":null,"id":5,"name":{"L":"department_id","O":"department_id"},"offset":4,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":11,"Tp":3}}],"comment":"","id":50,"is_partition_sub_table":true,"name":{"L":"employees_50","O":"employees_50"},"partition":{"definitions":[{"comment":"","id":49,"name":{"L":"p0","O":"p0"}},{"comment":"","id":50,"name":{"L":"p1","O":"p1"}},{"comment":"","id":51,"name":{"L":"p2","O":"p2"}},{"comment":"","id":52,"name":{"L":"p3","O":"p3"}}],"enable":true,"expr":"`id`","num":4,"type":1},"pk_is_handle":true,"schema_version":25,"state":5,"update_timestamp":417160265315647498}') diff --git a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_51.sql b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_51.sql deleted file mode 100644 index f96af626e07..00000000000 --- a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_51.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE emp_bak_51 -( - id Int32, - fname String, - lname String, - store_id Int32, - department_id Int32 -) -ENGINE = DeltaMerge(id, '{"belonging_table_id":48,"cols":[{"comment":"","default":null,"id":1,"name":{"L":"id","O":"id"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":515,"Flen":11,"Tp":3}},{"comment":"","default":null,"id":2,"name":{"L":"fname","O":"fname"},"offset":1,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":25,"Tp":15}},{"comment":"","default":null,"id":3,"name":{"L":"lname","O":"lname"},"offset":2,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":25,"Tp":15}},{"comment":"","default":null,"id":4,"name":{"L":"store_id","O":"store_id"},"offset":3,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":11,"Tp":3}},{"comment":"","default":null,"id":5,"name":{"L":"department_id","O":"department_id"},"offset":4,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":11,"Tp":3}}],"comment":"","id":51,"is_partition_sub_table":true,"name":{"L":"employees_51","O":"employees_51"},"partition":{"definitions":[{"comment":"","id":49,"name":{"L":"p0","O":"p0"}},{"comment":"","id":50,"name":{"L":"p1","O":"p1"}},{"comment":"","id":51,"name":{"L":"p2","O":"p2"}},{"comment":"","id":52,"name":{"L":"p3","O":"p3"}}],"enable":true,"expr":"`id`","num":4,"type":1},"pk_is_handle":true,"schema_version":25,"state":5,"update_timestamp":417160265315647498}') diff --git a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_52.sql b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_52.sql deleted file mode 100644 index 6e1bb90b0fb..00000000000 --- a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_52.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Copyright 2022 PingCAP, Ltd. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ATTACH TABLE emp_bak_52 -( - id Int32, - fname String, - lname String, - store_id Int32, - department_id Int32 -) -ENGINE = DeltaMerge(id, '{"belonging_table_id":48,"cols":[{"comment":"","default":null,"id":1,"name":{"L":"id","O":"id"},"offset":0,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":515,"Flen":11,"Tp":3}},{"comment":"","default":null,"id":2,"name":{"L":"fname","O":"fname"},"offset":1,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":25,"Tp":15}},{"comment":"","default":null,"id":3,"name":{"L":"lname","O":"lname"},"offset":2,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":25,"Tp":15}},{"comment":"","default":null,"id":4,"name":{"L":"store_id","O":"store_id"},"offset":3,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":11,"Tp":3}},{"comment":"","default":null,"id":5,"name":{"L":"department_id","O":"department_id"},"offset":4,"origin_default":null,"state":5,"type":{"Decimal":0,"Elems":null,"Flag":4097,"Flen":11,"Tp":3}}],"comment":"","id":52,"is_partition_sub_table":true,"name":{"L":"employees_52","O":"employees_52"},"partition":{"definitions":[{"comment":"","id":49,"name":{"L":"p0","O":"p0"}},{"comment":"","id":50,"name":{"L":"p1","O":"p1"}},{"comment":"","id":51,"name":{"L":"p2","O":"p2"}},{"comment":"","id":52,"name":{"L":"p3","O":"p3"}}],"enable":true,"expr":"`id`","num":4,"type":1},"pk_is_handle":true,"schema_version":25,"state":5,"update_timestamp":417160265315647498}')