Merge pull request duckdb#9967 from taniabogatsch/block-size

[Block Size] CI test for 16KB block size and related code changes

.github/workflows/NightlyTests.yml

@@ -613,6 +613,33 @@ jobs:
       shell: bash
       run: python scripts/test_vector_sizes.py
 
+  block-sizes:
+    name: Block Sizes
+    runs-on: ubuntu-20.04
+    needs: linux-memory-leaks
+    env:
+      CC: gcc-10
+      CXX: g++-10
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.7'
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        with:
+          key: ${{ github.job }}
+          save: ${{ env.CCACHE_SAVE }}
+
+      - name: Test
+        shell: bash
+        run: python scripts/test_block_sizes.py
+
   linux-wasm-experimental:
     name: WebAssembly duckdb-wasm builds
     runs-on: ubuntu-22.04

benchmark/micro/CMakeLists.txt

@@ -1,13 +1,7 @@
 include_directories(../../third_party/sqlite/include)
 add_library(
-  duckdb_benchmark_micro OBJECT
-  append.cpp
-  append_mix.cpp
-  attach.cpp
-  bulkupdate.cpp
-  cast.cpp
-  in.cpp
-  storage.cpp)
+  duckdb_benchmark_micro OBJECT append.cpp append_mix.cpp bulkupdate.cpp
+                                cast.cpp in.cpp storage.cpp)
 
 set(BENCHMARK_OBJECT_FILES
     ${BENCHMARK_OBJECT_FILES} $<TARGET_OBJECTS:duckdb_benchmark_micro>

scripts/run_tests_one_by_one.py

@@ -1,7 +1,5 @@
 import sys
 import subprocess
-import re
-import os
 import time
 
 import argparse

scripts/test_block_sizes.py

@@ -0,0 +1,57 @@
+import os
+import re
+from python_helpers import open_utf8
+
+
+def execute_system_command(cmd):
+    print(cmd)
+    retcode = os.system(cmd)
+    print(retcode)
+    if retcode != 0:
+        raise Exception
+
+
+def replace_in_file(fname, regex, replace):
+    with open_utf8(fname, 'r') as f:
+        contents = f.read()
+    contents = re.sub(regex, replace, contents)
+    with open_utf8(fname, 'w+') as f:
+        f.write(contents)
+
+
+current_dir = os.getcwd()
+build_dir = os.path.join(os.getcwd(), 'build', 'release')
+
+# run the fast tests and all storage-related tests
+# with a block size of 16KB and a standard vector size
+block_size = 16384
+print("TESTING BLOCK_ALLOC_SIZE=%d" % (block_size,))
+print("TESTING STANDARD_VECTOR_SIZE")
+
+replace_in_file(
+    'src/include/duckdb/storage/storage_info.hpp',
+    r'constexpr static idx_t BLOCK_ALLOC_SIZE = \w+',
+    'constexpr static idx_t BLOCK_ALLOC_SIZE = %d' % (block_size,),
+)
+
+execute_system_command('rm -rf build')
+execute_system_command('make relassert')
+execute_system_command('build/relassert/test/unittest')
+execute_system_command('build/relassert/test/unittest "test/sql/storage/*"')
+
+# run the fast tests and all storage-related tests
+# with a block size of 16KB and a vector size of 512
+vector_size = 512
+print("TESTING BLOCK_ALLOC_SIZE=%d" % (block_size,))
+print("TESTING STANDARD_VECTOR_SIZE=%d" % (vector_size,))
+
+replace_in_file(
+    'src/include/duckdb/common/vector_size.hpp',
+    r'#define STANDARD_VECTOR_SIZE \w+',
+    '#define STANDARD_VECTOR_SIZE %d' % (vector_size,),
+)
+
+execute_system_command('rm -rf build')
+execute_system_command('make relassert')
+execute_system_command('build/relassert/test/unittest')
+execute_system_command('build/relassert/test/unittest "test/sql/storage/*"')

scripts/test_vector_sizes.py

@@ -27,7 +27,7 @@ def replace_in_file(fname, regex, replace):
     print("TESTING STANDARD_VECTOR_SIZE=%d" % (vector_size,))
     replace_in_file(
         'src/include/duckdb/common/vector_size.hpp',
-        r'#define STANDARD_VECTOR_SIZE \d+',
+        r'#define STANDARD_VECTOR_SIZE \w+',
         '#define STANDARD_VECTOR_SIZE %d' % (vector_size,),
     )
     execute_system_command('rm -rf build')

src/catalog/default/default_functions.cpp

@@ -162,6 +162,9 @@ static DefaultMacro internal_macros[] = {
 	// date functions
 	{DEFAULT_SCHEMA, "date_add", {"date", "interval", nullptr}, "date + interval"},
 
+    // storage helper functions
+    {DEFAULT_SCHEMA, "get_block_size", {"db_name"}, "(SELECT block_size FROM pragma_database_size() WHERE database_name = db_name)"},
+
 	{nullptr, nullptr, {nullptr}, nullptr}
 	};
 

src/common/constants.cpp

@@ -9,7 +9,6 @@ constexpr const idx_t DConstants::INVALID_INDEX;
 const row_t MAX_ROW_ID = 36028797018960000ULL;       // 2^55
 const row_t MAX_ROW_ID_LOCAL = 72057594037920000ULL; // 2^56
 const column_t COLUMN_IDENTIFIER_ROW_ID = (column_t)-1;
-const sel_t ZERO_VECTOR[STANDARD_VECTOR_SIZE] = {0};
 const double PI = 3.141592653589793;
 
 const transaction_t TRANSACTION_ID_START = 4611686018427388000ULL;                // 2^62

src/include/duckdb/common/helper.hpp

@@ -151,7 +151,7 @@ T MaxValue(T a, T b) {
 }
 
 template <typename T>
-T MinValue(T a, T b) {
+constexpr T MinValue(T a, T b) {
 	return a < b ? a : b;
 }
 
@@ -160,12 +160,17 @@ T AbsValue(T a) {
 	return a < 0 ? -a : a;
 }
 
-//Align value (ceiling)
+//! Align value (ceiling)
 template<class T, T val=8>
 static inline T AlignValue(T n) {
 	return ((n + (val - 1)) / val) * val;
 }
 
+template<class T, T val=8>
+constexpr inline T AlignValueFloor(T n) {
+	return (n / val) * val;
+}
+
 template<class T, T val=8>
 static inline bool ValueIsAligned(T n) {
 	return (n % val) == 0;

src/include/duckdb/common/vector_size.hpp

@@ -12,16 +12,16 @@
 
 namespace duckdb {
 
+//! The default standard vector size
+#define DEFAULT_STANDARD_VECTOR_SIZE 2048
+
 //! The vector size used in the execution engine
 #ifndef STANDARD_VECTOR_SIZE
-#define STANDARD_VECTOR_SIZE 2048
+#define STANDARD_VECTOR_SIZE DEFAULT_STANDARD_VECTOR_SIZE
 #endif
 
-#if ((STANDARD_VECTOR_SIZE & (STANDARD_VECTOR_SIZE - 1)) != 0)
-#error Vector size should be a power of two
+#if (STANDARD_VECTOR_SIZE & (STANDARD_VECTOR_SIZE - 1) != 0)
+#error The vector size must be a power of two
 #endif
 
-//! Zero selection vector: completely filled with the value 0 [READ ONLY]
-extern const sel_t ZERO_VECTOR[STANDARD_VECTOR_SIZE];
-
 } // namespace duckdb

src/include/duckdb/storage/checkpoint/string_checkpoint_state.hpp

@@ -39,7 +39,7 @@ struct string_location_t {
 	string_location_t() {
 	}
 	bool IsValid() {
-		return offset < Storage::BLOCK_SIZE && (block_id == INVALID_BLOCK || block_id >= MAXIMUM_BLOCK);
+		return offset < int32_t(Storage::BLOCK_SIZE) && (block_id == INVALID_BLOCK || block_id >= MAXIMUM_BLOCK);
 	}
 	block_id_t block_id;
 	int32_t offset;

src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -42,10 +42,10 @@ struct MetadataHandle {
 
 class MetadataManager {
 public:
-	//! The size of metadata blocks
-	static constexpr const idx_t METADATA_BLOCK_SIZE = 4088;
 	//! The amount of metadata blocks per storage block
 	static constexpr const idx_t METADATA_BLOCK_COUNT = 64;
+	//! The size of metadata blocks
+	static constexpr const idx_t METADATA_BLOCK_SIZE = AlignValueFloor(Storage::BLOCK_SIZE / METADATA_BLOCK_COUNT);
 
 public:
 	MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager);
@@ -88,4 +88,8 @@ class MetadataManager {
 	void ConvertToTransient(MetadataBlock &block);
 };
 
+//! Detect mismatching constant values
+static_assert(MetadataManager::METADATA_BLOCK_SIZE * MetadataManager::METADATA_BLOCK_COUNT <= Storage::BLOCK_SIZE,
+              "metadata block count exceeds total block alloc size");
+
 } // namespace duckdb