Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: merge blocks when prepare #6369

Merged
merged 7 commits into from
Nov 28, 2022
Merged

test: merge blocks when prepare #6369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d68847c
test: merge blocks when prepare
breezewish Nov 25, 2022
14c3b2a
Reformat
breezewish Nov 25, 2022
41e7971
Simplify code
breezewish Nov 25, 2022
d251a43
Simplify
breezewish Nov 26, 2022
46f667c
Merge branch 'master' into wenxuan/test_merge_blocks
breezewish Nov 26, 2022
9eab6fa
Update dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp
breezewish Nov 28, 2022
27ccdc0
Merge branch 'master' into wenxuan/test_merge_blocks
ti-chi-bot Nov 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 29 additions & 158 deletions dbms/src/Storages/DeltaMerge/tests/gtest_segment.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,119 +54,6 @@ bool shouldCompactStableWithTooMuchDataOutOfSegmentRange(const DMContext & conte
}
namespace tests
{
class SegmentFrameworkTest : public SegmentTestBasic
{
};

TEST_F(SegmentFrameworkTest, PrepareWriteBlock)
try
{
reloadWithOptions(SegmentTestOptions{.is_common_handle = false});

auto s1_id = splitSegmentAt(DELTA_MERGE_FIRST_SEGMENT_ID, 10);
ASSERT_TRUE(s1_id.has_value());
auto s2_id = splitSegmentAt(*s1_id, 20);
ASSERT_TRUE(s2_id.has_value());

// s1 has range [10, 20)
{
auto [begin, end] = getSegmentKeyRange(*s1_id);
ASSERT_EQ(10, begin);
ASSERT_EQ(20, end);
}

{
// write_rows == segment_rows, start_key not specified
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 10);
ASSERT_EQ(1, blocks.size());
auto handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({10, 11, 12, 13, 14, 15, 16, 17, 18, 19}), handle_data);
}
{
// write_rows > segment_rows, start_key not specified
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 13);
ASSERT_EQ(2, blocks.size());
{
auto handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({10, 11, 12, 13, 14, 15, 16, 17, 18, 19}), handle_data);
}
{
auto handle_column = blocks[1].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({10, 11, 12}), handle_data);
}
}
{
// start_key specified, end_key - start_key < write_rows
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 2, /* at */ 16);
ASSERT_EQ(1, blocks.size());
const auto & handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17}), handle_data);
}
{
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 4, /* at */ 16);
ASSERT_EQ(1, blocks.size());
const auto & handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17, 18, 19}), handle_data);
}
{
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 5, /* at */ 16);
ASSERT_EQ(2, blocks.size());
{
const auto & handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17, 18, 19}), handle_data);
}
{
const auto & handle_column = blocks[1].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16}), handle_data);
}
}
{
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 10, /* at */ 16);
ASSERT_EQ(3, blocks.size());
{
const auto & handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17, 18, 19}), handle_data);
}
{
const auto & handle_column = blocks[1].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17, 18, 19}), handle_data);
}
{
const auto & handle_column = blocks[2].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17}), handle_data);
}
}
{
// write rows < segment rows, start key not specified, should choose a random start.
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 3);
ASSERT_EQ(1, blocks.size());
ASSERT_EQ(3, blocks[0].rows());
}
{
// Let's check whether the generated handles will be starting from 12, for at least once.
auto start_from_12 = 0;
for (size_t i = 0; i < 100; i++)
{
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 3);
if (blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column->getInt(0) == 12)
start_from_12++;
}
ASSERT_TRUE(start_from_12 > 0); // We should hit at least 1 times in 100 iters.
ASSERT_TRUE(start_from_12 < 50); // We should not hit 50 times in 100 iters :)
}
}
CATCH


class SegmentOperationTest : public SegmentTestBasic
{
Expand Down Expand Up @@ -525,12 +412,11 @@ CATCH
TEST_F(SegmentOperationTest, SegmentLogicalSplit)
try
{
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
}
reloadWithOptions(
{.db_settings = {
.dt_segment_stable_pack_rows = 100,
.dt_enable_logical_split = true,
}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 400, /* at */ 0);
flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
Expand Down Expand Up @@ -561,12 +447,8 @@ CATCH
TEST_F(SegmentOperationTest, Issue5570)
try
{
{
SegmentTestOptions options;
// a smaller pack rows for logical split
options.db_settings.dt_segment_stable_pack_rows = 100;
reloadWithOptions(options);
}
// a smaller pack rows for logical split
reloadWithOptions({.db_settings = {.dt_segment_stable_pack_rows = 100}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 200);
flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
Expand Down Expand Up @@ -623,13 +505,12 @@ CATCH
TEST_F(SegmentOperationTest, DeltaPagesAfterDeltaMerge)
try
{
{
SegmentTestOptions options;
// a smaller pack rows for logical split
options.db_settings.dt_segment_stable_pack_rows = 100;
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
}
// a smaller pack rows for logical split
reloadWithOptions(
{.db_settings = {
.dt_segment_stable_pack_rows = 100,
.dt_enable_logical_split = true,
}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 100);
writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 100);
Expand Down Expand Up @@ -717,10 +598,11 @@ class SegmentEnableLogicalSplitTest : public SegmentOperationTest
void SetUp() override
{
SegmentOperationTest::SetUp();
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
reloadWithOptions(
{.db_settings = {
.dt_segment_stable_pack_rows = 100,
.dt_enable_logical_split = true,
}});
ASSERT_TRUE(dm_context->enable_logical_split);
}
};
Expand Down Expand Up @@ -793,9 +675,7 @@ class SegmentSplitTest : public SegmentTestBasic
TEST_F(SegmentSplitTest, AutoModePhycialSplitByDefault)
try
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_segment_stable_pack_rows = 100}});
ASSERT_FALSE(dm_context->enable_logical_split);

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 1000);
Expand All @@ -812,11 +692,12 @@ CATCH
TEST_F(SegmentSplitTest, PhysicalSplitMode)
try
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
// Even if we explicitly set enable_logical_split, we will still do physical split in SplitMode::Physical.
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
reloadWithOptions(
{.db_settings = {
.dt_segment_stable_pack_rows = 100,
.dt_enable_logical_split = true,
}});
ASSERT_TRUE(dm_context->enable_logical_split);

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 1000);
Expand Down Expand Up @@ -871,9 +752,7 @@ CATCH
TEST_F(SegmentSplitTest, LogicalSplitModeDoesLogicalSplit)
try
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_segment_stable_pack_rows = 100}});
// Logical split will be performed if we use logical split mode, even when enable_logical_split is false.
ASSERT_FALSE(dm_context->enable_logical_split);

Expand Down Expand Up @@ -913,9 +792,7 @@ CATCH
TEST_F(SegmentSplitTest, LogicalSplitModeOnePackInStable)
try
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_segment_stable_pack_rows = 100}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 50);
flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
Expand All @@ -933,9 +810,7 @@ CATCH
TEST_F(SegmentSplitTest, LogicalSplitModeOnePackWithHoleInStable)
try
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_segment_stable_pack_rows = 100}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 10, /* at */ 0);
writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 10, /* at */ 90);
Expand Down Expand Up @@ -1009,9 +884,7 @@ CATCH
TEST_F(SegmentSplitAtTest, AutoModeEnableLogicalSplit)
try
{
SegmentTestOptions options;
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_enable_logical_split = true}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 100, /* at */ 0);
flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
Expand Down Expand Up @@ -1048,9 +921,7 @@ CATCH
TEST_F(SegmentSplitAtTest, PhysicalSplitMode)
try
{
SegmentTestOptions options;
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_enable_logical_split = true}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 100, /* at */ 0);
flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
Expand Down
Loading