Skip to content

Commit

Permalink
test: merge blocks when prepare (#6369)
Browse files Browse the repository at this point in the history 
ref #5237
  • Loading branch information
@breezewish
breezewish authored Nov 28, 2022
1 parent cfbe7f3 commit a0972b2
Show file tree
Hide file tree
Showing 5 changed files with 199 additions and 226 deletions.
187 changes: 29 additions & 158 deletions dbms/src/Storages/DeltaMerge/tests/gtest_segment.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,119 +54,6 @@ bool shouldCompactStableWithTooMuchDataOutOfSegmentRange(const DMContext & conte
}
namespace tests
{
class SegmentFrameworkTest : public SegmentTestBasic
{
};

TEST_F(SegmentFrameworkTest, PrepareWriteBlock)
try
{
reloadWithOptions(SegmentTestOptions{.is_common_handle = false});

auto s1_id = splitSegmentAt(DELTA_MERGE_FIRST_SEGMENT_ID, 10);
ASSERT_TRUE(s1_id.has_value());
auto s2_id = splitSegmentAt(*s1_id, 20);
ASSERT_TRUE(s2_id.has_value());

// s1 has range [10, 20)
{
auto [begin, end] = getSegmentKeyRange(*s1_id);
ASSERT_EQ(10, begin);
ASSERT_EQ(20, end);
}

{
// write_rows == segment_rows, start_key not specified
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 10);
ASSERT_EQ(1, blocks.size());
auto handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({10, 11, 12, 13, 14, 15, 16, 17, 18, 19}), handle_data);
}
{
// write_rows > segment_rows, start_key not specified
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 13);
ASSERT_EQ(2, blocks.size());
{
auto handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({10, 11, 12, 13, 14, 15, 16, 17, 18, 19}), handle_data);
}
{
auto handle_column = blocks[1].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({10, 11, 12}), handle_data);
}
}
{
// start_key specified, end_key - start_key < write_rows
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 2, /* at */ 16);
ASSERT_EQ(1, blocks.size());
const auto & handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17}), handle_data);
}
{
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 4, /* at */ 16);
ASSERT_EQ(1, blocks.size());
const auto & handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17, 18, 19}), handle_data);
}
{
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 5, /* at */ 16);
ASSERT_EQ(2, blocks.size());
{
const auto & handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17, 18, 19}), handle_data);
}
{
const auto & handle_column = blocks[1].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16}), handle_data);
}
}
{
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 10, /* at */ 16);
ASSERT_EQ(3, blocks.size());
{
const auto & handle_column = blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17, 18, 19}), handle_data);
}
{
const auto & handle_column = blocks[1].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17, 18, 19}), handle_data);
}
{
const auto & handle_column = blocks[2].getByName(EXTRA_HANDLE_COLUMN_NAME).column;
const auto & handle_data = typeid_cast<const ColumnVector<Handle> &>(*handle_column).getData();
ASSERT_EQ(PaddedPODArray<Handle>({16, 17}), handle_data);
}
}
{
// write rows < segment rows, start key not specified, should choose a random start.
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 3);
ASSERT_EQ(1, blocks.size());
ASSERT_EQ(3, blocks[0].rows());
}
{
// Let's check whether the generated handles will be starting from 12, for at least once.
auto start_from_12 = 0;
for (size_t i = 0; i < 100; i++)
{
auto blocks = prepareWriteBlocksInSegmentRange(*s1_id, 3);
if (blocks[0].getByName(EXTRA_HANDLE_COLUMN_NAME).column->getInt(0) == 12)
start_from_12++;
}
ASSERT_TRUE(start_from_12 > 0); // We should hit at least 1 times in 100 iters.
ASSERT_TRUE(start_from_12 < 50); // We should not hit 50 times in 100 iters :)
}
}
CATCH


class SegmentOperationTest : public SegmentTestBasic
{
Expand Down Expand Up @@ -525,12 +412,11 @@ CATCH
TEST_F(SegmentOperationTest, SegmentLogicalSplit)
try
{
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
}
reloadWithOptions(
{.db_settings = {
.dt_segment_stable_pack_rows = 100,
.dt_enable_logical_split = true,
}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 400, /* at */ 0);
flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
Expand Down Expand Up @@ -561,12 +447,8 @@ CATCH
TEST_F(SegmentOperationTest, Issue5570)
try
{
{
SegmentTestOptions options;
// a smaller pack rows for logical split
options.db_settings.dt_segment_stable_pack_rows = 100;
reloadWithOptions(options);
}
// a smaller pack rows for logical split
reloadWithOptions({.db_settings = {.dt_segment_stable_pack_rows = 100}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 200);
flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
Expand Down Expand Up @@ -623,13 +505,12 @@ CATCH
TEST_F(SegmentOperationTest, DeltaPagesAfterDeltaMerge)
try
{
{
SegmentTestOptions options;
// a smaller pack rows for logical split
options.db_settings.dt_segment_stable_pack_rows = 100;
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
}
// a smaller pack rows for logical split
reloadWithOptions(
{.db_settings = {
.dt_segment_stable_pack_rows = 100,
.dt_enable_logical_split = true,
}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 100);
writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 100);
Expand Down Expand Up @@ -717,10 +598,11 @@ class SegmentEnableLogicalSplitTest : public SegmentOperationTest
void SetUp() override
{
SegmentOperationTest::SetUp();
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
reloadWithOptions(
{.db_settings = {
.dt_segment_stable_pack_rows = 100,
.dt_enable_logical_split = true,
}});
ASSERT_TRUE(dm_context->enable_logical_split);
}
};
Expand Down Expand Up @@ -793,9 +675,7 @@ class SegmentSplitTest : public SegmentTestBasic
TEST_F(SegmentSplitTest, AutoModePhycialSplitByDefault)
try
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_segment_stable_pack_rows = 100}});
ASSERT_FALSE(dm_context->enable_logical_split);

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 1000);
Expand All @@ -812,11 +692,12 @@ CATCH
TEST_F(SegmentSplitTest, PhysicalSplitMode)
try
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
// Even if we explicitly set enable_logical_split, we will still do physical split in SplitMode::Physical.
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
reloadWithOptions(
{.db_settings = {
.dt_segment_stable_pack_rows = 100,
.dt_enable_logical_split = true,
}});
ASSERT_TRUE(dm_context->enable_logical_split);

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 1000);
Expand Down Expand Up @@ -871,9 +752,7 @@ CATCH
TEST_F(SegmentSplitTest, LogicalSplitModeDoesLogicalSplit)
try
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_segment_stable_pack_rows = 100}});
// Logical split will be performed if we use logical split mode, even when enable_logical_split is false.
ASSERT_FALSE(dm_context->enable_logical_split);

Expand Down Expand Up @@ -913,9 +792,7 @@ CATCH
TEST_F(SegmentSplitTest, LogicalSplitModeOnePackInStable)
try
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_segment_stable_pack_rows = 100}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 50);
flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
Expand All @@ -933,9 +810,7 @@ CATCH
TEST_F(SegmentSplitTest, LogicalSplitModeOnePackWithHoleInStable)
try
{
SegmentTestOptions options;
options.db_settings.dt_segment_stable_pack_rows = 100;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_segment_stable_pack_rows = 100}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 10, /* at */ 0);
writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 10, /* at */ 90);
Expand Down Expand Up @@ -1009,9 +884,7 @@ CATCH
TEST_F(SegmentSplitAtTest, AutoModeEnableLogicalSplit)
try
{
SegmentTestOptions options;
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_enable_logical_split = true}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 100, /* at */ 0);
flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
Expand Down Expand Up @@ -1048,9 +921,7 @@ CATCH
TEST_F(SegmentSplitAtTest, PhysicalSplitMode)
try
{
SegmentTestOptions options;
options.db_settings.dt_enable_logical_split = true;
reloadWithOptions(options);
reloadWithOptions({.db_settings = {.dt_enable_logical_split = true}});

writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 100, /* at */ 0);
flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
Expand Down
Loading

0 comments on commit a0972b2

Please sign in to comment.