From ea2ad20e1fc2d846a0738b558ad8c36085452e36 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 24 Feb 2025 17:02:28 +0800 Subject: [PATCH 1/3] update --- be/src/common/config.cpp | 4 ++++ be/src/common/config.h | 4 ++++ be/src/olap/tablet_reader.cpp | 3 ++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 1bef84520eb86d..08bf67595fcacd 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1426,6 +1426,10 @@ DEFINE_mInt32(check_tablet_delete_bitmap_interval_seconds, "300"); DEFINE_mInt32(check_tablet_delete_bitmap_score_top_n, "10"); DEFINE_mBool(enable_check_tablet_delete_bitmap_score, "true"); +// whether to prune rows with delete sign = 1 in base compaction +// ATTN: this config is only for test +DEFINE_mBool(prune_delete_sign_when_base_compaction, "false"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 9a7da04e8afb01..19b73d3db51984 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1512,6 +1512,10 @@ DECLARE_mInt32(check_tablet_delete_bitmap_interval_seconds); DECLARE_mInt32(check_tablet_delete_bitmap_score_top_n); DECLARE_mBool(enable_check_tablet_delete_bitmap_score); +// whether to prune rows with delete sign = 1 in base compaction +// ATTN: this config is only for test +DECLARE_mBool(prune_delete_sign_when_base_compaction); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp index 416d0fea476b32..3abfb66147c050 100644 --- a/be/src/olap/tablet_reader.cpp +++ b/be/src/olap/tablet_reader.cpp @@ -657,7 +657,8 @@ Status TabletReader::_init_delete_condition(const ReaderParams& read_params) { // Delete sign could not be applied when delete on cumu compaction is enabled, bucause it is meant for delete with predicates. // If delete design is applied on cumu compaction, it will lose effect when doing base compaction. // `_delete_sign_available` indicates the condition where we could apply delete signs to data. - _delete_sign_available = (read_params.reader_type == ReaderType::READER_BASE_COMPACTION || + _delete_sign_available = ((read_params.reader_type == ReaderType::READER_BASE_COMPACTION && + config::prune_delete_sign_when_base_compaction) || read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION || read_params.reader_type == ReaderType::READER_CHECKSUM); From 4113ad7e3d8603834021a46ff9f170cbfc8e86a4 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 24 Feb 2025 17:30:47 +0800 Subject: [PATCH 2/3] update --- be/src/common/config.cpp | 2 +- be/src/common/config.h | 2 +- be/src/olap/tablet_reader.cpp | 2 +- .../test_config_prune_delete_sign.groovy | 83 +++++++++++++++++++ 4 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 regression-test/suites/compaction/test_config_prune_delete_sign.groovy diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 08bf67595fcacd..59580e5b5e1c6c 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1428,7 +1428,7 @@ DEFINE_mBool(enable_check_tablet_delete_bitmap_score, "true"); // whether to prune rows with delete sign = 1 in base compaction // ATTN: this config is only for test -DEFINE_mBool(prune_delete_sign_when_base_compaction, "false"); +DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true"); // clang-format off #ifdef BE_TEST diff --git a/be/src/common/config.h b/be/src/common/config.h index 19b73d3db51984..cb50f7a7e36c02 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1514,7 +1514,7 @@ DECLARE_mBool(enable_check_tablet_delete_bitmap_score); // whether to prune rows with delete sign = 1 in base compaction // ATTN: this config is only for test -DECLARE_mBool(prune_delete_sign_when_base_compaction); +DECLARE_mBool(enable_prune_delete_sign_when_base_compaction); #ifdef BE_TEST // test s3 diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp index 3abfb66147c050..585a787dc943e6 100644 --- a/be/src/olap/tablet_reader.cpp +++ b/be/src/olap/tablet_reader.cpp @@ -658,7 +658,7 @@ Status TabletReader::_init_delete_condition(const ReaderParams& read_params) { // If delete design is applied on cumu compaction, it will lose effect when doing base compaction. // `_delete_sign_available` indicates the condition where we could apply delete signs to data. _delete_sign_available = ((read_params.reader_type == ReaderType::READER_BASE_COMPACTION && - config::prune_delete_sign_when_base_compaction) || + config::enable_prune_delete_sign_when_base_compaction) || read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION || read_params.reader_type == ReaderType::READER_CHECKSUM); diff --git a/regression-test/suites/compaction/test_config_prune_delete_sign.groovy b/regression-test/suites/compaction/test_config_prune_delete_sign.groovy new file mode 100644 index 00000000000000..6fadeac95b84d1 --- /dev/null +++ b/regression-test/suites/compaction/test_config_prune_delete_sign.groovy @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert +import java.util.concurrent.TimeUnit +import org.awaitility.Awaitility + +// test cases to ensure that inject points for mow correctness work as expected +suite("test_config_prune_delete_sign", "nonConcurrent") { + + def inspectRows = { sqlStr -> + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_inspect sqlStr + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + } + + def custoBeConfig = [ + enable_prune_delete_sign_when_base_compaction : true + ] + + setBeConfigTemporary(custoBeConfig) { + def table1 = "test_config_prune_delete_sign" + sql "DROP TABLE IF EXISTS ${table1} FORCE;" + sql """ CREATE TABLE IF NOT EXISTS ${table1} ( + `k1` int NOT NULL, + `c1` int, + `c2` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_mow_light_delete" = "false", + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1"); """ + + (1..20).each { + sql "insert into ${table1} values($it,$it,$it);" + } + sql "sync;" + qt_sql "select count() from ${table1};" + + sql "delete from ${table1} where k<=10;" + + qt_sql "select count() from ${table1};" + + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_sql "select count() from ${table1} where __DORIS_DELETE_SIGN__=1;" + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + + trigger_and_wait_compaction(table1, "base") + qt_sql "select count() from ${table1};" + + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_sql "select count() from ${table1} where __DORIS_DELETE_SIGN__=1;" + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + } +} From 24376acd8325eb8087eb5131569ac9513a604ccc Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 24 Feb 2025 19:05:46 +0800 Subject: [PATCH 3/3] add case --- .../test_config_prune_delete_sign.out | 13 +++++ .../test_config_prune_delete_sign.groovy | 56 ++++++++++--------- 2 files changed, 44 insertions(+), 25 deletions(-) create mode 100644 regression-test/data/compaction/test_config_prune_delete_sign.out diff --git a/regression-test/data/compaction/test_config_prune_delete_sign.out b/regression-test/data/compaction/test_config_prune_delete_sign.out new file mode 100644 index 00000000000000..9bbb4251318f7d --- /dev/null +++ b/regression-test/data/compaction/test_config_prune_delete_sign.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +10 + +-- !del_cnt -- +20 + +-- !sql -- +40 + +-- !del_cnt -- +20 + diff --git a/regression-test/suites/compaction/test_config_prune_delete_sign.groovy b/regression-test/suites/compaction/test_config_prune_delete_sign.groovy index 6fadeac95b84d1..06c56ce6405d04 100644 --- a/regression-test/suites/compaction/test_config_prune_delete_sign.groovy +++ b/regression-test/suites/compaction/test_config_prune_delete_sign.groovy @@ -15,11 +15,6 @@ // specific language governing permissions and limitations // under the License. -import org.junit.Assert -import java.util.concurrent.TimeUnit -import org.awaitility.Awaitility - -// test cases to ensure that inject points for mow correctness work as expected suite("test_config_prune_delete_sign", "nonConcurrent") { def inspectRows = { sqlStr -> @@ -33,7 +28,7 @@ suite("test_config_prune_delete_sign", "nonConcurrent") { } def custoBeConfig = [ - enable_prune_delete_sign_when_base_compaction : true + enable_prune_delete_sign_when_base_compaction : false ] setBeConfigTemporary(custoBeConfig) { @@ -51,33 +46,44 @@ suite("test_config_prune_delete_sign", "nonConcurrent") { "disable_auto_compaction" = "true", "replication_num" = "1"); """ - (1..20).each { - sql "insert into ${table1} values($it,$it,$it);" + def getDeleteSignCnt = { + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_del_cnt "select count() from ${table1} where __DORIS_DELETE_SIGN__=1;" + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" } - sql "sync;" - qt_sql "select count() from ${table1};" - sql "delete from ${table1} where k<=10;" + (1..30).each { + sql "insert into ${table1} values($it,$it,$it);" + } + trigger_and_wait_compaction(table1, "cumulative") + sql "delete from ${table1} where k1<=20;" + sql "sync;" qt_sql "select count() from ${table1};" + getDeleteSignCnt() - sql "set skip_delete_sign=true;" - sql "set skip_delete_bitmap=true;" - sql "sync" - qt_sql "select count() from ${table1} where __DORIS_DELETE_SIGN__=1;" - sql "set skip_delete_sign=false;" - sql "set skip_delete_bitmap=false;" - sql "sync" + (31..60).each { + sql "insert into ${table1} values($it,$it,$it);" + } + trigger_and_wait_compaction(table1, "cumulative") trigger_and_wait_compaction(table1, "base") qt_sql "select count() from ${table1};" + getDeleteSignCnt() - sql "set skip_delete_sign=true;" - sql "set skip_delete_bitmap=true;" - sql "sync" - qt_sql "select count() from ${table1} where __DORIS_DELETE_SIGN__=1;" - sql "set skip_delete_sign=false;" - sql "set skip_delete_bitmap=false;" - sql "sync" + def tablets = sql_return_maparray """ show tablets from ${table1}; """ + logger.info("tablets: ${tablets}") + assert 1 == tablets.size() + String compactionUrl = tablets[0]["CompactionStatus"] + def (code, out, err) = curl("GET", compactionUrl) + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assert code == 0 + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets.size() == 1 + assert tabletJson.rowsets[0].contains("[0-62]") } }