[compaction](config) Add a config to control whether to prune rows with delete sign=1 in base compaction

be/src/common/config.cpp

@@ -1426,6 +1426,10 @@ DEFINE_mInt32(check_tablet_delete_bitmap_interval_seconds, "300");
 DEFINE_mInt32(check_tablet_delete_bitmap_score_top_n, "10");
 DEFINE_mBool(enable_check_tablet_delete_bitmap_score, "true");
 
+// whether to prune rows with delete sign = 1 in base compaction
+// ATTN: this config is only for test
+DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true");
+
 // clang-format off
 #ifdef BE_TEST
 // test s3

be/src/common/config.h

@@ -1512,6 +1512,10 @@ DECLARE_mInt32(check_tablet_delete_bitmap_interval_seconds);
 DECLARE_mInt32(check_tablet_delete_bitmap_score_top_n);
 DECLARE_mBool(enable_check_tablet_delete_bitmap_score);
 
+// whether to prune rows with delete sign = 1 in base compaction
+// ATTN: this config is only for test
+DECLARE_mBool(enable_prune_delete_sign_when_base_compaction);
+
 #ifdef BE_TEST
 // test s3
 DECLARE_String(test_s3_resource);

be/src/olap/tablet_reader.cpp

@@ -657,7 +657,8 @@ Status TabletReader::_init_delete_condition(const ReaderParams& read_params) {
     // Delete sign could not be applied when delete on cumu compaction is enabled, bucause it is meant for delete with predicates.
     // If delete design is applied on cumu compaction, it will lose effect when doing base compaction.
     // `_delete_sign_available` indicates the condition where we could apply delete signs to data.
-    _delete_sign_available = (read_params.reader_type == ReaderType::READER_BASE_COMPACTION ||
+    _delete_sign_available = ((read_params.reader_type == ReaderType::READER_BASE_COMPACTION &&
+                               config::enable_prune_delete_sign_when_base_compaction) ||
                               read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION ||
                               read_params.reader_type == ReaderType::READER_CHECKSUM);
 

regression-test/data/compaction/test_config_prune_delete_sign.out

@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+10
+
+-- !del_cnt --
+20
+
+-- !sql --
+40
+
+-- !del_cnt --
+20
+

regression-test/suites/compaction/test_config_prune_delete_sign.groovy

@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_config_prune_delete_sign", "nonConcurrent") {
+
+    def inspectRows = { sqlStr ->
+        sql "set skip_delete_sign=true;"
+        sql "set skip_delete_bitmap=true;"
+        sql "sync"
+        qt_inspect sqlStr
+        sql "set skip_delete_sign=false;"
+        sql "set skip_delete_bitmap=false;"
+        sql "sync"
+    }
+
+    def custoBeConfig = [
+        enable_prune_delete_sign_when_base_compaction : false
+    ]
+
+    setBeConfigTemporary(custoBeConfig) {
+        def table1 = "test_config_prune_delete_sign"
+        sql "DROP TABLE IF EXISTS ${table1} FORCE;"
+        sql """ CREATE TABLE IF NOT EXISTS ${table1} (
+                `k1` int NOT NULL,
+                `c1` int,
+                `c2` int
+                )UNIQUE KEY(k1)
+            DISTRIBUTED BY HASH(k1) BUCKETS 1
+            PROPERTIES (
+                "enable_mow_light_delete" = "false",
+                "enable_unique_key_merge_on_write" = "true",
+                "disable_auto_compaction" = "true",
+                "replication_num" = "1"); """
+
+        def getDeleteSignCnt = {
+            sql "set skip_delete_sign=true;"
+            sql "set skip_delete_bitmap=true;"
+            sql "sync"
+            qt_del_cnt "select count() from ${table1} where __DORIS_DELETE_SIGN__=1;"
+            sql "set skip_delete_sign=false;"
+            sql "set skip_delete_bitmap=false;"
+            sql "sync"
+        }
+
+        (1..30).each {
+            sql "insert into ${table1} values($it,$it,$it);"
+        }
+        trigger_and_wait_compaction(table1, "cumulative")
+
+        sql "delete from ${table1} where k1<=20;"
+        sql "sync;"
+        qt_sql "select count() from ${table1};"
+        getDeleteSignCnt()
+
+        (31..60).each {
+            sql "insert into ${table1} values($it,$it,$it);"
+        }
+        trigger_and_wait_compaction(table1, "cumulative")
+
+        trigger_and_wait_compaction(table1, "base")
+        qt_sql "select count() from ${table1};"
+        getDeleteSignCnt()
+
+        def tablets = sql_return_maparray """ show tablets from ${table1}; """
+        logger.info("tablets: ${tablets}")
+        assert 1 == tablets.size()
+        String compactionUrl = tablets[0]["CompactionStatus"]
+        def (code, out, err) = curl("GET", compactionUrl)
+        logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+        assert code == 0
+        def tabletJson = parseJson(out.trim())
+        assert tabletJson.rowsets.size() == 1
+        assert tabletJson.rowsets[0].contains("[0-62]")
+    }
+}