Adding a new config skipSegmentPreprocess in table IndexingConfig (ap…

…ache#14982)
xiangfu0 · Feb 4, 2025 · 3c02ec4 · 3c02ec4
1 parent c7fcda9
commit 3c02ec4
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 0 deletions.
diff --git a/...in/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java b/...in/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
@@ -282,6 +282,9 @@ public static ImmutableSegment load(SegmentDirectory segmentDirectory, IndexLoad
   public static boolean needPreprocess(SegmentDirectory segmentDirectory, IndexLoadingConfig indexLoadingConfig,
       @Nullable Schema schema)
       throws Exception {
+    if (indexLoadingConfig.isSkipSegmentPreprocess()) {
+      return false;
+    }
     if (needConvertSegmentFormat(indexLoadingConfig, segmentDirectory.getSegmentMetadata())) {
       return true;
     }

diff --git a/...src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java b/...src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java
@@ -337,6 +337,10 @@ public void setErrorOnColumnBuildFailure(boolean errorOnColumnBuildFailure) {
     _errorOnColumnBuildFailure = errorOnColumnBuildFailure;
   }
 
+  public boolean isSkipSegmentPreprocess() {
+    return _tableConfig != null && _tableConfig.getIndexingConfig().isSkipSegmentPreprocess();
+  }
+
   @Nullable
   public FieldIndexConfigs getFieldIndexConfig(String columnName) {
     if (_indexConfigsByColName == null || _dirty) {

diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexingConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/IndexingConfig.java
@@ -66,6 +66,7 @@ public class IndexingConfig extends BaseJsonConfig {
   private boolean _aggregateMetrics;
   private boolean _nullHandlingEnabled;
   private boolean _columnMajorSegmentBuilderEnabled = true;
+  private boolean _skipSegmentPreprocess;
 
   /**
    * If `optimizeDictionary` enabled, dictionary is not created for the high-cardinality
@@ -361,6 +362,14 @@ public void setColumnMajorSegmentBuilderEnabled(boolean columnMajorSegmentBuilde
     _columnMajorSegmentBuilderEnabled = columnMajorSegmentBuilderEnabled;
   }
 
+  public boolean isSkipSegmentPreprocess() {
+    return _skipSegmentPreprocess;
+  }
+
+  public void setSkipSegmentPreprocess(boolean skipSegmentPreprocess) {
+    _skipSegmentPreprocess = skipSegmentPreprocess;
+  }
+
   public boolean isOptimizeDictionary() {
     return _optimizeDictionary;
   }