Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support segment storage format without forward index #9333

Merged
merged 27 commits into from
Oct 12, 2022
Merged

Support segment storage format without forward index #9333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
4d312be
Add the option to create columns without a forward index
somandal Aug 23, 2022
2d4e051
Fix unit test
somandal Sep 3, 2022
3476338
Set and unset flag in test
somandal Sep 3, 2022
425d51d
Empty commit to trigger tests
somandal Sep 3, 2022
4468399
Empty commit to trigger tests
somandal Sep 3, 2022
4aae649
Trying test changes to see if unit tests pass
somandal Sep 3, 2022
66910ab
Add order by to tests for better determinism
somandal Sep 3, 2022
1c4386e
Change disk size used in OfflineClusterIntegrationTest:testAggregateM…
somandal Sep 4, 2022
3da307d
Empty commit to trigger tests
somandal Sep 4, 2022
6714228
Address review comments
somandal Sep 8, 2022
d5fc71c
Empty commit to trigger tests
somandal Sep 8, 2022
31c6f8b
Address review comments, add file for MV forward index disabled tests
somandal Sep 9, 2022
d7a6425
Enhance tests and add fixes and address some comments
somandal Sep 10, 2022
43ae966
Address review comments, remove no-op reader, allow sorted column, re…
somandal Sep 15, 2022
338e45d
Address review comments
somandal Sep 15, 2022
3a00a91
Address review comments
somandal Sep 19, 2022
6af27d3
Empty-Commit
somandal Sep 19, 2022
530af2b
Address review comments, remove default column handling for forwardIn…
somandal Oct 5, 2022
0f80f7f
Fix formatting
somandal Oct 5, 2022
6403869
Fix formatting
somandal Oct 5, 2022
1fc8b8d
Update tests to check error message
somandal Oct 5, 2022
9aed5a8
Empty commit to trigger tests
somandal Oct 5, 2022
51df55a
Empty commit to trigger tests
somandal Oct 5, 2022
8ddcdfa
Add checks for forwardIndexDisabled for derived column handling
somandal Oct 5, 2022
eb68745
Empty commit to trigger tests
somandal Oct 5, 2022
8e8c910
Address review comments
somandal Oct 6, 2022
df04878
Fix tests
somandal Oct 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.common;

import com.google.common.base.Preconditions;
import java.io.Closeable;
import java.io.IOException;
import java.math.BigDecimal;
Expand Down Expand Up @@ -66,10 +67,13 @@ public DataFetcher(Map<String, DataSource> dataSourceMap) {
for (Map.Entry<String, DataSource> entry : dataSourceMap.entrySet()) {
String column = entry.getKey();
DataSource dataSource = entry.getValue();
DataSourceMetadata dataSourceMetadata = dataSource.getDataSourceMetadata();
ForwardIndexReader<?> forwardIndexReader = dataSource.getForwardIndex();
Preconditions.checkState(forwardIndexReader != null,
"Forward index disabled for column: %s, cannot create DataFetcher!", column);
ColumnValueReader columnValueReader =
new ColumnValueReader(dataSource.getForwardIndex(), dataSource.getDictionary());
new ColumnValueReader(forwardIndexReader, dataSource.getDictionary());
_columnValueReaderMap.put(column, columnValueReader);
DataSourceMetadata dataSourceMetadata = dataSource.getDataSourceMetadata();
if (!dataSourceMetadata.isSingleValue()) {
maxNumValuesPerMVEntry = Math.max(maxNumValuesPerMVEntry, dataSourceMetadata.getMaxNumValuesPerMVEntry());
}
Expand Down
19 changes: 11 additions & 8 deletions pinot-core/src/main/java/org/apache/pinot/core/minion/RawIndexConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.minion;

import com.google.common.base.Preconditions;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -203,7 +204,9 @@ private void convertColumn(FieldSpec fieldSpec)

// Create the raw index
DataSource dataSource = _originalImmutableSegment.getDataSource(columnName);
ForwardIndexReader reader = dataSource.getForwardIndex();
ForwardIndexReader forwardIndexReader = dataSource.getForwardIndex();
Preconditions.checkState(forwardIndexReader != null,
"Forward index disabled for column: %s, raw index conversion operation unsupported!", columnName);
Dictionary dictionary = dataSource.getDictionary();
assert dictionary != null;
DataType storedType = dictionary.getValueType();
Expand All @@ -213,36 +216,36 @@ private void convertColumn(FieldSpec fieldSpec)
IndexCreationContext.builder().withIndexDir(_convertedIndexDir).withColumnMetadata(columnMetadata)
.withFieldSpec(new DimensionFieldSpec(columnName, storedType, columnMetadata.isSingleValue()))
.withDictionary(false).build().forForwardIndex(ChunkCompressionType.LZ4, null));
ForwardIndexReaderContext readerContext = reader.createContext()) {
ForwardIndexReaderContext readerContext = forwardIndexReader.createContext()) {
switch (storedType) {
case INT:
for (int docId = 0; docId < numDocs; docId++) {
rawIndexCreator.putInt(dictionary.getIntValue(reader.getDictId(docId, readerContext)));
rawIndexCreator.putInt(dictionary.getIntValue(forwardIndexReader.getDictId(docId, readerContext)));
}
break;
case LONG:
for (int docId = 0; docId < numDocs; docId++) {
rawIndexCreator.putLong(dictionary.getLongValue(reader.getDictId(docId, readerContext)));
rawIndexCreator.putLong(dictionary.getLongValue(forwardIndexReader.getDictId(docId, readerContext)));
}
break;
case FLOAT:
for (int docId = 0; docId < numDocs; docId++) {
rawIndexCreator.putFloat(dictionary.getFloatValue(reader.getDictId(docId, readerContext)));
rawIndexCreator.putFloat(dictionary.getFloatValue(forwardIndexReader.getDictId(docId, readerContext)));
}
break;
case DOUBLE:
for (int docId = 0; docId < numDocs; docId++) {
rawIndexCreator.putDouble(dictionary.getDoubleValue(reader.getDictId(docId, readerContext)));
rawIndexCreator.putDouble(dictionary.getDoubleValue(forwardIndexReader.getDictId(docId, readerContext)));
}
break;
case STRING:
for (int docId = 0; docId < numDocs; docId++) {
rawIndexCreator.putString(dictionary.getStringValue(reader.getDictId(docId, readerContext)));
rawIndexCreator.putString(dictionary.getStringValue(forwardIndexReader.getDictId(docId, readerContext)));
}
break;
case BYTES:
for (int docId = 0; docId < numDocs; docId++) {
rawIndexCreator.putBytes(dictionary.getBytesValue(reader.getDictId(docId, readerContext)));
rawIndexCreator.putBytes(dictionary.getBytesValue(forwardIndexReader.getDictId(docId, readerContext)));
}
break;
default:
Expand Down
4 changes: 4 additions & 0 deletions pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ScanBasedFilterOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.operator.filter;

import com.google.common.base.Preconditions;
import java.util.Collections;
import java.util.List;
import org.apache.pinot.core.common.Operator;
Expand All @@ -43,6 +44,9 @@ public class ScanBasedFilterOperator extends BaseFilterOperator {
_dataSource = dataSource;
_numDocs = numDocs;
_nullHandlingEnabled = nullHandlingEnabled;
Preconditions.checkState(_dataSource.getForwardIndex() != null,
"Forward index disabled for column: %s, scan based filtering not supported!",
_dataSource.getDataSourceMetadata().getFieldSpec().getName());
}

@Override
Expand Down
Loading