diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java index 789cb35cb7c7..0830f203a593 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctions.java @@ -43,6 +43,7 @@ private DataTypeConversionFunctions() { public static Object cast(Object value, String targetTypeLiteral) { try { Class clazz = value.getClass(); + // TODO: Support cast for MV Preconditions.checkArgument(!clazz.isArray() | clazz == byte[].class, "%s must not be an array type", clazz); PinotDataType sourceType = PinotDataType.getSingleValueType(clazz); String transformed = targetTypeLiteral.toUpperCase(); diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java b/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java index 1fc1fa2c0511..491fced74806 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java @@ -21,7 +21,6 @@ import java.io.Closeable; import java.io.IOException; import java.math.BigDecimal; -import java.sql.Timestamp; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -67,11 +66,10 @@ public DataFetcher(Map dataSourceMap) { for (Map.Entry entry : dataSourceMap.entrySet()) { String column = entry.getKey(); DataSource dataSource = entry.getValue(); - DataSourceMetadata dataSourceMetadata = dataSource.getDataSourceMetadata(); ColumnValueReader columnValueReader = - new ColumnValueReader(dataSource.getForwardIndex(), dataSource.getDictionary(), - dataSourceMetadata.getDataType()); + new ColumnValueReader(dataSource.getForwardIndex(), dataSource.getDictionary()); _columnValueReaderMap.put(column, columnValueReader); + DataSourceMetadata dataSourceMetadata = dataSource.getDataSourceMetadata(); if (!dataSourceMetadata.isSingleValue()) { maxNumValuesPerMVEntry = Math.max(maxNumValuesPerMVEntry, dataSourceMetadata.getMaxNumValuesPerMVEntry()); } @@ -425,20 +423,22 @@ public void fetchNumValues(String column, int[] inDocIds, int length, int[] outN * Helper class to read values for a column from forward index and dictionary. For raw (non-dictionary-encoded) * forward index, similar to Dictionary, type conversion among INT, LONG, FLOAT, DOUBLE, STRING is supported; type * conversion between STRING and BYTES via Hex encoding/decoding is supported. + * + * TODO: Type conversion for BOOLEAN and TIMESTAMP is not handled */ private class ColumnValueReader implements Closeable { final ForwardIndexReader _reader; final Dictionary _dictionary; - final DataType _dataType; + final DataType _storedType; final boolean _singleValue; boolean _readerContextCreated; ForwardIndexReaderContext _readerContext; - ColumnValueReader(ForwardIndexReader reader, @Nullable Dictionary dictionary, DataType dataType) { + ColumnValueReader(ForwardIndexReader reader, @Nullable Dictionary dictionary) { _reader = reader; _dictionary = dictionary; - _dataType = dataType; + _storedType = reader.getStoredType(); _singleValue = reader.isSingleValue(); } @@ -452,12 +452,12 @@ private ForwardIndexReaderContext getReaderContext() { } void readDictIds(int[] docIds, int length, int[] dictIdBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); _reader.readDictIds(docIds, length, dictIdBuffer, getReaderContext()); } void readIntValues(int[] docIds, int length, int[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { int[] dictIdBuffer = THREAD_LOCAL_DICT_IDS.get(); @@ -469,13 +469,13 @@ void readIntValues(int[] docIds, int length, int[] valueBuffer) { } void readIntValues(TransformEvaluator evaluator, int[] docIds, int length, int[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valueBuffer); } void readLongValues(int[] docIds, int length, long[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { int[] dictIdBuffer = THREAD_LOCAL_DICT_IDS.get(); @@ -487,13 +487,13 @@ void readLongValues(int[] docIds, int length, long[] valueBuffer) { } void readLongValues(TransformEvaluator evaluator, int[] docIds, int length, long[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valueBuffer); } void readFloatValues(int[] docIds, int length, float[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { int[] dictIdBuffer = THREAD_LOCAL_DICT_IDS.get(); @@ -505,13 +505,13 @@ void readFloatValues(int[] docIds, int length, float[] valueBuffer) { } void readFloatValues(TransformEvaluator evaluator, int[] docIds, int length, float[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valueBuffer); } void readDoubleValues(int[] docIds, int length, double[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { int[] dictIdBuffer = THREAD_LOCAL_DICT_IDS.get(); @@ -523,13 +523,13 @@ void readDoubleValues(int[] docIds, int length, double[] valueBuffer) { } void readDoubleValues(TransformEvaluator evaluator, int[] docIds, int length, double[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valueBuffer); } void readBigDecimalValues(int[] docIds, int length, BigDecimal[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { int[] dictIdBuffer = THREAD_LOCAL_DICT_IDS.get(); @@ -541,30 +541,20 @@ void readBigDecimalValues(int[] docIds, int length, BigDecimal[] valueBuffer) { } void readBigDecimalValues(TransformEvaluator evaluator, int[] docIds, int length, BigDecimal[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valueBuffer); } void readStringValues(int[] docIds, int length, String[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { int[] dictIdBuffer = THREAD_LOCAL_DICT_IDS.get(); _reader.readDictIds(docIds, length, dictIdBuffer, readerContext); - if (_dataType == DataType.BOOLEAN) { - for (int i = 0; i < length; i++) { - valueBuffer[i] = Boolean.toString(_dictionary.getIntValue(dictIdBuffer[i]) == 1); - } - } else if (_dataType == DataType.TIMESTAMP) { - for (int i = 0; i < length; i++) { - valueBuffer[i] = new Timestamp(_dictionary.getLongValue(dictIdBuffer[i])).toString(); - } - } else { - _dictionary.readStringValues(dictIdBuffer, length, valueBuffer); - } + _dictionary.readStringValues(dictIdBuffer, length, valueBuffer); } else { - switch (_dataType) { + switch (_storedType) { case INT: for (int i = 0; i < length; i++) { valueBuffer[i] = Integer.toString(_reader.getInt(docIds[i], readerContext)); @@ -590,18 +580,7 @@ void readStringValues(int[] docIds, int length, String[] valueBuffer) { valueBuffer[i] = _reader.getBigDecimal(docIds[i], readerContext).toPlainString(); } break; - case BOOLEAN: - for (int i = 0; i < length; i++) { - valueBuffer[i] = Boolean.toString(_reader.getInt(docIds[i], readerContext) == 1); - } - break; - case TIMESTAMP: - for (int i = 0; i < length; i++) { - valueBuffer[i] = new Timestamp(_reader.getLong(docIds[i], readerContext)).toString(); - } - break; case STRING: - case JSON: for (int i = 0; i < length; i++) { valueBuffer[i] = _reader.getString(docIds[i], readerContext); } @@ -618,20 +597,20 @@ void readStringValues(int[] docIds, int length, String[] valueBuffer) { } void readStringValues(TransformEvaluator evaluator, int[] docIds, int length, String[] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valueBuffer); } void readBytesValues(int[] docIds, int length, byte[][] valueBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { int[] dictIdBuffer = THREAD_LOCAL_DICT_IDS.get(); _reader.readDictIds(docIds, length, dictIdBuffer, readerContext); _dictionary.readBytesValues(dictIdBuffer, length, valueBuffer); } else { - switch (_reader.getStoredType()) { + switch (_storedType) { case STRING: for (int i = 0; i < length; i++) { valueBuffer[i] = BytesUtils.toBytes(_reader.getString(docIds[i], readerContext)); @@ -649,7 +628,7 @@ void readBytesValues(int[] docIds, int length, byte[][] valueBuffer) { } void readDictIdsMV(int[] docIds, int length, int[][] dictIdsBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); for (int i = 0; i < length; i++) { int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds, readerContext); @@ -658,7 +637,7 @@ void readDictIdsMV(int[] docIds, int length, int[][] dictIdsBuffer) { } void readIntValuesMV(int[] docIds, int length, int[][] valuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { for (int i = 0; i < length; i++) { @@ -673,13 +652,13 @@ void readIntValuesMV(int[] docIds, int length, int[][] valuesBuffer) { } void readIntValuesMV(TransformEvaluator evaluator, int[] docIds, int length, int[][] valuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valuesBuffer); } void readLongValuesMV(int[] docIds, int length, long[][] valuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { for (int i = 0; i < length; i++) { @@ -694,13 +673,13 @@ void readLongValuesMV(int[] docIds, int length, long[][] valuesBuffer) { } void readLongValuesMV(TransformEvaluator evaluator, int[] docIds, int length, long[][] valuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valuesBuffer); } void readFloatValuesMV(int[] docIds, int length, float[][] valuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { for (int i = 0; i < length; i++) { @@ -715,13 +694,13 @@ void readFloatValuesMV(int[] docIds, int length, float[][] valuesBuffer) { } void readFloatValuesMV(TransformEvaluator evaluator, int[] docIds, int length, float[][] valuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valuesBuffer); } void readDoubleValuesMV(int[] docIds, int length, double[][] valuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { for (int i = 0; i < length; i++) { @@ -736,80 +715,34 @@ void readDoubleValuesMV(int[] docIds, int length, double[][] valuesBuffer) { } void readDoubleValuesMV(TransformEvaluator evaluator, int[] docIds, int length, double[][] valuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valuesBuffer); } void readStringValuesMV(int[] docIds, int length, String[][] valuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); ForwardIndexReaderContext readerContext = getReaderContext(); if (_dictionary != null) { - if (_dataType == DataType.BOOLEAN) { - for (int i = 0; i < length; i++) { - int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds, readerContext); - int[] intValues = new int[numValues]; - _dictionary.readIntValues(_reusableMVDictIds, numValues, intValues); - String[] values = new String[numValues]; - for (int j = 0; j < numValues; j++) { - values[i] = Boolean.toString(intValues[i] == 1); - } - valuesBuffer[i] = values; - } - } else if (_dataType == DataType.TIMESTAMP) { - for (int i = 0; i < length; i++) { - int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds, readerContext); - long[] longValues = new long[numValues]; - _dictionary.readLongValues(_reusableMVDictIds, numValues, longValues); - String[] values = new String[numValues]; - for (int j = 0; j < numValues; j++) { - values[i] = new Timestamp(longValues[i]).toString(); - } - valuesBuffer[i] = values; - } - } else { - for (int i = 0; i < length; i++) { - int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds, readerContext); - String[] values = new String[numValues]; - _dictionary.readStringValues(_reusableMVDictIds, numValues, values); - valuesBuffer[i] = values; - } + for (int i = 0; i < length; i++) { + int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds, readerContext); + String[] values = new String[numValues]; + _dictionary.readStringValues(_reusableMVDictIds, numValues, values); + valuesBuffer[i] = values; } } else { - if (_dataType == DataType.BOOLEAN) { - int[] intValueBuffer = new int[_maxNumValuesPerMVEntry]; - for (int i = 0; i < length; i++) { - int numValues = _reader.getIntMV(docIds[i], intValueBuffer, readerContext); - String[] values = new String[numValues]; - for (int j = 0; j < numValues; j++) { - values[i] = Boolean.toString(intValueBuffer[i] == 1); - } - valuesBuffer[i] = values; - } - } else if (_dataType == DataType.TIMESTAMP) { - long[] longValueBuffer = new long[_maxNumValuesPerMVEntry]; - for (int i = 0; i < length; i++) { - int numValues = _reader.getLongMV(docIds[i], longValueBuffer, readerContext); - String[] values = new String[numValues]; - for (int j = 0; j < numValues; j++) { - values[i] = new Timestamp(longValueBuffer[i]).toString(); - } - valuesBuffer[i] = values; - } - } else { - _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry, valuesBuffer, readerContext); - } + _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry, valuesBuffer, readerContext); } } void readStringValuesMV(TransformEvaluator evaluator, int[] docIds, int length, String[][] valuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); evaluator.evaluateBlock(docIds, length, _reader, getReaderContext(), _dictionary, getSVDictIdsBuffer(), valuesBuffer); } public void readNumValuesMV(int[] docIds, int length, int[] numValuesBuffer) { - Tracing.activeRecording().setInputDataType(_dataType, _singleValue); + Tracing.activeRecording().setInputDataType(_storedType, _singleValue); for (int i = 0; i < length; i++) { numValuesBuffer[i] = _reader.getNumValuesMV(docIds[i], getReaderContext()); } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java index cf54ce7723a7..429d8d62b303 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java @@ -18,9 +18,7 @@ */ package org.apache.pinot.core.operator.transform.function; -import com.google.common.base.Preconditions; import java.math.BigDecimal; -import java.sql.Timestamp; import org.apache.pinot.core.operator.blocks.ProjectionBlock; import org.apache.pinot.core.operator.transform.TransformResultMetadata; import org.apache.pinot.segment.spi.index.reader.Dictionary; @@ -48,20 +46,33 @@ public abstract class BaseTransformFunction implements TransformFunction { new TransformResultMetadata(DataType.TIMESTAMP, true, false); protected static final TransformResultMetadata STRING_SV_NO_DICTIONARY_METADATA = new TransformResultMetadata(DataType.STRING, true, false); - protected static final TransformResultMetadata STRING_MV_NO_DICTIONARY_METADATA = - new TransformResultMetadata(DataType.STRING, false, false); protected static final TransformResultMetadata JSON_SV_NO_DICTIONARY_METADATA = new TransformResultMetadata(DataType.JSON, true, false); protected static final TransformResultMetadata BYTES_SV_NO_DICTIONARY_METADATA = new TransformResultMetadata(DataType.BYTES, true, false); - protected static final TransformResultMetadata LONG_MV_NO_DICTIONARY_METADATA = - new TransformResultMetadata(DataType.LONG, false, false); - protected static final TransformResultMetadata DOUBLE_MV_NO_DICTIONARY_METADATA = - new TransformResultMetadata(DataType.DOUBLE, false, false); + protected static final TransformResultMetadata INT_MV_NO_DICTIONARY_METADATA = new TransformResultMetadata(DataType.INT, false, false); + protected static final TransformResultMetadata LONG_MV_NO_DICTIONARY_METADATA = + new TransformResultMetadata(DataType.LONG, false, false); protected static final TransformResultMetadata FLOAT_MV_NO_DICTIONARY_METADATA = new TransformResultMetadata(DataType.FLOAT, false, false); + protected static final TransformResultMetadata DOUBLE_MV_NO_DICTIONARY_METADATA = + new TransformResultMetadata(DataType.DOUBLE, false, false); + // TODO: Support MV BIG_DECIMAL + protected static final TransformResultMetadata BIG_DECIMAL_MV_NO_DICTIONARY_METADATA = + new TransformResultMetadata(DataType.BIG_DECIMAL, false, false); + protected static final TransformResultMetadata BOOLEAN_MV_NO_DICTIONARY_METADATA = + new TransformResultMetadata(DataType.BOOLEAN, false, false); + protected static final TransformResultMetadata TIMESTAMP_MV_NO_DICTIONARY_METADATA = + new TransformResultMetadata(DataType.TIMESTAMP, false, false); + protected static final TransformResultMetadata STRING_MV_NO_DICTIONARY_METADATA = + new TransformResultMetadata(DataType.STRING, false, false); + protected static final TransformResultMetadata JSON_MV_NO_DICTIONARY_METADATA = + new TransformResultMetadata(DataType.JSON, false, false); + // TODO: Support MV BYTES + protected static final TransformResultMetadata BYTES_MV_NO_DICTIONARY_METADATA = + new TransformResultMetadata(DataType.BYTES, false, false); protected int[] _intValuesSV; protected long[] _longValuesSV; @@ -69,12 +80,13 @@ public abstract class BaseTransformFunction implements TransformFunction { protected double[] _doubleValuesSV; protected BigDecimal[] _bigDecimalValuesSV; protected String[] _stringValuesSV; - protected byte[][] _byteValuesSV; + protected byte[][] _bytesValuesSV; protected int[][] _intValuesMV; protected long[][] _longValuesMV; protected float[][] _floatValuesMV; protected double[][] _doubleValuesMV; protected String[][] _stringValuesMV; + protected byte[][][] _bytesValuesMV; @Override public Dictionary getDictionary() { @@ -102,7 +114,8 @@ public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { int[] dictIds = transformToDictIdsSV(projectionBlock); dictionary.readIntValues(dictIds, length, _intValuesSV); } else { - switch (getResultMetadata().getDataType().getStoredType()) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { case LONG: long[] longValues = transformToLongValuesSV(projectionBlock); ArrayCopyUtils.copy(longValues, _intValuesSV, length); @@ -124,7 +137,7 @@ public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { ArrayCopyUtils.copy(stringValues, _intValuesSV, length); break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read SV %s as INT", resultDataType)); } } return _intValuesSV; @@ -141,7 +154,8 @@ public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) { int[] dictIds = transformToDictIdsSV(projectionBlock); dictionary.readLongValues(dictIds, length, _longValuesSV); } else { - switch (getResultMetadata().getDataType().getStoredType()) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { case INT: int[] intValues = transformToIntValuesSV(projectionBlock); ArrayCopyUtils.copy(intValues, _longValuesSV, length); @@ -163,7 +177,7 @@ public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) { ArrayCopyUtils.copy(stringValues, _longValuesSV, length); break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read SV %s as LONG", resultDataType)); } } return _longValuesSV; @@ -180,7 +194,8 @@ public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) { int[] dictIds = transformToDictIdsSV(projectionBlock); dictionary.readFloatValues(dictIds, length, _floatValuesSV); } else { - switch (getResultMetadata().getDataType().getStoredType()) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { case INT: int[] intValues = transformToIntValuesSV(projectionBlock); ArrayCopyUtils.copy(intValues, _floatValuesSV, length); @@ -202,7 +217,7 @@ public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) { ArrayCopyUtils.copy(stringValues, _floatValuesSV, length); break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read SV %s as FLOAT", resultDataType)); } } return _floatValuesSV; @@ -219,7 +234,8 @@ public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { int[] dictIds = transformToDictIdsSV(projectionBlock); dictionary.readDoubleValues(dictIds, length, _doubleValuesSV); } else { - switch (getResultMetadata().getDataType().getStoredType()) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { case INT: int[] intValues = transformToIntValuesSV(projectionBlock); ArrayCopyUtils.copy(intValues, _doubleValuesSV, length); @@ -241,7 +257,7 @@ public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { ArrayCopyUtils.copy(stringValues, _doubleValuesSV, length); break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read SV %s as DOUBLE", resultDataType)); } } return _doubleValuesSV; @@ -258,7 +274,8 @@ public BigDecimal[] transformToBigDecimalValuesSV(ProjectionBlock projectionBloc int[] dictIds = transformToDictIdsSV(projectionBlock); dictionary.readBigDecimalValues(dictIds, length, _bigDecimalValuesSV); } else { - switch (getResultMetadata().getDataType().getStoredType()) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { case INT: int[] intValues = transformToIntValuesSV(projectionBlock); ArrayCopyUtils.copy(intValues, _bigDecimalValuesSV, length); @@ -284,7 +301,7 @@ public BigDecimal[] transformToBigDecimalValuesSV(ProjectionBlock projectionBloc ArrayCopyUtils.copy(bytesValues, _bigDecimalValuesSV, length); break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read SV %s as BIG_DECIMAL", resultDataType)); } } return _bigDecimalValuesSV; @@ -296,23 +313,13 @@ public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) { if (_stringValuesSV == null) { _stringValuesSV = new String[length]; } - DataType dataType = getResultMetadata().getDataType(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[] dictIds = transformToDictIdsSV(projectionBlock); - if (dataType == DataType.BOOLEAN) { - for (int i = 0; i < length; i++) { - _stringValuesSV[i] = Boolean.toString(dictionary.getIntValue(dictIds[i]) == 1); - } - } else if (dataType == DataType.TIMESTAMP) { - for (int i = 0; i < length; i++) { - _stringValuesSV[i] = new Timestamp(dictionary.getLongValue(dictIds[i])).toString(); - } - } else { - dictionary.readStringValues(dictIds, length, _stringValuesSV); - } + dictionary.readStringValues(dictIds, length, _stringValuesSV); } else { - switch (dataType) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { case INT: int[] intValues = transformToIntValuesSV(projectionBlock); ArrayCopyUtils.copy(intValues, _stringValuesSV, length); @@ -333,20 +340,12 @@ public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) { BigDecimal[] bigDecimalValues = transformToBigDecimalValuesSV(projectionBlock); ArrayCopyUtils.copy(bigDecimalValues, _stringValuesSV, length); break; - case BOOLEAN: - intValues = transformToIntValuesSV(projectionBlock); - ArrayCopyUtils.copyBoolean(intValues, _stringValuesSV, length); - break; - case TIMESTAMP: - longValues = transformToLongValuesSV(projectionBlock); - ArrayCopyUtils.copyTimestamp(longValues, _stringValuesSV, length); - break; case BYTES: byte[][] bytesValues = transformToBytesValuesSV(projectionBlock); ArrayCopyUtils.copy(bytesValues, _stringValuesSV, length); break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read SV %s as STRING", resultDataType)); } } return _stringValuesSV; @@ -355,24 +354,29 @@ public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) { @Override public byte[][] transformToBytesValuesSV(ProjectionBlock projectionBlock) { int length = projectionBlock.getNumDocs(); - if (_byteValuesSV == null) { - _byteValuesSV = new byte[length][]; + if (_bytesValuesSV == null) { + _bytesValuesSV = new byte[length][]; } Dictionary dictionary = getDictionary(); if (dictionary != null) { int[] dictIds = transformToDictIdsSV(projectionBlock); - dictionary.readBytesValues(dictIds, length, _byteValuesSV); + dictionary.readBytesValues(dictIds, length, _bytesValuesSV); } else { - if (getResultMetadata().getDataType().getStoredType() == DataType.BIG_DECIMAL) { - BigDecimal[] bigDecimalValues = transformToBigDecimalValuesSV(projectionBlock); - ArrayCopyUtils.copy(bigDecimalValues, _byteValuesSV, length); - } else { - Preconditions.checkState(getResultMetadata().getDataType().getStoredType() == DataType.STRING); - String[] stringValues = transformToStringValuesSV(projectionBlock); - ArrayCopyUtils.copy(stringValues, _byteValuesSV, length); + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { + case BIG_DECIMAL: + BigDecimal[] bigDecimalValues = transformToBigDecimalValuesSV(projectionBlock); + ArrayCopyUtils.copy(bigDecimalValues, _bytesValuesSV, length); + break; + case STRING: + String[] stringValues = transformToStringValuesSV(projectionBlock); + ArrayCopyUtils.copy(stringValues, _bytesValuesSV, length); + break; + default: + throw new IllegalStateException(String.format("Cannot read SV %s as BYTES", resultDataType)); } } - return _byteValuesSV; + return _bytesValuesSV; } @Override @@ -392,7 +396,8 @@ public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) { _intValuesMV[i] = intValues; } } else { - switch (getResultMetadata().getDataType().getStoredType()) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { case LONG: long[][] longValuesMV = transformToLongValuesMV(projectionBlock); ArrayCopyUtils.copy(longValuesMV, _intValuesMV, length); @@ -410,7 +415,7 @@ public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) { ArrayCopyUtils.copy(stringValuesMV, _intValuesMV, length); break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read MV %s as INT", resultDataType)); } } return _intValuesMV; @@ -433,7 +438,8 @@ public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) { _longValuesMV[i] = longValues; } } else { - switch (getResultMetadata().getDataType().getStoredType()) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { case INT: int[][] intValuesMV = transformToIntValuesMV(projectionBlock); ArrayCopyUtils.copy(intValuesMV, _longValuesMV, length); @@ -451,7 +457,7 @@ public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) { ArrayCopyUtils.copy(stringValuesMV, _longValuesMV, length); break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read MV %s as LONG", resultDataType)); } } return _longValuesMV; @@ -474,7 +480,8 @@ public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) { _floatValuesMV[i] = floatValues; } } else { - switch (getResultMetadata().getDataType().getStoredType()) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { case INT: int[][] intValuesMV = transformToIntValuesMV(projectionBlock); ArrayCopyUtils.copy(intValuesMV, _floatValuesMV, length); @@ -492,7 +499,7 @@ public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) { ArrayCopyUtils.copy(stringValuesMV, _floatValuesMV, length); break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read MV %s as FLOAT", resultDataType)); } } return _floatValuesMV; @@ -515,7 +522,8 @@ public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) { _doubleValuesMV[i] = doubleValues; } } else { - switch (getResultMetadata().getDataType().getStoredType()) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType.getStoredType()) { case INT: int[][] intValuesMV = transformToIntValuesMV(projectionBlock); ArrayCopyUtils.copy(intValuesMV, _doubleValuesMV, length); @@ -533,7 +541,7 @@ public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) { ArrayCopyUtils.copy(stringValuesMV, _doubleValuesMV, length); break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read MV %s as DOUBLE", resultDataType)); } } return _doubleValuesMV; @@ -545,41 +553,19 @@ public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) { if (_stringValuesMV == null) { _stringValuesMV = new String[length][]; } - DataType dataType = getResultMetadata().getDataType(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[][] dictIdsMV = transformToDictIdsMV(projectionBlock); - if (dataType == DataType.BOOLEAN) { - for (int i = 0; i < length; i++) { - int[] dictIds = dictIdsMV[i]; - int numValues = dictIds.length; - String[] stringValues = new String[numValues]; - for (int j = 0; j < numValues; j++) { - stringValues[j] = Boolean.toString(dictionary.getIntValue(dictIds[i]) == 1); - } - _stringValuesMV[i] = stringValues; - } - } else if (dataType == DataType.TIMESTAMP) { - for (int i = 0; i < length; i++) { - int[] dictIds = dictIdsMV[i]; - int numValues = dictIds.length; - String[] stringValues = new String[numValues]; - for (int j = 0; j < numValues; j++) { - stringValues[j] = new Timestamp(dictionary.getLongValue(dictIds[i])).toString(); - } - _stringValuesMV[i] = stringValues; - } - } else { - for (int i = 0; i < length; i++) { - int[] dictIds = dictIdsMV[i]; - int numValues = dictIds.length; - String[] stringValues = new String[numValues]; - dictionary.readStringValues(dictIds, numValues, stringValues); - _stringValuesMV[i] = stringValues; - } + for (int i = 0; i < length; i++) { + int[] dictIds = dictIdsMV[i]; + int numValues = dictIds.length; + String[] stringValues = new String[numValues]; + dictionary.readStringValues(dictIds, numValues, stringValues); + _stringValuesMV[i] = stringValues; } } else { - switch (dataType) { + DataType resultDataType = getResultMetadata().getDataType(); + switch (resultDataType) { case INT: int[][] intValuesMV = transformToIntValuesMV(projectionBlock); ArrayCopyUtils.copy(intValuesMV, _stringValuesMV, length); @@ -596,16 +582,8 @@ public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) { double[][] doubleValuesMV = transformToDoubleValuesMV(projectionBlock); ArrayCopyUtils.copy(doubleValuesMV, _stringValuesMV, length); break; - case BOOLEAN: - intValuesMV = transformToIntValuesMV(projectionBlock); - ArrayCopyUtils.copyBoolean(intValuesMV, _stringValuesMV, length); - break; - case TIMESTAMP: - longValuesMV = transformToLongValuesMV(projectionBlock); - ArrayCopyUtils.copyTimestamp(longValuesMV, _stringValuesMV, length); - break; default: - throw new IllegalStateException(); + throw new IllegalStateException(String.format("Cannot read MV %s as STRING", resultDataType)); } } return _stringValuesMV; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CaseTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CaseTransformFunction.java index 641107bc1b23..ddb951eacf4f 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CaseTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CaseTransformFunction.java @@ -436,7 +436,7 @@ public byte[][] transformToBytesValuesSV(ProjectionBlock projectionBlock) { TransformFunction transformFunction = _elseThenStatements.get(i); byte[][] bytesValues = transformFunction.transformToBytesValuesSV(projectionBlock); if (_numSelections == 1) { - System.arraycopy(bytesValues, 0, _byteValuesSV, 0, numDocs); + System.arraycopy(bytesValues, 0, _bytesValuesSV, 0, numDocs); } else { for (int j = 0; j < numDocs; j++) { if (selected[j] == i) { diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java index 993094d6cf9b..52ab528e8973 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java @@ -18,6 +18,7 @@ */ package org.apache.pinot.core.operator.transform.function; +import com.google.common.base.Preconditions; import java.math.BigDecimal; import java.util.List; import java.util.Map; @@ -25,12 +26,14 @@ import org.apache.pinot.core.operator.transform.TransformResultMetadata; import org.apache.pinot.segment.spi.datasource.DataSource; import org.apache.pinot.spi.data.FieldSpec.DataType; +import org.apache.pinot.spi.utils.ArrayCopyUtils; public class CastTransformFunction extends BaseTransformFunction { public static final String FUNCTION_NAME = "cast"; private TransformFunction _transformFunction; + private DataType _sourceDataType; private TransformResultMetadata _resultMetadata; @Override @@ -46,60 +49,46 @@ public void init(List arguments, Map data } _transformFunction = arguments.get(0); + TransformResultMetadata sourceMetadata = _transformFunction.getResultMetadata(); + _sourceDataType = sourceMetadata.getDataType(); + boolean sourceSV = sourceMetadata.isSingleValue(); TransformFunction castFormatTransformFunction = arguments.get(1); - boolean isSVCol = _transformFunction.getResultMetadata().isSingleValue(); - if (castFormatTransformFunction instanceof LiteralTransformFunction) { String targetType = ((LiteralTransformFunction) castFormatTransformFunction).getLiteral().toUpperCase(); switch (targetType) { case "INT": case "INTEGER": - _resultMetadata = isSVCol ? INT_SV_NO_DICTIONARY_METADATA : INT_MV_NO_DICTIONARY_METADATA; + _resultMetadata = sourceSV ? INT_SV_NO_DICTIONARY_METADATA : INT_MV_NO_DICTIONARY_METADATA; break; case "LONG": - _resultMetadata = isSVCol ? LONG_SV_NO_DICTIONARY_METADATA : LONG_MV_NO_DICTIONARY_METADATA; + _resultMetadata = sourceSV ? LONG_SV_NO_DICTIONARY_METADATA : LONG_MV_NO_DICTIONARY_METADATA; break; case "FLOAT": - _resultMetadata = isSVCol ? FLOAT_SV_NO_DICTIONARY_METADATA : FLOAT_MV_NO_DICTIONARY_METADATA; + _resultMetadata = sourceSV ? FLOAT_SV_NO_DICTIONARY_METADATA : FLOAT_MV_NO_DICTIONARY_METADATA; break; case "DOUBLE": - _resultMetadata = isSVCol ? DOUBLE_SV_NO_DICTIONARY_METADATA : DOUBLE_MV_NO_DICTIONARY_METADATA; + _resultMetadata = sourceSV ? DOUBLE_SV_NO_DICTIONARY_METADATA : DOUBLE_MV_NO_DICTIONARY_METADATA; break; case "DECIMAL": case "BIGDECIMAL": case "BIG_DECIMAL": - if (!isSVCol) { - // TODO: MV cast to BIG_DECIMAL type - throw new IllegalArgumentException( - "Cast is not supported on multi-value column to target type: " + targetType); - } + // TODO: Support MV BIG_DECIMAL + Preconditions.checkState(sourceSV, "Cannot cast from MV to BIG_DECIMAL"); _resultMetadata = BIG_DECIMAL_SV_NO_DICTIONARY_METADATA; break; case "BOOL": case "BOOLEAN": - if (!isSVCol) { - throw new IllegalArgumentException( - "Cast is not supported on multi-value column to target type: " + targetType); - } - _resultMetadata = BOOLEAN_SV_NO_DICTIONARY_METADATA; + _resultMetadata = sourceSV ? BOOLEAN_SV_NO_DICTIONARY_METADATA : BOOLEAN_MV_NO_DICTIONARY_METADATA; break; case "TIMESTAMP": - if (!isSVCol) { - throw new IllegalArgumentException( - "Cast is not supported on multi-value column to target type: " + targetType); - } - _resultMetadata = TIMESTAMP_SV_NO_DICTIONARY_METADATA; + _resultMetadata = sourceSV ? TIMESTAMP_SV_NO_DICTIONARY_METADATA : TIMESTAMP_MV_NO_DICTIONARY_METADATA; break; case "STRING": case "VARCHAR": - _resultMetadata = isSVCol ? STRING_SV_NO_DICTIONARY_METADATA : STRING_MV_NO_DICTIONARY_METADATA; + _resultMetadata = sourceSV ? STRING_SV_NO_DICTIONARY_METADATA : STRING_MV_NO_DICTIONARY_METADATA; break; case "JSON": - if (!isSVCol) { - throw new IllegalArgumentException( - "Cast is not supported on multi-value column to target type: " + targetType); - } - _resultMetadata = JSON_SV_NO_DICTIONARY_METADATA; + _resultMetadata = sourceSV ? JSON_SV_NO_DICTIONARY_METADATA : JSON_MV_NO_DICTIONARY_METADATA; break; default: throw new IllegalArgumentException("Unable to cast expression to type - " + targetType); @@ -115,105 +104,325 @@ public TransformResultMetadata getResultMetadata() { } @Override - public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) { - DataType resultStoredType = _resultMetadata.getDataType().getStoredType(); - if (resultStoredType == DataType.DOUBLE) { - return _transformFunction.transformToDoubleValuesMV(projectionBlock); - } else { - return super.transformToDoubleValuesMV(projectionBlock); + public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { + switch (_resultMetadata.getDataType()) { + case INT: + return _transformFunction.transformToIntValuesSV(projectionBlock); + case BOOLEAN: + return transformToBooleanValuesSV(projectionBlock); + default: + return super.transformToIntValuesSV(projectionBlock); + } + } + + // TODO: Add it to the interface + private int[] transformToBooleanValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_intValuesSV == null) { + _intValuesSV = new int[length]; + } + switch (_sourceDataType.getStoredType()) { + case INT: + int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock); + ArrayCopyUtils.copyToBoolean(intValues, _intValuesSV, length); + break; + case LONG: + long[] longValues = _transformFunction.transformToLongValuesSV(projectionBlock); + ArrayCopyUtils.copyToBoolean(longValues, _intValuesSV, length); + break; + case FLOAT: + float[] floatValues = _transformFunction.transformToFloatValuesSV(projectionBlock); + ArrayCopyUtils.copyToBoolean(floatValues, _intValuesSV, length); + break; + case DOUBLE: + double[] doubleValues = _transformFunction.transformToDoubleValuesSV(projectionBlock); + ArrayCopyUtils.copyToBoolean(doubleValues, _intValuesSV, length); + break; + case BIG_DECIMAL: + BigDecimal[] bigDecimalValues = _transformFunction.transformToBigDecimalValuesSV(projectionBlock); + ArrayCopyUtils.copyToBoolean(bigDecimalValues, _intValuesSV, length); + break; + case STRING: + String[] stringValues = _transformFunction.transformToStringValuesSV(projectionBlock); + ArrayCopyUtils.copyToBoolean(stringValues, _intValuesSV, length); + break; + default: + throw new IllegalStateException(String.format("Cannot cast from SV %s to BOOLEAN", _sourceDataType)); } + return _intValuesSV; } @Override - public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) { - if (_resultMetadata.getDataType().getStoredType() == DataType.STRING) { - return _transformFunction.transformToStringValuesMV(projectionBlock); + public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) { + switch (_resultMetadata.getDataType()) { + case LONG: + return _transformFunction.transformToLongValuesSV(projectionBlock); + case TIMESTAMP: + return transformToTimestampValuesSV(projectionBlock); + default: + return super.transformToLongValuesSV(projectionBlock); + } + } + + // TODO: Add it to the interface + private long[] transformToTimestampValuesSV(ProjectionBlock projectionBlock) { + if (_sourceDataType.getStoredType() == DataType.STRING) { + int length = projectionBlock.getNumDocs(); + if (_longValuesSV == null) { + _longValuesSV = new long[length]; + } + String[] stringValues = _transformFunction.transformToStringValuesSV(projectionBlock); + ArrayCopyUtils.copyToTimestamp(stringValues, _longValuesSV, length); + return _longValuesSV; } else { - return super.transformToStringValuesMV(projectionBlock); + return _transformFunction.transformToLongValuesSV(projectionBlock); } } @Override - public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) { - DataType resultStoredType = _resultMetadata.getDataType().getStoredType(); - if (resultStoredType == DataType.INT) { - return _transformFunction.transformToIntValuesMV(projectionBlock); + public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) { + if (_resultMetadata.getDataType().getStoredType() == DataType.FLOAT) { + return _transformFunction.transformToFloatValuesSV(projectionBlock); } else { - return super.transformToIntValuesMV(projectionBlock); + return super.transformToFloatValuesSV(projectionBlock); } } @Override - public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) { - DataType resultStoredType = _resultMetadata.getDataType().getStoredType(); - if (resultStoredType == DataType.FLOAT) { - return _transformFunction.transformToFloatValuesMV(projectionBlock); + public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { + if (_resultMetadata.getDataType().getStoredType() == DataType.DOUBLE) { + return _transformFunction.transformToDoubleValuesSV(projectionBlock); } else { - return super.transformToFloatValuesMV(projectionBlock); + return super.transformToDoubleValuesSV(projectionBlock); } } @Override - public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) { - DataType resultStoredType = _resultMetadata.getDataType().getStoredType(); - if (resultStoredType == DataType.LONG) { - return _transformFunction.transformToLongValuesMV(projectionBlock); + public BigDecimal[] transformToBigDecimalValuesSV(ProjectionBlock projectionBlock) { + if (_resultMetadata.getDataType().getStoredType() == DataType.BIG_DECIMAL) { + return _transformFunction.transformToBigDecimalValuesSV(projectionBlock); } else { - return super.transformToLongValuesMV(projectionBlock); + return super.transformToBigDecimalValuesSV(projectionBlock); } } @Override - public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { - if (_resultMetadata.getDataType().getStoredType() == DataType.INT) { - return _transformFunction.transformToIntValuesSV(projectionBlock); + public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) { + DataType resultDataType = _resultMetadata.getDataType(); + if (resultDataType.getStoredType() == DataType.STRING) { + switch (_sourceDataType) { + case BOOLEAN: + int length = projectionBlock.getNumDocs(); + if (_stringValuesSV == null) { + _stringValuesSV = new String[length]; + } + int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock); + ArrayCopyUtils.copyFromBoolean(intValues, _stringValuesSV, length); + return _stringValuesSV; + case TIMESTAMP: + length = projectionBlock.getNumDocs(); + if (_stringValuesSV == null) { + _stringValuesSV = new String[length]; + } + long[] longValues = _transformFunction.transformToLongValuesSV(projectionBlock); + ArrayCopyUtils.copyFromTimestamp(longValues, _stringValuesSV, length); + return _stringValuesSV; + default: + return _transformFunction.transformToStringValuesSV(projectionBlock); + } } else { - return super.transformToIntValuesSV(projectionBlock); + int length = projectionBlock.getNumDocs(); + if (_stringValuesSV == null) { + _stringValuesSV = new String[length]; + } + switch (resultDataType) { + case INT: + int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock); + ArrayCopyUtils.copy(intValues, _stringValuesSV, length); + break; + case LONG: + long[] longValues = _transformFunction.transformToLongValuesSV(projectionBlock); + ArrayCopyUtils.copy(longValues, _stringValuesSV, length); + break; + case FLOAT: + float[] floatValues = _transformFunction.transformToFloatValuesSV(projectionBlock); + ArrayCopyUtils.copy(floatValues, _stringValuesSV, length); + break; + case DOUBLE: + double[] doubleValues = _transformFunction.transformToDoubleValuesSV(projectionBlock); + ArrayCopyUtils.copy(doubleValues, _stringValuesSV, length); + break; + case BIG_DECIMAL: + BigDecimal[] bigDecimalValues = _transformFunction.transformToBigDecimalValuesSV(projectionBlock); + ArrayCopyUtils.copy(bigDecimalValues, _stringValuesSV, length); + break; + case BOOLEAN: + intValues = transformToBooleanValuesSV(projectionBlock); + ArrayCopyUtils.copyFromBoolean(intValues, _stringValuesSV, length); + break; + case TIMESTAMP: + longValues = transformToTimestampValuesSV(projectionBlock); + ArrayCopyUtils.copyFromTimestamp(longValues, _stringValuesSV, length); + break; + case BYTES: + byte[][] bytesValues = transformToBytesValuesSV(projectionBlock); + ArrayCopyUtils.copy(bytesValues, _stringValuesSV, length); + break; + default: + throw new IllegalStateException(String.format("Cannot cast from SV %s to STRING", resultDataType)); + } } + return _stringValuesSV; } @Override - public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) { - if (_resultMetadata.getDataType().getStoredType() == DataType.LONG) { - return _transformFunction.transformToLongValuesSV(projectionBlock); - } else { - return super.transformToLongValuesSV(projectionBlock); + public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) { + switch (_resultMetadata.getDataType()) { + case INT: + return _transformFunction.transformToIntValuesMV(projectionBlock); + case BOOLEAN: + return transformToBooleanValuesMV(projectionBlock); + default: + return super.transformToIntValuesMV(projectionBlock); } } + // TODO: Add it to the interface + private int[][] transformToBooleanValuesMV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_intValuesMV == null) { + _intValuesMV = new int[length][]; + } + switch (_sourceDataType.getStoredType()) { + case INT: + int[][] intValuesMV = _transformFunction.transformToIntValuesMV(projectionBlock); + ArrayCopyUtils.copyToBoolean(intValuesMV, _intValuesMV, length); + break; + case LONG: + long[][] longValuesMV = _transformFunction.transformToLongValuesMV(projectionBlock); + ArrayCopyUtils.copyToBoolean(longValuesMV, _intValuesMV, length); + break; + case FLOAT: + float[][] floatValuesMV = _transformFunction.transformToFloatValuesMV(projectionBlock); + ArrayCopyUtils.copyToBoolean(floatValuesMV, _intValuesMV, length); + break; + case DOUBLE: + double[][] doubleValuesMV = _transformFunction.transformToDoubleValuesMV(projectionBlock); + ArrayCopyUtils.copyToBoolean(doubleValuesMV, _intValuesMV, length); + break; + case STRING: + String[][] stringValuesMV = _transformFunction.transformToStringValuesMV(projectionBlock); + ArrayCopyUtils.copyToBoolean(stringValuesMV, _intValuesMV, length); + break; + default: + throw new IllegalStateException(String.format("Cannot cast from MV %s to BOOLEAN", _sourceDataType)); + } + return _intValuesMV; + } + @Override - public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) { - if (_resultMetadata.getDataType().getStoredType() == DataType.FLOAT) { - return _transformFunction.transformToFloatValuesSV(projectionBlock); + public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) { + switch (_resultMetadata.getDataType()) { + case LONG: + return _transformFunction.transformToLongValuesMV(projectionBlock); + case TIMESTAMP: + return transformToTimestampValuesMV(projectionBlock); + default: + return super.transformToLongValuesMV(projectionBlock); + } + } + + // TODO: Add it to the interface + private long[][] transformToTimestampValuesMV(ProjectionBlock projectionBlock) { + if (_sourceDataType.getStoredType() == DataType.STRING) { + int length = projectionBlock.getNumDocs(); + if (_longValuesMV == null) { + _longValuesMV = new long[length][]; + } + String[][] stringValuesMV = _transformFunction.transformToStringValuesMV(projectionBlock); + ArrayCopyUtils.copyToTimestamp(stringValuesMV, _longValuesMV, length); + return _longValuesMV; } else { - return super.transformToFloatValuesSV(projectionBlock); + return _transformFunction.transformToLongValuesMV(projectionBlock); } } @Override - public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { - if (_resultMetadata.getDataType().getStoredType() == DataType.DOUBLE) { - return _transformFunction.transformToDoubleValuesSV(projectionBlock); + public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) { + if (_resultMetadata.getDataType().getStoredType() == DataType.FLOAT) { + return _transformFunction.transformToFloatValuesMV(projectionBlock); } else { - return super.transformToDoubleValuesSV(projectionBlock); + return super.transformToFloatValuesMV(projectionBlock); } } @Override - public BigDecimal[] transformToBigDecimalValuesSV(ProjectionBlock projectionBlock) { - if (_resultMetadata.getDataType().getStoredType() == DataType.BIG_DECIMAL) { - return _transformFunction.transformToBigDecimalValuesSV(projectionBlock); + public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) { + if (_resultMetadata.getDataType().getStoredType() == DataType.DOUBLE) { + return _transformFunction.transformToDoubleValuesMV(projectionBlock); } else { - return super.transformToBigDecimalValuesSV(projectionBlock); + return super.transformToDoubleValuesMV(projectionBlock); } } @Override - public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) { - if (_resultMetadata.getDataType().getStoredType() == DataType.STRING) { - return _transformFunction.transformToStringValuesSV(projectionBlock); + public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) { + DataType resultDataType = _resultMetadata.getDataType(); + if (resultDataType.getStoredType() == DataType.STRING) { + switch (_sourceDataType) { + case BOOLEAN: + int length = projectionBlock.getNumDocs(); + if (_stringValuesMV == null) { + _stringValuesMV = new String[length][]; + } + int[][] intValuesMV = _transformFunction.transformToIntValuesMV(projectionBlock); + ArrayCopyUtils.copyFromBoolean(intValuesMV, _stringValuesMV, length); + return _stringValuesMV; + case TIMESTAMP: + length = projectionBlock.getNumDocs(); + if (_stringValuesMV == null) { + _stringValuesMV = new String[length][]; + } + long[][] longValuesMV = _transformFunction.transformToLongValuesMV(projectionBlock); + ArrayCopyUtils.copyFromTimestamp(longValuesMV, _stringValuesMV, length); + return _stringValuesMV; + default: + return _transformFunction.transformToStringValuesMV(projectionBlock); + } } else { - return super.transformToStringValuesSV(projectionBlock); + int length = projectionBlock.getNumDocs(); + if (_stringValuesMV == null) { + _stringValuesMV = new String[length][]; + } + switch (resultDataType) { + case INT: + int[][] intValuesMV = _transformFunction.transformToIntValuesMV(projectionBlock); + ArrayCopyUtils.copy(intValuesMV, _stringValuesMV, length); + break; + case LONG: + long[][] longValuesMV = _transformFunction.transformToLongValuesMV(projectionBlock); + ArrayCopyUtils.copy(longValuesMV, _stringValuesMV, length); + break; + case FLOAT: + float[][] floatValuesMV = _transformFunction.transformToFloatValuesMV(projectionBlock); + ArrayCopyUtils.copy(floatValuesMV, _stringValuesMV, length); + break; + case DOUBLE: + double[][] doubleValuesMV = _transformFunction.transformToDoubleValuesMV(projectionBlock); + ArrayCopyUtils.copy(doubleValuesMV, _stringValuesMV, length); + break; + case BOOLEAN: + intValuesMV = transformToBooleanValuesMV(projectionBlock); + ArrayCopyUtils.copyFromBoolean(intValuesMV, _stringValuesMV, length); + break; + case TIMESTAMP: + longValuesMV = transformToTimestampValuesMV(projectionBlock); + ArrayCopyUtils.copyFromTimestamp(longValuesMV, _stringValuesMV, length); + break; + default: + throw new IllegalStateException(String.format("Cannot cast from MV %s to STRING", resultDataType)); + } } + return _stringValuesMV; } } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java index c96f4b9d07ad..10e834c63544 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java @@ -24,7 +24,6 @@ import org.apache.pinot.core.operator.blocks.ProjectionBlock; import org.apache.pinot.core.operator.transform.TransformResultMetadata; import org.apache.pinot.segment.spi.datasource.DataSource; -import org.apache.pinot.spi.utils.ArrayCopyUtils; /** @@ -33,15 +32,15 @@ */ public abstract class LogicalOperatorTransformFunction extends BaseTransformFunction { protected List _arguments; - protected int[] _results; @Override public void init(List arguments, Map dataSourceMap) { _arguments = arguments; int numArguments = arguments.size(); if (numArguments <= 1) { - throw new IllegalArgumentException("Expect more than 1 argument for logical operator [" + getName() + "], args [" - + Arrays.toString(arguments.toArray()) + "]."); + throw new IllegalArgumentException( + "Expect more than 1 argument for logical operator [" + getName() + "], args [" + Arrays.toString( + arguments.toArray()) + "]."); } for (int i = 0; i < numArguments; i++) { TransformResultMetadata argumentMetadata = arguments.get(i).getResultMetadata(); @@ -60,20 +59,19 @@ public TransformResultMetadata getResultMetadata() { @Override public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { int numDocs = projectionBlock.getNumDocs(); - - if (_results == null || _results.length < numDocs) { - _results = new int[numDocs]; + if (_intValuesSV == null) { + _intValuesSV = new int[numDocs]; } - ArrayCopyUtils.copy(_arguments.get(0).transformToIntValuesSV(projectionBlock), _results, numDocs); + System.arraycopy(_arguments.get(0).transformToIntValuesSV(projectionBlock), 0, _intValuesSV, 0, numDocs); int numArguments = _arguments.size(); for (int i = 1; i < numArguments; i++) { TransformFunction transformFunction = _arguments.get(i); int[] results = transformFunction.transformToIntValuesSV(projectionBlock); for (int j = 0; j < numDocs; j++) { - _results[j] = getLogicalFuncResult(_results[j], results[j]); + _intValuesSV[j] = getLogicalFuncResult(_intValuesSV[j], results[j]); } } - return _results; + return _intValuesSV; } abstract int getLogicalFuncResult(int left, int right); diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LookupTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LookupTransformFunction.java index b931a13c036d..460966972fd3 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LookupTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LookupTransformFunction.java @@ -227,8 +227,11 @@ private void lookup(ProjectionBlock projectionBlock, ValueAcceptor valueAcceptor @Override public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.INT) { + return super.transformToIntValuesSV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_intValuesSV == null || _intValuesSV.length < numDocs) { + if (_intValuesSV == null) { _intValuesSV = new int[numDocs]; } lookup(projectionBlock, this::setIntSV); @@ -237,8 +240,11 @@ public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { @Override public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.LONG) { + return super.transformToLongValuesSV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_longValuesSV == null || _longValuesSV.length < numDocs) { + if (_longValuesSV == null) { _longValuesSV = new long[numDocs]; } lookup(projectionBlock, this::setLongSV); @@ -247,8 +253,11 @@ public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) { @Override public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.FLOAT) { + return super.transformToFloatValuesSV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_floatValuesSV == null || _floatValuesSV.length < numDocs) { + if (_floatValuesSV == null) { _floatValuesSV = new float[numDocs]; } lookup(projectionBlock, this::setFloatSV); @@ -257,8 +266,11 @@ public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) { @Override public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.DOUBLE) { + return super.transformToDoubleValuesSV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_doubleValuesSV == null || _doubleValuesSV.length < numDocs) { + if (_doubleValuesSV == null) { _doubleValuesSV = new double[numDocs]; } lookup(projectionBlock, this::setDoubleSV); @@ -267,8 +279,11 @@ public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { @Override public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.STRING) { + return super.transformToStringValuesSV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_stringValuesSV == null || _stringValuesSV.length < numDocs) { + if (_stringValuesSV == null) { _stringValuesSV = new String[numDocs]; } lookup(projectionBlock, this::setStringSV); @@ -277,18 +292,24 @@ public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) { @Override public byte[][] transformToBytesValuesSV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.BYTES) { + return super.transformToBytesValuesSV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_byteValuesSV == null || _byteValuesSV.length < numDocs) { - _byteValuesSV = new byte[numDocs][]; + if (_bytesValuesSV == null) { + _bytesValuesSV = new byte[numDocs][]; } lookup(projectionBlock, this::setBytesSV); - return _byteValuesSV; + return _bytesValuesSV; } @Override public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.INT) { + return super.transformToIntValuesMV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_intValuesMV == null || _intValuesMV.length < numDocs) { + if (_intValuesMV == null) { _intValuesMV = new int[numDocs][]; } lookup(projectionBlock, this::setIntMV); @@ -297,8 +318,11 @@ public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) { @Override public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.LONG) { + return super.transformToLongValuesMV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_longValuesMV == null || _longValuesMV.length < numDocs) { + if (_longValuesMV == null) { _longValuesMV = new long[numDocs][]; } lookup(projectionBlock, this::setLongMV); @@ -307,8 +331,11 @@ public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) { @Override public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.FLOAT) { + return super.transformToFloatValuesMV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_floatValuesMV == null || _floatValuesMV.length < numDocs) { + if (_floatValuesMV == null) { _floatValuesMV = new float[numDocs][]; } lookup(projectionBlock, this::setFloatMV); @@ -317,8 +344,11 @@ public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) { @Override public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.DOUBLE) { + return super.transformToDoubleValuesMV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_doubleValuesMV == null || _doubleValuesMV.length < numDocs) { + if (_doubleValuesMV == null) { _doubleValuesMV = new double[numDocs][]; } lookup(projectionBlock, this::setDoubleMV); @@ -327,8 +357,11 @@ public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) { @Override public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) { + if (_lookupColumnFieldSpec.getDataType().getStoredType() != DataType.STRING) { + return super.transformToStringValuesMV(projectionBlock); + } int numDocs = projectionBlock.getNumDocs(); - if (_stringValuesMV == null || _stringValuesMV.length < numDocs) { + if (_stringValuesMV == null) { _stringValuesMV = new String[numDocs][]; } lookup(projectionBlock, this::setStringMV); @@ -377,9 +410,9 @@ private void setStringSV(int index, Object value) { private void setBytesSV(int index, Object value) { if (value instanceof byte[]) { - _byteValuesSV[index] = (byte[]) value; + _bytesValuesSV[index] = (byte[]) value; } else { - _byteValuesSV[index] = EMPTY_BYTES; + _bytesValuesSV[index] = EMPTY_BYTES; } } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java index 0fcf6a8a5028..16c186826d51 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java @@ -43,7 +43,7 @@ import org.apache.pinot.core.operator.transform.TransformResultMetadata; import org.apache.pinot.segment.spi.datasource.DataSource; import org.apache.pinot.segment.spi.index.reader.Dictionary; -import org.apache.pinot.spi.data.FieldSpec; +import org.apache.pinot.spi.data.FieldSpec.DataType; public class ValueInTransformFunction extends BaseTransformFunction { @@ -132,7 +132,7 @@ public int[][] transformToDictIdsMV(ProjectionBlock projectionBlock) { @Override public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) { - if (_dictionary != null || _resultMetadata.getDataType() != FieldSpec.DataType.INT) { + if (_dictionary != null || _resultMetadata.getDataType().getStoredType() != DataType.INT) { return super.transformToIntValuesMV(projectionBlock); } @@ -155,7 +155,7 @@ public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) { @Override public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) { - if (_dictionary != null || _resultMetadata.getDataType() != FieldSpec.DataType.LONG) { + if (_dictionary != null || _resultMetadata.getDataType().getStoredType() != DataType.LONG) { return super.transformToLongValuesMV(projectionBlock); } @@ -178,7 +178,7 @@ public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) { @Override public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) { - if (_dictionary != null || _resultMetadata.getDataType() != FieldSpec.DataType.FLOAT) { + if (_dictionary != null || _resultMetadata.getDataType().getStoredType() != DataType.FLOAT) { return super.transformToFloatValuesMV(projectionBlock); } @@ -201,7 +201,7 @@ public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) { @Override public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) { - if (_dictionary != null || _resultMetadata.getDataType() != FieldSpec.DataType.DOUBLE) { + if (_dictionary != null || _resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) { return super.transformToDoubleValuesMV(projectionBlock); } @@ -224,7 +224,7 @@ public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) { @Override public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) { - if (_dictionary != null || _resultMetadata.getDataType() != FieldSpec.DataType.STRING) { + if (_dictionary != null || _resultMetadata.getDataType().getStoredType() != DataType.STRING) { return super.transformToStringValuesMV(projectionBlock); } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunctionTest.java index 07d82d6d2df5..a0aece3a1f86 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunctionTest.java @@ -18,17 +18,18 @@ */ package org.apache.pinot.core.operator.transform.function; +import org.apache.pinot.common.function.TransformFunctionType; + + public class AndOperatorTransformFunctionTest extends LogicalOperatorTransformFunctionTest { + @Override - int getExpectedValue(boolean left, boolean right) { - if (left && right) { - return 1; - } - return 0; + boolean getExpectedValue(boolean left, boolean right) { + return left && right; } @Override - String getFuncName() { - return new AndOperatorTransformFunction().getName(); + String getFunctionName() { + return TransformFunctionType.AND.getName(); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java index bfd60f6cc082..d4580b607299 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java @@ -52,13 +52,15 @@ import org.apache.pinot.spi.data.TimeGranularitySpec; import org.apache.pinot.spi.data.readers.GenericRow; import org.apache.pinot.spi.utils.BigDecimalUtils; +import org.apache.pinot.spi.utils.BytesUtils; import org.apache.pinot.spi.utils.JsonUtils; import org.apache.pinot.spi.utils.ReadMode; import org.apache.pinot.spi.utils.builder.TableConfigBuilder; -import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import static org.testng.Assert.assertEquals; + public abstract class BaseTransformFunctionTest { private static final String SEGMENT_NAME = "testSegment"; @@ -221,16 +223,12 @@ protected void testTransformFunction(TransformFunction transformFunction, int[] BigDecimal[] bigDecimalValues = transformFunction.transformToBigDecimalValuesSV(_projectionBlock); String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(intValues[i], expectedValues[i]); - Assert.assertEquals(longValues[i], expectedValues[i]); - Assert.assertEquals(floatValues[i], (float) expectedValues[i]); - Assert.assertEquals(doubleValues[i], (double) expectedValues[i]); - Assert.assertEquals(bigDecimalValues[i].intValue(), expectedValues[i]); - if (transformFunction.getResultMetadata().getDataType() == FieldSpec.DataType.BOOLEAN) { - Assert.assertEquals(stringValues[i], Boolean.toString(expectedValues[i] == 1)); - } else { - Assert.assertEquals(stringValues[i], Integer.toString(expectedValues[i])); - } + assertEquals(intValues[i], expectedValues[i]); + assertEquals(longValues[i], expectedValues[i]); + assertEquals(floatValues[i], (float) expectedValues[i]); + assertEquals(doubleValues[i], (double) expectedValues[i]); + assertEquals(bigDecimalValues[i].intValue(), expectedValues[i]); + assertEquals(stringValues[i], Integer.toString(expectedValues[i])); } } @@ -242,16 +240,12 @@ protected void testTransformFunction(TransformFunction transformFunction, long[] BigDecimal[] bigDecimalValues = transformFunction.transformToBigDecimalValuesSV(_projectionBlock); String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(intValues[i], (int) expectedValues[i]); - Assert.assertEquals(longValues[i], expectedValues[i]); - Assert.assertEquals(floatValues[i], (float) expectedValues[i]); - Assert.assertEquals(doubleValues[i], (double) expectedValues[i]); - Assert.assertEquals(bigDecimalValues[i].longValue(), expectedValues[i]); - if (transformFunction.getResultMetadata().getDataType() == FieldSpec.DataType.TIMESTAMP) { - Assert.assertEquals(stringValues[i], new Timestamp(expectedValues[i]).toString()); - } else { - Assert.assertEquals(stringValues[i], Long.toString(expectedValues[i])); - } + assertEquals(intValues[i], (int) expectedValues[i]); + assertEquals(longValues[i], expectedValues[i]); + assertEquals(floatValues[i], (float) expectedValues[i]); + assertEquals(doubleValues[i], (double) expectedValues[i]); + assertEquals(bigDecimalValues[i].longValue(), expectedValues[i]); + assertEquals(stringValues[i], Long.toString(expectedValues[i])); } } @@ -263,12 +257,12 @@ protected void testTransformFunction(TransformFunction transformFunction, float[ BigDecimal[] bigDecimalValues = transformFunction.transformToBigDecimalValuesSV(_projectionBlock); String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(intValues[i], (int) expectedValues[i]); - Assert.assertEquals(longValues[i], (long) expectedValues[i]); - Assert.assertEquals(floatValues[i], expectedValues[i]); - Assert.assertEquals(doubleValues[i], (double) expectedValues[i]); - Assert.assertEquals(bigDecimalValues[i].floatValue(), expectedValues[i]); - Assert.assertEquals(stringValues[i], Float.toString(expectedValues[i])); + assertEquals(intValues[i], (int) expectedValues[i]); + assertEquals(longValues[i], (long) expectedValues[i]); + assertEquals(floatValues[i], expectedValues[i]); + assertEquals(doubleValues[i], (double) expectedValues[i]); + assertEquals(bigDecimalValues[i].floatValue(), expectedValues[i]); + assertEquals(stringValues[i], Float.toString(expectedValues[i])); } } @@ -287,14 +281,14 @@ protected void testTransformFunction(TransformFunction transformFunction, double } String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(intValues[i], (int) expectedValues[i]); - Assert.assertEquals(longValues[i], (long) expectedValues[i]); - Assert.assertEquals(floatValues[i], (float) expectedValues[i]); - Assert.assertEquals(doubleValues[i], expectedValues[i]); + assertEquals(intValues[i], (int) expectedValues[i]); + assertEquals(longValues[i], (long) expectedValues[i]); + assertEquals(floatValues[i], (float) expectedValues[i]); + assertEquals(doubleValues[i], expectedValues[i]); if (bigDecimalValues != null) { - Assert.assertEquals(bigDecimalValues[i].doubleValue(), expectedValues[i]); + assertEquals(bigDecimalValues[i].doubleValue(), expectedValues[i]); } - Assert.assertEquals(stringValues[i], Double.toString(expectedValues[i])); + assertEquals(stringValues[i], Double.toString(expectedValues[i])); } } @@ -304,65 +298,171 @@ protected void testTransformFunction(TransformFunction transformFunction, BigDec float[] floatValues = transformFunction.transformToFloatValuesSV(_projectionBlock); double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock); BigDecimal[] bigDecimalValues = transformFunction.transformToBigDecimalValuesSV(_projectionBlock); - byte[][] bytesValues = transformFunction.transformToBytesValuesSV(_projectionBlock); String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); + byte[][] bytesValues = transformFunction.transformToBytesValuesSV(_projectionBlock); + for (int i = 0; i < NUM_ROWS; i++) { + assertEquals(intValues[i], expectedValues[i].intValue()); + assertEquals(longValues[i], expectedValues[i].longValue()); + assertEquals(floatValues[i], expectedValues[i].floatValue()); + assertEquals(doubleValues[i], expectedValues[i].doubleValue()); + assertEquals(bigDecimalValues[i].compareTo(expectedValues[i]), 0); + assertEquals((new BigDecimal(stringValues[i])).compareTo(expectedValues[i]), 0); + assertEquals(BigDecimalUtils.deserialize(bytesValues[i]).compareTo(expectedValues[i]), 0); + } + } + + protected void testTransformFunction(TransformFunction transformFunction, boolean[] expectedValues) { + int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock); + long[] longValues = transformFunction.transformToLongValuesSV(_projectionBlock); + float[] floatValues = transformFunction.transformToFloatValuesSV(_projectionBlock); + double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock); + BigDecimal[] bigDecimalValues = transformFunction.transformToBigDecimalValuesSV(_projectionBlock); + // TODO: Support implicit cast from BOOLEAN to STRING +// String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); + for (int i = 0; i < NUM_ROWS; i++) { + assertEquals(intValues[i] == 1, expectedValues[i]); + assertEquals(longValues[i] == 1, expectedValues[i]); + assertEquals(floatValues[i] == 1, expectedValues[i]); + assertEquals(doubleValues[i] == 1, expectedValues[i]); + assertEquals(bigDecimalValues[i].intValue() == 1, expectedValues[i]); +// assertEquals(stringValues[i], Boolean.toString(expectedValues[i])); + } + } + + protected void testTransformFunction(TransformFunction transformFunction, Timestamp[] expectedValues) { + int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock); + long[] longValues = transformFunction.transformToLongValuesSV(_projectionBlock); + float[] floatValues = transformFunction.transformToFloatValuesSV(_projectionBlock); + double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock); + BigDecimal[] bigDecimalValues = transformFunction.transformToBigDecimalValuesSV(_projectionBlock); + // TODO: Support implicit cast from TIMESTAMP to STRING +// String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(intValues[i], expectedValues[i].intValue()); - Assert.assertEquals(longValues[i], expectedValues[i].longValue()); - Assert.assertEquals(floatValues[i], expectedValues[i].floatValue()); - Assert.assertEquals(doubleValues[i], expectedValues[i].doubleValue()); - Assert.assertEquals(bigDecimalValues[i].compareTo(expectedValues[i]), 0); - Assert.assertEquals(BigDecimalUtils.deserialize(bytesValues[i]).compareTo(expectedValues[i]), 0); - Assert.assertEquals((new BigDecimal(stringValues[i])).compareTo(expectedValues[i]), 0); + assertEquals(intValues[i], (int) expectedValues[i].getTime()); + assertEquals(longValues[i], expectedValues[i].getTime()); + assertEquals(floatValues[i], (float) expectedValues[i].getTime()); + assertEquals(doubleValues[i], (double) expectedValues[i].getTime()); + assertEquals(bigDecimalValues[i], BigDecimal.valueOf(expectedValues[i].getTime())); +// assertEquals(stringValues[i], expectedValues[i].toString()); } } protected void testTransformFunction(TransformFunction transformFunction, String[] expectedValues) { String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(stringValues[i], expectedValues[i]); + assertEquals(stringValues[i], expectedValues[i]); } } protected void testTransformFunction(TransformFunction transformFunction, byte[][] expectedValues) { + String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); byte[][] bytesValues = transformFunction.transformToBytesValuesSV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(bytesValues[i], expectedValues[i]); + assertEquals(bytesValues[i], BytesUtils.toBytes(stringValues[i])); + assertEquals(bytesValues[i], expectedValues[i]); } } protected void testTransformFunctionMV(TransformFunction transformFunction, int[][] expectedValues) { - int[][] intMVValues = transformFunction.transformToIntValuesMV(_projectionBlock); + int[][] intValuesMV = transformFunction.transformToIntValuesMV(_projectionBlock); + long[][] longValuesMV = transformFunction.transformToLongValuesMV(_projectionBlock); + float[][] floatValuesMV = transformFunction.transformToFloatValuesMV(_projectionBlock); + double[][] doubleValuesMV = transformFunction.transformToDoubleValuesMV(_projectionBlock); + String[][] stringValuesMV = transformFunction.transformToStringValuesMV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(intMVValues[i], expectedValues[i]); + int[] expectedValueMV = expectedValues[i]; + int numValues = expectedValueMV.length; + assertEquals(intValuesMV[i].length, numValues); + assertEquals(longValuesMV[i].length, numValues); + assertEquals(floatValuesMV[i].length, numValues); + assertEquals(doubleValuesMV[i].length, numValues); + assertEquals(stringValuesMV[i].length, numValues); + for (int j = 0; j < numValues; j++) { + assertEquals(intValuesMV[i][j], expectedValues[i][j]); + assertEquals(longValuesMV[i][j], expectedValues[i][j]); + assertEquals(floatValuesMV[i][j], (float) expectedValues[i][j]); + assertEquals(doubleValuesMV[i][j], (double) expectedValues[i][j]); + assertEquals(stringValuesMV[i][j], Integer.toString(expectedValues[i][j])); + } } } protected void testTransformFunctionMV(TransformFunction transformFunction, long[][] expectedValues) { - long[][] longMVValues = transformFunction.transformToLongValuesMV(_projectionBlock); + int[][] intValuesMV = transformFunction.transformToIntValuesMV(_projectionBlock); + long[][] longValuesMV = transformFunction.transformToLongValuesMV(_projectionBlock); + float[][] floatValuesMV = transformFunction.transformToFloatValuesMV(_projectionBlock); + double[][] doubleValuesMV = transformFunction.transformToDoubleValuesMV(_projectionBlock); + String[][] stringValuesMV = transformFunction.transformToStringValuesMV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(longMVValues[i], expectedValues[i]); + long[] expectedValueMV = expectedValues[i]; + int numValues = expectedValueMV.length; + assertEquals(intValuesMV[i].length, numValues); + assertEquals(longValuesMV[i].length, numValues); + assertEquals(floatValuesMV[i].length, numValues); + assertEquals(doubleValuesMV[i].length, numValues); + assertEquals(stringValuesMV[i].length, numValues); + for (int j = 0; j < numValues; j++) { + assertEquals(intValuesMV[i][j], (int) expectedValues[i][j]); + assertEquals(longValuesMV[i][j], expectedValues[i][j]); + assertEquals(floatValuesMV[i][j], (float) expectedValues[i][j]); + assertEquals(doubleValuesMV[i][j], (double) expectedValues[i][j]); + assertEquals(stringValuesMV[i][j], Long.toString(expectedValues[i][j])); + } } } protected void testTransformFunctionMV(TransformFunction transformFunction, float[][] expectedValues) { - float[][] floatMVValues = transformFunction.transformToFloatValuesMV(_projectionBlock); + int[][] intValuesMV = transformFunction.transformToIntValuesMV(_projectionBlock); + long[][] longValuesMV = transformFunction.transformToLongValuesMV(_projectionBlock); + float[][] floatValuesMV = transformFunction.transformToFloatValuesMV(_projectionBlock); + double[][] doubleValuesMV = transformFunction.transformToDoubleValuesMV(_projectionBlock); + String[][] stringValuesMV = transformFunction.transformToStringValuesMV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(floatMVValues[i], expectedValues[i]); + float[] expectedValueMV = expectedValues[i]; + int numValues = expectedValueMV.length; + assertEquals(intValuesMV[i].length, numValues); + assertEquals(longValuesMV[i].length, numValues); + assertEquals(floatValuesMV[i].length, numValues); + assertEquals(doubleValuesMV[i].length, numValues); + assertEquals(stringValuesMV[i].length, numValues); + for (int j = 0; j < numValues; j++) { + assertEquals(intValuesMV[i][j], (int) expectedValues[i][j]); + assertEquals(longValuesMV[i][j], (long) expectedValues[i][j]); + assertEquals(floatValuesMV[i][j], expectedValues[i][j]); + assertEquals(doubleValuesMV[i][j], (double) expectedValues[i][j]); + assertEquals(stringValuesMV[i][j], Float.toString(expectedValues[i][j])); + } } } protected void testTransformFunctionMV(TransformFunction transformFunction, double[][] expectedValues) { - double[][] doubleMVValues = transformFunction.transformToDoubleValuesMV(_projectionBlock); + int[][] intValuesMV = transformFunction.transformToIntValuesMV(_projectionBlock); + long[][] longValuesMV = transformFunction.transformToLongValuesMV(_projectionBlock); + float[][] floatValuesMV = transformFunction.transformToFloatValuesMV(_projectionBlock); + double[][] doubleValuesMV = transformFunction.transformToDoubleValuesMV(_projectionBlock); + String[][] stringValuesMV = transformFunction.transformToStringValuesMV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(doubleMVValues[i], expectedValues[i]); + double[] expectedValueMV = expectedValues[i]; + int numValues = expectedValueMV.length; + assertEquals(intValuesMV[i].length, numValues); + assertEquals(longValuesMV[i].length, numValues); + assertEquals(floatValuesMV[i].length, numValues); + assertEquals(doubleValuesMV[i].length, numValues); + assertEquals(stringValuesMV[i].length, numValues); + for (int j = 0; j < numValues; j++) { + assertEquals(intValuesMV[i][j], (int) expectedValues[i][j]); + assertEquals(longValuesMV[i][j], (long) expectedValues[i][j]); + assertEquals(floatValuesMV[i][j], (float) expectedValues[i][j]); + assertEquals(doubleValuesMV[i][j], expectedValues[i][j]); + assertEquals(stringValuesMV[i][j], Double.toString(expectedValues[i][j])); + } } } protected void testTransformFunctionMV(TransformFunction transformFunction, String[][] expectedValues) { - String[][] stringMVValues = transformFunction.transformToStringValuesMV(_projectionBlock); + String[][] stringValuesMV = transformFunction.transformToStringValuesMV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { - Assert.assertEquals(stringMVValues[i], expectedValues[i]); + assertEquals(stringValuesMV[i], expectedValues[i]); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BinaryOperatorTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BinaryOperatorTransformFunctionTest.java index 8cbb8dadeb66..774f24721a8a 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BinaryOperatorTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BinaryOperatorTransformFunctionTest.java @@ -18,14 +18,18 @@ */ package org.apache.pinot.core.operator.transform.function; -import java.math.BigDecimal; import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.RequestContextUtils; +import org.apache.pinot.core.operator.transform.TransformResultMetadata; +import org.apache.pinot.spi.data.FieldSpec.DataType; import org.apache.pinot.spi.exception.BadQueryRequestException; -import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + /** * BinaryOperatorTransformFunctionTest abstracts common test methods for EqualsTransformFunctionTest, @@ -35,77 +39,67 @@ */ public abstract class BinaryOperatorTransformFunctionTest extends BaseTransformFunctionTest { - abstract int getExpectedValue(int value, int toCompare); - - abstract int getExpectedValue(long value, long toCompare); - - abstract int getExpectedValue(float value, float toCompare); - - abstract int getExpectedValue(double value, double toCompare); - - abstract int getExpectedValue(BigDecimal value, BigDecimal toCompare); + abstract boolean getExpectedValue(int compareResult); - abstract int getExpectedValue(String value, String toCompare); - - abstract String getFuncName(); + abstract String getFunctionName(); @Test public void testBinaryOperatorTransformFunction() { - ExpressionContext expression = - RequestContextUtils.getExpression(String.format("%s(%s, %d)", getFuncName(), INT_SV_COLUMN, _intSVValues[0])); + String functionName = getFunctionName(); + ExpressionContext expression = RequestContextUtils.getExpression( + String.format("%s(%s, %d)", functionName, INT_SV_COLUMN, _intSVValues[0])); TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertEquals(transformFunction.getName(), getFuncName().toLowerCase()); - int[] expectedIntValues = new int[NUM_ROWS]; + assertEquals(transformFunction.getName(), functionName); + TransformResultMetadata resultMetadata = transformFunction.getResultMetadata(); + assertEquals(resultMetadata.getDataType(), DataType.BOOLEAN); + assertTrue(resultMetadata.isSingleValue()); + assertFalse(resultMetadata.hasDictionary()); + boolean[] expectedValues = new boolean[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - expectedIntValues[i] = getExpectedValue(_intSVValues[i], _intSVValues[0]); + expectedValues[i] = getExpectedValue(Integer.compare(_intSVValues[i], _intSVValues[0])); } - testTransformFunction(transformFunction, expectedIntValues); + testTransformFunction(transformFunction, expectedValues); - expression = - RequestContextUtils.getExpression(String.format("%s(%s, %d)", getFuncName(), LONG_SV_COLUMN, _longSVValues[0])); + expression = RequestContextUtils.getExpression( + String.format("%s(%s, %d)", functionName, LONG_SV_COLUMN, _longSVValues[0])); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - int[] expectedLongValues = new int[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - expectedLongValues[i] = getExpectedValue(_longSVValues[i], _longSVValues[0]); + expectedValues[i] = getExpectedValue(Long.compare(_longSVValues[i], _longSVValues[0])); } - testTransformFunction(transformFunction, expectedLongValues); + testTransformFunction(transformFunction, expectedValues); expression = RequestContextUtils.getExpression( - String.format("%s(%s, %f)", getFuncName(), FLOAT_SV_COLUMN, _floatSVValues[0])); + String.format("%s(%s, %f)", functionName, FLOAT_SV_COLUMN, _floatSVValues[0])); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - int[] expectedFloatValues = new int[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - expectedFloatValues[i] = getExpectedValue(_floatSVValues[i], _floatSVValues[0]); + expectedValues[i] = getExpectedValue(Float.compare(_floatSVValues[i], _floatSVValues[0])); } - testTransformFunction(transformFunction, expectedFloatValues); + testTransformFunction(transformFunction, expectedValues); expression = RequestContextUtils.getExpression( - String.format("%s(%s, %.20f)", getFuncName(), DOUBLE_SV_COLUMN, _doubleSVValues[0])); + String.format("%s(%s, %.20f)", functionName, DOUBLE_SV_COLUMN, _doubleSVValues[0])); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - int[] expectedDoubleValues = new int[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - expectedDoubleValues[i] = getExpectedValue(_doubleSVValues[i], _doubleSVValues[0]); + expectedValues[i] = getExpectedValue(Double.compare(_doubleSVValues[i], _doubleSVValues[0])); } - testTransformFunction(transformFunction, expectedDoubleValues); + testTransformFunction(transformFunction, expectedValues); + // Note: defining decimal literals within quotes ('%s') preserves precision. expression = RequestContextUtils.getExpression( - String.format("%s(%s, '%s')", getFuncName(), STRING_SV_COLUMN, _stringSVValues[0])); + String.format("%s(%s, '%s')", functionName, BIG_DECIMAL_SV_COLUMN, _bigDecimalSVValues[0])); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - int[] expectedStringValues = new int[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - expectedStringValues[i] = getExpectedValue(_stringSVValues[i], _stringSVValues[0]); + expectedValues[i] = getExpectedValue(_bigDecimalSVValues[i].compareTo(_bigDecimalSVValues[0])); } - testTransformFunction(transformFunction, expectedStringValues); + testTransformFunction(transformFunction, expectedValues); - // Note: defining decimal literals within quotes ('%s') preserves precision. expression = RequestContextUtils.getExpression( - String.format("%s(%s, '%s')", getFuncName(), BIG_DECIMAL_SV_COLUMN, _bigDecimalSVValues[0])); + String.format("%s(%s, '%s')", functionName, STRING_SV_COLUMN, _stringSVValues[0])); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - int[] expectedBigDecimalValues = new int[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - expectedBigDecimalValues[i] = getExpectedValue(_bigDecimalSVValues[i], _bigDecimalSVValues[0]); + expectedValues[i] = getExpectedValue(_stringSVValues[i].compareTo(_stringSVValues[0])); } - testTransformFunction(transformFunction, expectedBigDecimalValues); + testTransformFunction(transformFunction, expectedValues); } @Test(dataProvider = "testIllegalArguments", expectedExceptions = {BadQueryRequestException.class}) @@ -118,9 +112,9 @@ public void testIllegalArguments(String expressionStr) { public Object[][] testIllegalArguments() { return new Object[][]{ new Object[]{ - String.format("%s(%s)", getFuncName(), INT_SV_COLUMN) + String.format("%s(%s)", getFunctionName(), INT_SV_COLUMN) }, new Object[]{ - String.format("%s(%s, %s, %s)", getFuncName(), LONG_SV_COLUMN, INT_SV_COLUMN, STRING_SV_COLUMN) + String.format("%s(%s, %s, %s)", getFunctionName(), LONG_SV_COLUMN, INT_SV_COLUMN, STRING_SV_COLUMN) } }; } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/CastTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/CastTransformFunctionTest.java index 5bc734156bf3..f24c3bcaf526 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/CastTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/CastTransformFunctionTest.java @@ -19,104 +19,40 @@ package org.apache.pinot.core.operator.transform.function; import java.math.BigDecimal; +import java.sql.Timestamp; import java.util.Arrays; import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.RequestContextUtils; import org.apache.pinot.spi.data.FieldSpec; import org.apache.pinot.spi.utils.ArrayCopyUtils; -import org.testng.Assert; import org.testng.annotations.Test; import static org.apache.pinot.common.function.scalar.DataTypeConversionFunctions.cast; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; public class CastTransformFunctionTest extends BaseTransformFunctionTest { - @Test - public void testCastTransformFunctionMV() { - ExpressionContext expression = - RequestContextUtils.getExpression(String.format("CAST(%s AS LONG)", STRING_LONG_MV_COLUMN)); - TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); - assertEquals(transformFunction.getName(), CastTransformFunction.FUNCTION_NAME); - long[][] expectedLongValues = new long[NUM_ROWS][]; - ArrayCopyUtils.copy(_stringLongFormatMVValues, expectedLongValues, NUM_ROWS); - testCastTransformFunctionMV(transformFunction, expectedLongValues); - - expression = RequestContextUtils.getExpression( - String.format("CAST(CAST(CAST(%s AS LONG) as DOUBLE) as INT)", STRING_LONG_MV_COLUMN)); - transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); - assertEquals(transformFunction.getName(), CastTransformFunction.FUNCTION_NAME); - long[][] innerLongValues = new long[NUM_ROWS][]; - ArrayCopyUtils.copy(_stringLongFormatMVValues, innerLongValues, NUM_ROWS); - double[][] innerDoubleValues = new double[NUM_ROWS][]; - ArrayCopyUtils.copy(innerLongValues, innerDoubleValues, NUM_ROWS); - int[][] expectedIntValues = new int[NUM_ROWS][]; - ArrayCopyUtils.copy(innerDoubleValues, expectedIntValues, NUM_ROWS); - testCastTransformFunctionMV(transformFunction, expectedIntValues); - - expression = - RequestContextUtils.getExpression(String.format("CAST(CAST(%s AS INT) as FLOAT)", FLOAT_MV_COLUMN)); - transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); - assertEquals(transformFunction.getName(), CastTransformFunction.FUNCTION_NAME); - int[][] innerLayerInt = new int[NUM_ROWS][]; - ArrayCopyUtils.copy(_floatMVValues, innerLayerInt, NUM_ROWS); - float[][] expectedFloatValues = new float[NUM_ROWS][]; - ArrayCopyUtils.copy(innerLayerInt, expectedFloatValues, NUM_ROWS); - testCastTransformFunctionMV(transformFunction, expectedFloatValues); - - expression = RequestContextUtils.getExpression( - String.format("CAST(CAST(CAST(%s AS FLOAT) as INT) as STRING)", INT_MV_COLUMN)); - transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); - assertEquals(transformFunction.getName(), CastTransformFunction.FUNCTION_NAME); - float[][] innerFloatValues = new float[NUM_ROWS][]; - ArrayCopyUtils.copy(_intMVValues, innerFloatValues, NUM_ROWS); - innerLayerInt = new int[NUM_ROWS][]; - ArrayCopyUtils.copy(innerFloatValues, innerLayerInt, NUM_ROWS); - String[][] expectedStringValues = new String[NUM_ROWS][]; - ArrayCopyUtils.copy(innerLayerInt, expectedStringValues, NUM_ROWS); - testCastTransformFunctionMV(transformFunction, expectedStringValues); - - expression = RequestContextUtils.getExpression(String.format("arrayMax(cAst(%s AS INT))", DOUBLE_MV_COLUMN)); - transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - FieldSpec.DataType resultDataType = transformFunction.getResultMetadata().getDataType(); - Assert.assertEquals(resultDataType, FieldSpec.DataType.INT); - - // checks that arraySum triggers transformToDoubleMV in cast function which correctly cast to INT - expression = RequestContextUtils.getExpression(String.format("arraySum(cAst(%s AS INT))", DOUBLE_MV_COLUMN)); - transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - int[][] afterCast = new int[NUM_ROWS][]; - ArrayCopyUtils.copy(_doubleMVValues, afterCast, NUM_ROWS); - double[] expectedArraySums = new double[NUM_ROWS]; - for (int i = 0; i < NUM_ROWS; i++) { - expectedArraySums[i] = Arrays.stream(afterCast[i]).sum(); - } - testTransformFunction(transformFunction, expectedArraySums); - } - @Test public void testCastTransformFunction() { ExpressionContext expression = RequestContextUtils.getExpression(String.format("CAST(%s AS string)", INT_SV_COLUMN)); TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); + assertTrue(transformFunction instanceof CastTransformFunction); assertEquals(transformFunction.getName(), CastTransformFunction.FUNCTION_NAME); - String[] expectedValues = new String[NUM_ROWS]; + String[] expectedStringValues = new String[NUM_ROWS]; String[] scalarStringValues = new String[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - expectedValues[i] = Integer.toString(_intSVValues[i]); + expectedStringValues[i] = Integer.toString(_intSVValues[i]); scalarStringValues[i] = (String) cast(_intSVValues[i], "string"); } - testTransformFunction(transformFunction, expectedValues); - assertEquals(expectedValues, scalarStringValues); + testTransformFunction(transformFunction, expectedStringValues); + assertEquals(expectedStringValues, scalarStringValues); expression = RequestContextUtils.getExpression(String.format("CAST(CAST(%s as INT) as FLOAT)", FLOAT_SV_COLUMN)); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); + assertTrue(transformFunction instanceof CastTransformFunction); float[] expectedFloatValues = new float[NUM_ROWS]; float[] scalarFloatValues = new float[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { @@ -126,22 +62,55 @@ public void testCastTransformFunction() { testTransformFunction(transformFunction, expectedFloatValues); assertEquals(expectedFloatValues, scalarFloatValues); + expression = RequestContextUtils.getExpression(String.format("CAST(CAST(%s as BOOLEAN) as STRING)", INT_SV_COLUMN)); + transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + assertTrue(transformFunction instanceof CastTransformFunction); + for (int i = 0; i < NUM_ROWS; i++) { + expectedStringValues[i] = Boolean.toString(_intSVValues[i] != 0); + scalarStringValues[i] = (String) cast(cast(_intSVValues[i], "boolean"), "string"); + } + testTransformFunction(transformFunction, expectedStringValues); + assertEquals(expectedStringValues, scalarStringValues); + + expression = + RequestContextUtils.getExpression(String.format("CAST(CAST(%s as TIMESTAMP) as STRING)", LONG_SV_COLUMN)); + transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + assertTrue(transformFunction instanceof CastTransformFunction); + for (int i = 0; i < NUM_ROWS; i++) { + expectedStringValues[i] = new Timestamp(_longSVValues[i]).toString(); + scalarStringValues[i] = (String) cast(cast(_longSVValues[i], "timestamp"), "string"); + } + testTransformFunction(transformFunction, expectedStringValues); + assertEquals(expectedStringValues, scalarStringValues); + + expression = RequestContextUtils.getExpression(String.format("CAST(CAST(%s as BOOLEAN) as INT)", INT_SV_COLUMN)); + transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + assertTrue(transformFunction instanceof CastTransformFunction); + int[] expectedIntValues = new int[NUM_ROWS]; + int[] scalarIntValues = new int[NUM_ROWS]; + for (int i = 0; i < NUM_ROWS; i++) { + expectedIntValues[i] = _intSVValues[i] != 0 ? 1 : 0; + scalarIntValues[i] = (int) cast(cast(_intSVValues[i], "boolean"), "int"); + } + testTransformFunction(transformFunction, expectedIntValues); + assertEquals(expectedIntValues, scalarIntValues); + expression = RequestContextUtils.getExpression( String.format("CAST(ADD(CAST(%s AS LONG), %s) AS STRING)", DOUBLE_SV_COLUMN, LONG_SV_COLUMN)); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); + assertTrue(transformFunction instanceof CastTransformFunction); for (int i = 0; i < NUM_ROWS; i++) { - expectedValues[i] = Double.toString((double) (long) _doubleSVValues[i] + (double) _longSVValues[i]); + expectedStringValues[i] = Double.toString((double) (long) _doubleSVValues[i] + (double) _longSVValues[i]); scalarStringValues[i] = (String) cast((double) (long) cast(_doubleSVValues[i], "long") + (double) _longSVValues[i], "string"); } - testTransformFunction(transformFunction, expectedValues); - assertEquals(expectedValues, scalarStringValues); + testTransformFunction(transformFunction, expectedStringValues); + assertEquals(expectedStringValues, scalarStringValues); expression = RequestContextUtils.getExpression( String.format("caSt(cAst(casT(%s as inT) + %s aS sTring) As DouBle)", FLOAT_SV_COLUMN, INT_SV_COLUMN)); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); + assertTrue(transformFunction instanceof CastTransformFunction); double[] expectedDoubleValues = new double[NUM_ROWS]; double[] scalarDoubleValues = new double[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { @@ -157,7 +126,7 @@ public void testCastTransformFunction() { String.format("CAST(CAST(%s AS INT) - CAST(%s AS FLOAT) / CAST(%s AS DOUBLE) AS LONG)", DOUBLE_SV_COLUMN, LONG_SV_COLUMN, INT_SV_COLUMN)); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); + assertTrue(transformFunction instanceof CastTransformFunction); long[] expectedLongValues = new long[NUM_ROWS]; long[] longScalarValues = new long[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { @@ -171,7 +140,7 @@ public void testCastTransformFunction() { expression = RequestContextUtils.getExpression(String.format("CAST(%s AS BIG_DECIMAL)", LONG_SV_COLUMN)); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); + assertTrue(transformFunction instanceof CastTransformFunction); BigDecimal[] expectedBigDecimalValues = new BigDecimal[NUM_ROWS]; BigDecimal[] bigDecimalScalarValues = new BigDecimal[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { @@ -185,7 +154,7 @@ public void testCastTransformFunction() { String.format("CAST(CAST(%s AS DOUBLE) - CAST(%s AS DOUBLE) / CAST(%s AS DOUBLE) AS BIG_DECIMAL)", BIG_DECIMAL_SV_COLUMN, LONG_SV_COLUMN, INT_SV_COLUMN)); transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof CastTransformFunction); + assertTrue(transformFunction instanceof CastTransformFunction); expectedBigDecimalValues = new BigDecimal[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { expectedBigDecimalValues[i] = BigDecimal.valueOf( @@ -199,93 +168,91 @@ public void testCastTransformFunction() { assertEquals(expectedBigDecimalValues, bigDecimalScalarValues); } - private void testCastTransformFunctionMV(TransformFunction transformFunction, int[][] expectedValues) { - int[][] intMVValues = transformFunction.transformToIntValuesMV(_projectionBlock); - long[][] longMVValues = transformFunction.transformToLongValuesMV(_projectionBlock); - float[][] floatMVValues = transformFunction.transformToFloatValuesMV(_projectionBlock); - double[][] doubleMVValues = transformFunction.transformToDoubleValuesMV(_projectionBlock); - String[][] stringMVValues = transformFunction.transformToStringValuesMV(_projectionBlock); - for (int i = 0; i < NUM_ROWS; i++) { - int rowLen = expectedValues[i].length; - for (int j = 0; j < rowLen; j++) { - Assert.assertEquals(intMVValues[i][j], expectedValues[i][j]); - Assert.assertEquals(longMVValues[i][j], (long) expectedValues[i][j]); - Assert.assertEquals(floatMVValues[i][j], (float) expectedValues[i][j]); - Assert.assertEquals(doubleMVValues[i][j], (double) expectedValues[i][j]); - Assert.assertEquals(stringMVValues[i][j], Integer.toString(expectedValues[i][j])); - } - } - } + @Test + public void testCastTransformFunctionMV() { + ExpressionContext expression = + RequestContextUtils.getExpression(String.format("CAST(%s AS LONG)", STRING_LONG_MV_COLUMN)); + TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + assertTrue(transformFunction instanceof CastTransformFunction); + assertEquals(transformFunction.getName(), CastTransformFunction.FUNCTION_NAME); + long[][] expectedLongValues = new long[NUM_ROWS][]; + ArrayCopyUtils.copy(_stringLongFormatMVValues, expectedLongValues, NUM_ROWS); + testTransformFunctionMV(transformFunction, expectedLongValues); - private void testCastTransformFunctionMV(TransformFunction transformFunction, long[][] expectedValues) { - int[][] intMVValues = transformFunction.transformToIntValuesMV(_projectionBlock); - long[][] longMVValues = transformFunction.transformToLongValuesMV(_projectionBlock); - float[][] floatMVValues = transformFunction.transformToFloatValuesMV(_projectionBlock); - double[][] doubleMVValues = transformFunction.transformToDoubleValuesMV(_projectionBlock); - String[][] stringMVValues = transformFunction.transformToStringValuesMV(_projectionBlock); - for (int i = 0; i < NUM_ROWS; i++) { - int rowLen = expectedValues[i].length; - for (int j = 0; j < rowLen; j++) { - Assert.assertEquals(intMVValues[i][j], (int) expectedValues[i][j]); - Assert.assertEquals(longMVValues[i][j], expectedValues[i][j]); - Assert.assertEquals(floatMVValues[i][j], (float) expectedValues[i][j]); - Assert.assertEquals(doubleMVValues[i][j], (double) expectedValues[i][j]); - Assert.assertEquals(stringMVValues[i][j], Long.toString(expectedValues[i][j])); - } - } - } + expression = RequestContextUtils.getExpression( + String.format("CAST(CAST(CAST(%s AS LONG) as DOUBLE) as INT)", STRING_LONG_MV_COLUMN)); + transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + assertTrue(transformFunction instanceof CastTransformFunction); + assertEquals(transformFunction.getName(), CastTransformFunction.FUNCTION_NAME); + long[][] innerLongValues = new long[NUM_ROWS][]; + ArrayCopyUtils.copy(_stringLongFormatMVValues, innerLongValues, NUM_ROWS); + double[][] innerDoubleValues = new double[NUM_ROWS][]; + ArrayCopyUtils.copy(innerLongValues, innerDoubleValues, NUM_ROWS); + int[][] expectedIntValues = new int[NUM_ROWS][]; + ArrayCopyUtils.copy(innerDoubleValues, expectedIntValues, NUM_ROWS); + testTransformFunctionMV(transformFunction, expectedIntValues); - private void testCastTransformFunctionMV(TransformFunction transformFunction, float[][] expectedValues) { - int[][] intMVValues = transformFunction.transformToIntValuesMV(_projectionBlock); - long[][] longMVValues = transformFunction.transformToLongValuesMV(_projectionBlock); - float[][] floatMVValues = transformFunction.transformToFloatValuesMV(_projectionBlock); - double[][] doubleMVValues = transformFunction.transformToDoubleValuesMV(_projectionBlock); - String[][] stringMVValues = transformFunction.transformToStringValuesMV(_projectionBlock); + expression = RequestContextUtils.getExpression(String.format("CAST(CAST(%s AS INT) as FLOAT)", FLOAT_MV_COLUMN)); + transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + assertTrue(transformFunction instanceof CastTransformFunction); + assertEquals(transformFunction.getName(), CastTransformFunction.FUNCTION_NAME); + int[][] innerLayerInt = new int[NUM_ROWS][]; + ArrayCopyUtils.copy(_floatMVValues, innerLayerInt, NUM_ROWS); + float[][] expectedFloatValues = new float[NUM_ROWS][]; + ArrayCopyUtils.copy(innerLayerInt, expectedFloatValues, NUM_ROWS); + testTransformFunctionMV(transformFunction, expectedFloatValues); + + expression = RequestContextUtils.getExpression( + String.format("CAST(CAST(CAST(%s AS FLOAT) as INT) as STRING)", INT_MV_COLUMN)); + transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + assertTrue(transformFunction instanceof CastTransformFunction); + assertEquals(transformFunction.getName(), CastTransformFunction.FUNCTION_NAME); + float[][] innerFloatValues = new float[NUM_ROWS][]; + ArrayCopyUtils.copy(_intMVValues, innerFloatValues, NUM_ROWS); + innerLayerInt = new int[NUM_ROWS][]; + ArrayCopyUtils.copy(innerFloatValues, innerLayerInt, NUM_ROWS); + String[][] expectedStringValues = new String[NUM_ROWS][]; + ArrayCopyUtils.copy(innerLayerInt, expectedStringValues, NUM_ROWS); + testTransformFunctionMV(transformFunction, expectedStringValues); + + expression = RequestContextUtils.getExpression(String.format("CAST(CAST(%s as BOOLEAN) as STRING)", INT_MV_COLUMN)); + transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + assertTrue(transformFunction instanceof CastTransformFunction); for (int i = 0; i < NUM_ROWS; i++) { - int rowLen = expectedValues[i].length; - for (int j = 0; j < rowLen; j++) { - Assert.assertEquals(intMVValues[i][j], (int) expectedValues[i][j]); - Assert.assertEquals(longMVValues[i][j], (long) expectedValues[i][j]); - Assert.assertEquals(floatMVValues[i][j], expectedValues[i][j]); - Assert.assertEquals(doubleMVValues[i][j], (double) expectedValues[i][j]); - Assert.assertEquals(stringMVValues[i][j], Float.toString(expectedValues[i][j])); + int length = _intMVValues[i].length; + expectedStringValues[i] = new String[length]; + for (int j = 0; j < length; j++) { + expectedStringValues[i][j] = Boolean.toString(_intMVValues[i][j] != 0); } } - } + testTransformFunctionMV(transformFunction, expectedStringValues); - private void testCastTransformFunctionMV(TransformFunction transformFunction, double[][] expectedValues) { - int[][] intMVValues = transformFunction.transformToIntValuesMV(_projectionBlock); - long[][] longMVValues = transformFunction.transformToLongValuesMV(_projectionBlock); - float[][] floatMVValues = transformFunction.transformToFloatValuesMV(_projectionBlock); - double[][] doubleMVValues = transformFunction.transformToDoubleValuesMV(_projectionBlock); - String[][] stringMVValues = transformFunction.transformToStringValuesMV(_projectionBlock); + expression = + RequestContextUtils.getExpression(String.format("CAST(CAST(%s as TIMESTAMP) as STRING)", LONG_SV_COLUMN)); + transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + assertTrue(transformFunction instanceof CastTransformFunction); for (int i = 0; i < NUM_ROWS; i++) { - int rowLen = expectedValues[i].length; - for (int j = 0; j < rowLen; j++) { - Assert.assertEquals(intMVValues[i][j], (int) expectedValues[i][j]); - Assert.assertEquals(longMVValues[i][j], (long) expectedValues[i][j]); - Assert.assertEquals(floatMVValues[i][j], (float) expectedValues[i][j]); - Assert.assertEquals(doubleMVValues[i][j], expectedValues[i][j]); - Assert.assertEquals(stringMVValues[i][j], Double.toString(expectedValues[i][j])); + int length = _longMVValues[i].length; + expectedStringValues[i] = new String[length]; + for (int j = 0; j < length; j++) { + expectedStringValues[i][j] = new Timestamp(_longMVValues[i][j]).toString(); } } - } - private void testCastTransformFunctionMV(TransformFunction transformFunction, String[][] expectedValues) { - int[][] intMVValues = transformFunction.transformToIntValuesMV(_projectionBlock); - long[][] longMVValues = transformFunction.transformToLongValuesMV(_projectionBlock); - float[][] floatMVValues = transformFunction.transformToFloatValuesMV(_projectionBlock); - double[][] doubleMVValues = transformFunction.transformToDoubleValuesMV(_projectionBlock); - String[][] stringMVValues = transformFunction.transformToStringValuesMV(_projectionBlock); + expression = RequestContextUtils.getExpression(String.format("arrayMax(cAst(%s AS INT))", DOUBLE_MV_COLUMN)); + transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + FieldSpec.DataType resultDataType = transformFunction.getResultMetadata().getDataType(); + assertEquals(resultDataType, FieldSpec.DataType.INT); + + // checks that arraySum triggers transformToDoubleMV in cast function which correctly cast to INT + expression = RequestContextUtils.getExpression(String.format("arraySum(cAst(%s AS INT))", DOUBLE_MV_COLUMN)); + transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); + int[][] afterCast = new int[NUM_ROWS][]; + ArrayCopyUtils.copy(_doubleMVValues, afterCast, NUM_ROWS); + double[] expectedArraySums = new double[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - int rowLen = expectedValues[i].length; - for (int j = 0; j < rowLen; j++) { - Assert.assertEquals(intMVValues[i][j], Integer.parseInt(expectedValues[i][j])); - Assert.assertEquals(longMVValues[i][j], Long.parseLong(expectedValues[i][j])); - Assert.assertEquals(floatMVValues[i][j], Float.parseFloat(expectedValues[i][j])); - Assert.assertEquals(doubleMVValues[i][j], Double.parseDouble(expectedValues[i][j])); - Assert.assertEquals(stringMVValues[i][j], expectedValues[i][j]); - } + expectedArraySums[i] = Arrays.stream(afterCast[i]).sum(); } + testTransformFunction(transformFunction, expectedArraySums); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java index 6c7ee6b3c923..2e1375f5e5e0 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java @@ -21,11 +21,15 @@ import java.util.concurrent.TimeUnit; import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.RequestContextUtils; +import org.apache.pinot.core.operator.transform.TransformResultMetadata; +import org.apache.pinot.spi.data.FieldSpec.DataType; import org.apache.pinot.spi.exception.BadQueryRequestException; -import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + public class DateTimeConversionTransformFunctionTest extends BaseTransformFunctionTest { @@ -35,21 +39,16 @@ public void testDateTimeConversionTransformFunction() { ExpressionContext expression = RequestContextUtils.getExpression( String.format("dateTimeConvert(%s,'1:MILLISECONDS:EPOCH','1:MINUTES:EPOCH','1:MINUTES')", TIME_COLUMN)); TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof DateTimeConversionTransformFunction); - Assert.assertEquals(transformFunction.getName(), DateTimeConversionTransformFunction.FUNCTION_NAME); - int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock); - long[] longValues = transformFunction.transformToLongValuesSV(_projectionBlock); - float[] floatValues = transformFunction.transformToFloatValuesSV(_projectionBlock); - double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock); - String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); + assertTrue(transformFunction instanceof DateTimeConversionTransformFunction); + assertEquals(transformFunction.getName(), DateTimeConversionTransformFunction.FUNCTION_NAME); + TransformResultMetadata resultMetadata = transformFunction.getResultMetadata(); + assertTrue(resultMetadata.isSingleValue()); + assertEquals(resultMetadata.getDataType(), DataType.LONG); + long[] expectedValues = new long[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - long expected = TimeUnit.MILLISECONDS.toMinutes(_timeValues[i]); - Assert.assertEquals(intValues[i], (int) expected); - Assert.assertEquals(longValues[i], expected); - Assert.assertEquals(floatValues[i], (float) expected); - Assert.assertEquals(doubleValues[i], (double) expected); - Assert.assertEquals(stringValues[i], Long.toString(expected)); + expectedValues[i] = TimeUnit.MILLISECONDS.toMinutes(_timeValues[i]); } + testTransformFunction(transformFunction, expectedValues); } @Test(dataProvider = "testIllegalArguments", expectedExceptions = {BadQueryRequestException.class}) diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DistinctFromTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DistinctFromTransformFunctionTest.java index 71dc9d213217..d9badf9e9986 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DistinctFromTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DistinctFromTransformFunctionTest.java @@ -155,14 +155,15 @@ protected void testTransformFunction(ExpressionContext expression, boolean[] exp getTransformFunctionInstance(expression, dataSourceMap).transformToFloatValuesSV(projectionBlock); double[] doubleValues = getTransformFunctionInstance(expression, dataSourceMap).transformToDoubleValuesSV(projectionBlock); - String[] stringValues = - getTransformFunctionInstance(expression, dataSourceMap).transformToStringValuesSV(projectionBlock); + // TODO: Support implicit cast from BOOLEAN to STRING +// String[] stringValues = +// getTransformFunctionInstance(expression, dataSourceMap).transformToStringValuesSV(projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { Assert.assertEquals(intValues[i] == 1, expectedValues[i]); Assert.assertEquals(longValues[i] == 1, expectedValues[i]); Assert.assertEquals(floatValues[i] == 1, expectedValues[i]); Assert.assertEquals(doubleValues[i] == 1, expectedValues[i]); - Assert.assertEquals(stringValues[i], Boolean.toString(expectedValues[i])); +// Assert.assertEquals(stringValues[i], Boolean.toString(expectedValues[i])); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/EqualsTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/EqualsTransformFunctionTest.java index dd473e2c7c85..65e9596fb27e 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/EqualsTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/EqualsTransformFunctionTest.java @@ -18,43 +18,15 @@ */ package org.apache.pinot.core.operator.transform.function; -import java.math.BigDecimal; - - public class EqualsTransformFunctionTest extends BinaryOperatorTransformFunctionTest { @Override - int getExpectedValue(int value, int toCompare) { - return (value == toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(long value, long toCompare) { - return (value == toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(float value, float toCompare) { - return (value == toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(double value, double toCompare) { - return (value == toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(BigDecimal value, BigDecimal toCompare) { - return value.compareTo(toCompare) == 0 ? 1 : 0; - } - - @Override - int getExpectedValue(String value, String toCompare) { - return (value.compareTo(toCompare) == 0) ? 1 : 0; + boolean getExpectedValue(int compareResult) { + return compareResult == 0; } @Override - String getFuncName() { + String getFunctionName() { return new EqualsTransformFunction().getName(); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/GreaterThanOrEqualTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/GreaterThanOrEqualTransformFunctionTest.java index 9e9b1a3a1dcf..83e60cdd7cf4 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/GreaterThanOrEqualTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/GreaterThanOrEqualTransformFunctionTest.java @@ -18,43 +18,15 @@ */ package org.apache.pinot.core.operator.transform.function; -import java.math.BigDecimal; - - public class GreaterThanOrEqualTransformFunctionTest extends BinaryOperatorTransformFunctionTest { @Override - int getExpectedValue(int value, int toCompare) { - return (value >= toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(long value, long toCompare) { - return (value >= toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(float value, float toCompare) { - return (value >= toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(double value, double toCompare) { - return (value >= toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(BigDecimal value, BigDecimal toCompare) { - return value.compareTo(toCompare) >= 0 ? 1 : 0; - } - - @Override - int getExpectedValue(String value, String toCompare) { - return (value.compareTo(toCompare) >= 0) ? 1 : 0; + boolean getExpectedValue(int compareResult) { + return compareResult >= 0; } @Override - String getFuncName() { + String getFunctionName() { return new GreaterThanOrEqualTransformFunction().getName(); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/GreaterThanTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/GreaterThanTransformFunctionTest.java index 40a93dba123e..06e8effacf3e 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/GreaterThanTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/GreaterThanTransformFunctionTest.java @@ -18,43 +18,15 @@ */ package org.apache.pinot.core.operator.transform.function; -import java.math.BigDecimal; - - public class GreaterThanTransformFunctionTest extends BinaryOperatorTransformFunctionTest { @Override - int getExpectedValue(int value, int toCompare) { - return (value > toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(long value, long toCompare) { - return (value > toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(float value, float toCompare) { - return (value > toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(double value, double toCompare) { - return (value > toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(BigDecimal value, BigDecimal toCompare) { - return value.compareTo(toCompare) > 0 ? 1 : 0; - } - - @Override - int getExpectedValue(String value, String toCompare) { - return (value.compareTo(toCompare) > 0) ? 1 : 0; + boolean getExpectedValue(int compareResult) { + return compareResult > 0; } @Override - String getFuncName() { + String getFunctionName() { return new GreaterThanTransformFunction().getName(); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LessThanOrEqualTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LessThanOrEqualTransformFunctionTest.java index adc9090ed637..652ced40b396 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LessThanOrEqualTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LessThanOrEqualTransformFunctionTest.java @@ -18,43 +18,15 @@ */ package org.apache.pinot.core.operator.transform.function; -import java.math.BigDecimal; - - public class LessThanOrEqualTransformFunctionTest extends BinaryOperatorTransformFunctionTest { @Override - int getExpectedValue(int value, int toCompare) { - return (value <= toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(long value, long toCompare) { - return (value <= toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(float value, float toCompare) { - return (value <= toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(double value, double toCompare) { - return (value <= toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(BigDecimal value, BigDecimal toCompare) { - return value.compareTo(toCompare) <= 0 ? 1 : 0; - } - - @Override - int getExpectedValue(String value, String toCompare) { - return (value.compareTo(toCompare) <= 0) ? 1 : 0; + boolean getExpectedValue(int compareResult) { + return compareResult <= 0; } @Override - String getFuncName() { + String getFunctionName() { return new LessThanOrEqualTransformFunction().getName(); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LessThanTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LessThanTransformFunctionTest.java index 67f211432279..556ecdd5b94c 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LessThanTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LessThanTransformFunctionTest.java @@ -18,43 +18,15 @@ */ package org.apache.pinot.core.operator.transform.function; -import java.math.BigDecimal; - - public class LessThanTransformFunctionTest extends BinaryOperatorTransformFunctionTest { @Override - int getExpectedValue(int value, int toCompare) { - return (value < toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(long value, long toCompare) { - return (value < toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(float value, float toCompare) { - return (value < toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(double value, double toCompare) { - return (value < toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(BigDecimal value, BigDecimal toCompare) { - return value.compareTo(toCompare) < 0 ? 1 : 0; - } - - @Override - int getExpectedValue(String value, String toCompare) { - return (value.compareTo(toCompare) < 0) ? 1 : 0; + boolean getExpectedValue(int compareResult) { + return compareResult < 0; } @Override - String getFuncName() { + String getFunctionName() { return new LessThanTransformFunction().getName(); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunctionTest.java index eec46e071d08..8093cad19d96 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunctionTest.java @@ -24,11 +24,16 @@ import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.FunctionContext; import org.apache.pinot.common.request.context.RequestContextUtils; +import org.apache.pinot.core.operator.transform.TransformResultMetadata; +import org.apache.pinot.spi.data.FieldSpec; import org.apache.pinot.spi.exception.BadQueryRequestException; -import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + /** * LogicalOperatorTransformFunctionTest abstracts common test methods for: @@ -38,9 +43,9 @@ */ public abstract class LogicalOperatorTransformFunctionTest extends BaseTransformFunctionTest { - abstract int getExpectedValue(boolean left, boolean right); + abstract boolean getExpectedValue(boolean left, boolean right); - abstract String getFuncName(); + abstract String getFunctionName(); @Test public void testLogicalOperatorTransformFunction() { @@ -48,27 +53,31 @@ public void testLogicalOperatorTransformFunction() { RequestContextUtils.getExpression(String.format("EQUALS(%s, %d)", INT_SV_COLUMN, _intSVValues[0])); ExpressionContext longEqualsExpr = RequestContextUtils.getExpression(String.format("EQUALS(%s, %d)", LONG_SV_COLUMN, _longSVValues[0])); + String functionName = getFunctionName(); ExpressionContext expression = ExpressionContext.forFunction( - new FunctionContext(FunctionContext.Type.TRANSFORM, getFuncName(), + new FunctionContext(FunctionContext.Type.TRANSFORM, functionName, Arrays.asList(intEqualsExpr, longEqualsExpr))); TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertEquals(transformFunction.getName(), getFuncName().toLowerCase()); - int[] expectedIntValues = new int[NUM_ROWS]; + assertEquals(transformFunction.getName(), functionName); + TransformResultMetadata resultMetadata = transformFunction.getResultMetadata(); + assertEquals(resultMetadata.getDataType(), FieldSpec.DataType.BOOLEAN); + assertTrue(resultMetadata.isSingleValue()); + assertFalse(resultMetadata.hasDictionary()); + boolean[] expectedValues = new boolean[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - expectedIntValues[i] = getExpectedValue(_intSVValues[i] == _intSVValues[0], _longSVValues[i] == _longSVValues[0]); + expectedValues[i] = getExpectedValue(_intSVValues[i] == _intSVValues[0], _longSVValues[i] == _longSVValues[0]); } - testTransformFunction(transformFunction, expectedIntValues); + testTransformFunction(transformFunction, expectedValues); } @Test(dataProvider = "testIllegalArguments", expectedExceptions = {BadQueryRequestException.class}) public void testIllegalArguments(String[] expressions) { List expressionContextList = new ArrayList<>(); - for (int i = 0; i < expressions.length; i++) { - expressionContextList.add(RequestContextUtils.getExpression(expressions[i])); + for (String expression : expressions) { + expressionContextList.add(RequestContextUtils.getExpression(expression)); } - TransformFunctionFactory.get(ExpressionContext - .forFunction(new FunctionContext(FunctionContext.Type.TRANSFORM, getFuncName(), expressionContextList)), - _dataSourceMap); + TransformFunctionFactory.get(ExpressionContext.forFunction( + new FunctionContext(FunctionContext.Type.TRANSFORM, getFunctionName(), expressionContextList)), _dataSourceMap); } @DataProvider(name = "testIllegalArguments") diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NotEqualsTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NotEqualsTransformFunctionTest.java index afe63c086bb9..9f3c4fa0dd5a 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NotEqualsTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NotEqualsTransformFunctionTest.java @@ -18,43 +18,15 @@ */ package org.apache.pinot.core.operator.transform.function; -import java.math.BigDecimal; - - public class NotEqualsTransformFunctionTest extends BinaryOperatorTransformFunctionTest { @Override - int getExpectedValue(int value, int toCompare) { - return (value != toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(long value, long toCompare) { - return (value != toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(float value, float toCompare) { - return (value != toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(double value, double toCompare) { - return (value != toCompare) ? 1 : 0; - } - - @Override - int getExpectedValue(BigDecimal value, BigDecimal toCompare) { - return value.compareTo(toCompare) != 0 ? 1 : 0; - } - - @Override - int getExpectedValue(String value, String toCompare) { - return (value.compareTo(toCompare) != 0) ? 1 : 0; + boolean getExpectedValue(int compareResult) { + return compareResult != 0; } @Override - String getFuncName() { + String getFunctionName() { return new NotEqualsTransformFunction().getName(); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NotOperatorTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NotOperatorTransformFunctionTest.java index 1ec3bbe70268..088192200cd9 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NotOperatorTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NotOperatorTransformFunctionTest.java @@ -18,57 +18,60 @@ */ package org.apache.pinot.core.operator.transform.function; +import java.util.Arrays; +import org.apache.pinot.common.function.TransformFunctionType; import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.RequestContextUtils; -import org.testng.Assert; +import org.apache.pinot.core.operator.transform.TransformResultMetadata; +import org.apache.pinot.spi.data.FieldSpec; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + public class NotOperatorTransformFunctionTest extends BaseTransformFunctionTest { + // Test not true returns false. @Test public void testNotTrueOperatorTransformFunction() { - ExpressionContext expr = - RequestContextUtils.getExpression(String.format("Not (%d = %d)", _intSVValues[0], _intSVValues[0])); - TransformFunction func = TransformFunctionFactory.get(expr, _dataSourceMap); - Assert.assertEquals(func.getName(), "not"); - int[] notTrueExpectedIntValues = new int[NUM_ROWS]; - for (int i = 0; i < NUM_ROWS; i++) { - notTrueExpectedIntValues[i] = 0; - } - testTransformFunction(func, notTrueExpectedIntValues); + String expression = String.format("Not (%d = %d)", _intSVValues[0], _intSVValues[0]); + boolean[] expectedValues = new boolean[NUM_ROWS]; + Arrays.fill(expectedValues, false); + testTransformFunction(expression, expectedValues); } // Test not false returns true. @Test public void testNotFalseOperatorTransformFunction() { - ExpressionContext expr = - RequestContextUtils.getExpression(String.format("Not (%d != %d)", _intSVValues[0], _intSVValues[0])); - TransformFunction func = TransformFunctionFactory.get(expr, _dataSourceMap); - Assert.assertEquals(func.getName(), "not"); - int[] notTrueExpectedIntValues = new int[NUM_ROWS]; - for (int i = 0; i < NUM_ROWS; i++) { - notTrueExpectedIntValues[i] = 1; - } - testTransformFunction(func, notTrueExpectedIntValues); + String expression = String.format("Not (%d != %d)", _intSVValues[0], _intSVValues[0]); + boolean[] expectedValues = new boolean[NUM_ROWS]; + Arrays.fill(expectedValues, true); + testTransformFunction(expression, expectedValues); } // Test that not operator also works for not literal @Test public void testNonLiteralSupport() { - ExpressionContext expr = - RequestContextUtils.getExpression(String.format("Not (%s != %d)", INT_SV_COLUMN, _intSVValues[0])); - TransformFunction func = TransformFunctionFactory.get(expr, _dataSourceMap); - Assert.assertEquals(func.getName(), "not"); - int[] expectedIntValues = new int[NUM_ROWS]; + String expression = String.format("Not (%s != %d)", INT_SV_COLUMN, _intSVValues[0]); + boolean[] expectedValues = new boolean[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - if (_intSVValues[i] == _intSVValues[0]) { - expectedIntValues[i] = 1; - } else { - expectedIntValues[i] = 0; - } + expectedValues[i] = _intSVValues[i] == _intSVValues[0]; } - testTransformFunction(func, expectedIntValues); + testTransformFunction(expression, expectedValues); + } + + private void testTransformFunction(String expression, boolean[] expectedValues) { + TransformFunction transformFunction = + TransformFunctionFactory.get(RequestContextUtils.getExpression(expression), _dataSourceMap); + assertEquals(transformFunction.getName(), TransformFunctionType.NOT.getName()); + TransformResultMetadata resultMetadata = transformFunction.getResultMetadata(); + assertEquals(resultMetadata.getDataType(), FieldSpec.DataType.BOOLEAN); + assertTrue(resultMetadata.isSingleValue()); + assertFalse(resultMetadata.hasDictionary()); + testTransformFunction(transformFunction, expectedValues); } // Test illegal arguments for not transform. @@ -76,13 +79,13 @@ public void testNonLiteralSupport() { public void testIllegalNotOperatorTransformFunction() { // Wrong argument type. ExpressionContext exprWrongType = RequestContextUtils.getExpression(String.format("Not(%s)", "test")); - Assert.assertThrows(RuntimeException.class, () -> { + assertThrows(RuntimeException.class, () -> { TransformFunctionFactory.get(exprWrongType, _dataSourceMap); }); // Wrong number of arguments. ExpressionContext exprNumArg = RequestContextUtils.getExpression(String.format("Not(%d, %d)", _intSVValues[0], _intSVValues[1])); - Assert.assertThrows(RuntimeException.class, () -> { + assertThrows(RuntimeException.class, () -> { TransformFunctionFactory.get(exprNumArg, _dataSourceMap); }); } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NullHandlingTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NullHandlingTransformFunctionTest.java index 4130cdd3998a..de8b8e3636ad 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NullHandlingTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NullHandlingTransformFunctionTest.java @@ -19,6 +19,7 @@ package org.apache.pinot.core.operator.transform.function; import java.io.File; +import java.math.BigDecimal; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.ArrayList; @@ -38,6 +39,7 @@ import org.apache.pinot.core.operator.ProjectionOperator; import org.apache.pinot.core.operator.blocks.ProjectionBlock; import org.apache.pinot.core.operator.filter.MatchAllFilterOperator; +import org.apache.pinot.core.operator.transform.TransformResultMetadata; import org.apache.pinot.core.plan.DocIdSetPlanNode; import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader; import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl; @@ -47,7 +49,7 @@ import org.apache.pinot.segment.spi.datasource.DataSource; import org.apache.pinot.spi.config.table.TableConfig; import org.apache.pinot.spi.config.table.TableType; -import org.apache.pinot.spi.data.FieldSpec; +import org.apache.pinot.spi.data.FieldSpec.DataType; import org.apache.pinot.spi.data.Schema; import org.apache.pinot.spi.data.TimeGranularitySpec; import org.apache.pinot.spi.data.readers.GenericRow; @@ -57,6 +59,7 @@ import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -131,14 +134,13 @@ public void setup() rows.add(row); } - Schema schema = new Schema.SchemaBuilder().addSingleValueDimension(INT_SV_COLUMN, FieldSpec.DataType.INT) - .addSingleValueDimension(LONG_SV_COLUMN, FieldSpec.DataType.LONG) - .addSingleValueDimension(FLOAT_SV_COLUMN, FieldSpec.DataType.FLOAT) - .addSingleValueDimension(DOUBLE_SV_COLUMN, FieldSpec.DataType.DOUBLE) - .addSingleValueDimension(STRING_SV_COLUMN, FieldSpec.DataType.STRING) - .addSingleValueDimension(BYTES_SV_COLUMN, FieldSpec.DataType.BYTES) - .addDateTime(TIMESTAMP_COLUMN, FieldSpec.DataType.TIMESTAMP, "1:MILLISECONDS:EPOCH", "1:MILLISECONDS") - .addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, TIME_COLUMN), null).build(); + Schema schema = new Schema.SchemaBuilder().addSingleValueDimension(INT_SV_COLUMN, DataType.INT) + .addSingleValueDimension(LONG_SV_COLUMN, DataType.LONG).addSingleValueDimension(FLOAT_SV_COLUMN, DataType.FLOAT) + .addSingleValueDimension(DOUBLE_SV_COLUMN, DataType.DOUBLE) + .addSingleValueDimension(STRING_SV_COLUMN, DataType.STRING) + .addSingleValueDimension(BYTES_SV_COLUMN, DataType.BYTES) + .addDateTime(TIMESTAMP_COLUMN, DataType.TIMESTAMP, "1:MILLISECONDS:EPOCH", "1:MILLISECONDS") + .addTime(new TimeGranularitySpec(DataType.LONG, TimeUnit.MILLISECONDS, TIME_COLUMN), null).build(); TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("testWithNulls").setNullHandlingEnabled(true) .setTimeColumnName(TIME_COLUMN).build(); @@ -178,6 +180,10 @@ public void testIsNullTransformFunction(String columnName) TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); assertTrue(transformFunction instanceof IsNullTransformFunction); assertEquals(transformFunction.getName(), TransformFunctionType.IS_NULL.getName()); + TransformResultMetadata resultMetadata = transformFunction.getResultMetadata(); + assertEquals(resultMetadata.getDataType(), DataType.BOOLEAN); + assertTrue(resultMetadata.isSingleValue()); + assertFalse(resultMetadata.hasDictionary()); boolean[] expectedValues = new boolean[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { expectedValues[i] = i % NULL_VALUE_MOD == 0; @@ -202,6 +208,10 @@ public void testIsNotNullTransformFunction(String columnName) TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); assertTrue(transformFunction instanceof IsNotNullTransformFunction); assertEquals(transformFunction.getName(), TransformFunctionType.IS_NOT_NULL.getName()); + TransformResultMetadata resultMetadata = transformFunction.getResultMetadata(); + assertEquals(resultMetadata.getDataType(), DataType.BOOLEAN); + assertTrue(resultMetadata.isSingleValue()); + assertFalse(resultMetadata.hasDictionary()); boolean[] expectedValues = new boolean[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { expectedValues[i] = i % NULL_VALUE_MOD != 0; @@ -215,13 +225,17 @@ protected void testTransformFunction(ExpressionContext expression, boolean[] exp long[] longValues = getTransformFunctionInstance(expression).transformToLongValuesSV(_projectionBlock); float[] floatValues = getTransformFunctionInstance(expression).transformToFloatValuesSV(_projectionBlock); double[] doubleValues = getTransformFunctionInstance(expression).transformToDoubleValuesSV(_projectionBlock); - String[] stringValues = getTransformFunctionInstance(expression).transformToStringValuesSV(_projectionBlock); + BigDecimal[] bigDecimalValues = + getTransformFunctionInstance(expression).transformToBigDecimalValuesSV(_projectionBlock); + // TODO: Support implicit cast from BOOLEAN to STRING +// String[] stringValues = getTransformFunctionInstance(expression).transformToStringValuesSV(_projectionBlock); for (int i = 0; i < NUM_ROWS; i++) { assertEquals(intValues[i] == 1, expectedValues[i]); assertEquals(longValues[i] == 1, expectedValues[i]); assertEquals(floatValues[i] == 1, expectedValues[i]); assertEquals(doubleValues[i] == 1, expectedValues[i]); - assertEquals(stringValues[i], Boolean.toString(expectedValues[i])); + assertEquals(bigDecimalValues[i].intValue() == 1, expectedValues[i]); +// assertEquals(stringValues[i], Boolean.toString(expectedValues[i])); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunctionTest.java index ca1c2210bfc8..fbf4271a0481 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunctionTest.java @@ -18,17 +18,18 @@ */ package org.apache.pinot.core.operator.transform.function; +import org.apache.pinot.common.function.TransformFunctionType; + + public class OrOperatorTransformFunctionTest extends LogicalOperatorTransformFunctionTest { + @Override - int getExpectedValue(boolean left, boolean right) { - if (left || right) { - return 1; - } - return 0; + boolean getExpectedValue(boolean left, boolean right) { + return left || right; } @Override - String getFuncName() { - return new OrOperatorTransformFunction().getName(); + String getFunctionName() { + return TransformFunctionType.OR.getName(); } } diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunctionTest.java index 1e79aaac0578..2540f6c9cfcc 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunctionTest.java @@ -21,11 +21,16 @@ import java.util.concurrent.TimeUnit; import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.RequestContextUtils; +import org.apache.pinot.core.operator.transform.TransformResultMetadata; +import org.apache.pinot.spi.data.FieldSpec.DataType; import org.apache.pinot.spi.exception.BadQueryRequestException; -import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + public class TimeConversionTransformFunctionTest extends BaseTransformFunctionTest { @@ -33,21 +38,17 @@ public class TimeConversionTransformFunctionTest extends BaseTransformFunctionTe public void testTimeConversionTransformFunction(String expressionStr) { ExpressionContext expression = RequestContextUtils.getExpression(expressionStr); TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof TimeConversionTransformFunction); - Assert.assertEquals(transformFunction.getName(), TimeConversionTransformFunction.FUNCTION_NAME); - int[] intValues = transformFunction.transformToIntValuesSV(_projectionBlock); - long[] longValues = transformFunction.transformToLongValuesSV(_projectionBlock); - float[] floatValues = transformFunction.transformToFloatValuesSV(_projectionBlock); - double[] doubleValues = transformFunction.transformToDoubleValuesSV(_projectionBlock); - String[] stringValues = transformFunction.transformToStringValuesSV(_projectionBlock); + assertTrue(transformFunction instanceof TimeConversionTransformFunction); + assertEquals(transformFunction.getName(), TimeConversionTransformFunction.FUNCTION_NAME); + TransformResultMetadata resultMetadata = transformFunction.getResultMetadata(); + assertEquals(resultMetadata.getDataType(), DataType.LONG); + assertTrue(resultMetadata.isSingleValue()); + assertFalse(resultMetadata.hasDictionary()); + long[] expectedValues = new long[NUM_ROWS]; for (int i = 0; i < NUM_ROWS; i++) { - long expected = TimeUnit.MILLISECONDS.toDays(_timeValues[i]); - Assert.assertEquals(intValues[i], (int) expected); - Assert.assertEquals(longValues[i], expected); - Assert.assertEquals(floatValues[i], (float) expected); - Assert.assertEquals(doubleValues[i], (double) expected); - Assert.assertEquals(stringValues[i], Long.toString(expected)); + expectedValues[i] = TimeUnit.MILLISECONDS.toDays(_timeValues[i]); } + testTransformFunction(transformFunction, expectedValues); } @DataProvider(name = "testTimeConversionTransformFunction") diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunctionTest.java index c43671ae8d29..e9771cdf5459 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunctionTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunctionTest.java @@ -22,12 +22,17 @@ import it.unimi.dsi.fastutil.ints.IntList; import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.RequestContextUtils; +import org.apache.pinot.core.operator.transform.TransformResultMetadata; import org.apache.pinot.segment.spi.index.reader.Dictionary; +import org.apache.pinot.spi.data.FieldSpec.DataType; import org.apache.pinot.spi.exception.BadQueryRequestException; -import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + public class ValueInTransformFunctionTest extends BaseTransformFunctionTest { @@ -35,15 +40,18 @@ public class ValueInTransformFunctionTest extends BaseTransformFunctionTest { public void testValueInTransformFunction(String expressionStr) { ExpressionContext expression = RequestContextUtils.getExpression(expressionStr); TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap); - Assert.assertTrue(transformFunction instanceof ValueInTransformFunction); - Assert.assertEquals(transformFunction.getName(), ValueInTransformFunction.FUNCTION_NAME); - Assert.assertTrue(transformFunction.getResultMetadata().hasDictionary()); - int[][] dictIds = transformFunction.transformToDictIdsMV(_projectionBlock); - int[][] intValues = transformFunction.transformToIntValuesMV(_projectionBlock); - long[][] longValues = transformFunction.transformToLongValuesMV(_projectionBlock); - float[][] floatValues = transformFunction.transformToFloatValuesMV(_projectionBlock); - double[][] doubleValues = transformFunction.transformToDoubleValuesMV(_projectionBlock); - String[][] stringValues = transformFunction.transformToStringValuesMV(_projectionBlock); + assertTrue(transformFunction instanceof ValueInTransformFunction); + assertEquals(transformFunction.getName(), ValueInTransformFunction.FUNCTION_NAME); + TransformResultMetadata resultMetadata = transformFunction.getResultMetadata(); + assertEquals(resultMetadata.getDataType(), DataType.INT); + assertFalse(resultMetadata.isSingleValue()); + assertTrue(resultMetadata.hasDictionary()); + int[][] dictIdsMV = transformFunction.transformToDictIdsMV(_projectionBlock); + int[][] intValuesMV = transformFunction.transformToIntValuesMV(_projectionBlock); + long[][] longValuesMV = transformFunction.transformToLongValuesMV(_projectionBlock); + float[][] floatValuesMV = transformFunction.transformToFloatValuesMV(_projectionBlock); + double[][] doubleValuesMV = transformFunction.transformToDoubleValuesMV(_projectionBlock); + String[][] stringValuesMV = transformFunction.transformToStringValuesMV(_projectionBlock); Dictionary dictionary = transformFunction.getDictionary(); for (int i = 0; i < NUM_ROWS; i++) { @@ -56,14 +64,20 @@ public void testValueInTransformFunction(String expressionStr) { int[] expectedValues = expectedList.toIntArray(); int numValues = expectedValues.length; + assertEquals(dictIdsMV[i].length, numValues); + assertEquals(intValuesMV[i].length, numValues); + assertEquals(longValuesMV[i].length, numValues); + assertEquals(floatValuesMV[i].length, numValues); + assertEquals(doubleValuesMV[i].length, numValues); + assertEquals(stringValuesMV[i].length, numValues); for (int j = 0; j < numValues; j++) { int expected = expectedValues[j]; - Assert.assertEquals(dictIds[i][j], dictionary.indexOf(Integer.toString(expected))); - Assert.assertEquals(intValues[i][j], expected); - Assert.assertEquals(longValues[i][j], (long) expected); - Assert.assertEquals(floatValues[i][j], (float) expected); - Assert.assertEquals(doubleValues[i][j], (double) expected); - Assert.assertEquals(stringValues[i][j], Integer.toString(expected)); + assertEquals(dictIdsMV[i][j], dictionary.indexOf(expected)); + assertEquals(intValuesMV[i][j], expected); + assertEquals(longValuesMV[i][j], expected); + assertEquals(floatValuesMV[i][j], (float) expected); + assertEquals(doubleValuesMV[i][j], (double) expected); + assertEquals(stringValuesMV[i][j], Integer.toString(expected)); } } } diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/ForwardIndexReader.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/ForwardIndexReader.java index 8233aa2894b8..09550a956b66 100644 --- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/ForwardIndexReader.java +++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/ForwardIndexReader.java @@ -322,14 +322,14 @@ default void readValuesSV(int[] docIds, int length, BigDecimal[] values, T conte values[i] = BigDecimal.valueOf(getDouble(docIds[i], context)); } break; - case STRING: + case BIG_DECIMAL: for (int i = 0; i < length; i++) { - values[i] = new BigDecimal(getString(docIds[i], context)); + values[i] = getBigDecimal(docIds[i], context); } break; - case BIG_DECIMAL: + case STRING: for (int i = 0; i < length; i++) { - values[i] = getBigDecimal(docIds[i], context); + values[i] = new BigDecimal(getString(docIds[i], context)); } break; case BYTES: diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java index d73816dbc6c3..02c825470faf 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java @@ -451,7 +451,8 @@ public int size() { } /** - * Returns {@code true} if the data type is numeric (INT, LONG, FLOAT, DOUBLE), {@code false} otherwise. + * Returns {@code true} if the data type is numeric (INT, LONG, FLOAT, DOUBLE, BIG_DECIMAL), {@code false} + * otherwise. */ public boolean isNumeric() { return _numeric; diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/ArrayCopyUtils.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/ArrayCopyUtils.java index 4ece095e3606..3fbe9fcaa82c 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/ArrayCopyUtils.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/ArrayCopyUtils.java @@ -29,12 +29,6 @@ public class ArrayCopyUtils { private ArrayCopyUtils() { } - public static void copy(int[] src, int[] dest, int length) { - for (int i = 0; i < length; i++) { - dest[i] = src[i]; - } - } - public static void copy(int[] src, long[] dest, int length) { for (int i = 0; i < length; i++) { dest[i] = src[i]; @@ -59,6 +53,12 @@ public static void copy(int[] src, BigDecimal[] dest, int length) { } } + public static void copyToBoolean(int[] src, int[] dest, int length) { + for (int i = 0; i < length; i++) { + dest[i] = src[i] != 0 ? 1 : 0; + } + } + public static void copy(int[] src, String[] dest, int length) { for (int i = 0; i < length; i++) { dest[i] = Integer.toString(src[i]); @@ -89,29 +89,15 @@ public static void copy(long[] src, BigDecimal[] dest, int length) { } } - public static void copy(long[] src, String[] dest, int length) { - for (int i = 0; i < length; i++) { - dest[i] = Long.toString(src[i]); - } - } - - // specialize copy from timestamp array to string array - public static void copyTimestamp(long[] src, String[] dest, int length) { - for (int i = 0; i < length; i++) { - dest[i] = new Timestamp(src[i]).toString(); - } - } - - // specialize copy from boolean array to string array - public static void copyBoolean(int[] src, String[] dest, int length) { + public static void copyToBoolean(long[] src, int[] dest, int length) { for (int i = 0; i < length; i++) { - dest[i] = Boolean.toString(src[i] == 1); + dest[i] = src[i] != 0 ? 1 : 0; } } - public static void copy(long[] src, long[] dest, int length) { + public static void copy(long[] src, String[] dest, int length) { for (int i = 0; i < length; i++) { - dest[i] = src[i]; + dest[i] = Long.toString(src[i]); } } @@ -139,15 +125,15 @@ public static void copy(float[] src, BigDecimal[] dest, int length) { } } - public static void copy(float[] src, String[] dest, int length) { + public static void copyToBoolean(float[] src, int[] dest, int length) { for (int i = 0; i < length; i++) { - dest[i] = Float.toString(src[i]); + dest[i] = src[i] != 0 ? 1 : 0; } } - public static void copy(float[] src, float[] dest, int length) { + public static void copy(float[] src, String[] dest, int length) { for (int i = 0; i < length; i++) { - dest[i] = src[i]; + dest[i] = Float.toString(src[i]); } } @@ -177,15 +163,15 @@ public static void copy(double[] src, BigDecimal[] dest, int length) { } } - public static void copy(double[] src, String[] dest, int length) { + public static void copyToBoolean(double[] src, int[] dest, int length) { for (int i = 0; i < length; i++) { - dest[i] = Double.toString(src[i]); + dest[i] = src[i] != 0 ? 1 : 0; } } - public static void copy(double[] src, double[] dest, int length) { + public static void copy(double[] src, String[] dest, int length) { for (int i = 0; i < length; i++) { - dest[i] = src[i]; + dest[i] = Double.toString(src[i]); } } @@ -213,6 +199,12 @@ public static void copy(BigDecimal[] src, double[] dest, int length) { } } + public static void copyToBoolean(BigDecimal[] src, int[] dest, int length) { + for (int i = 0; i < length; i++) { + dest[i] = !src[i].equals(BigDecimal.ZERO) ? 1 : 0; + } + } + public static void copy(BigDecimal[] src, String[] dest, int length) { for (int i = 0; i < length; i++) { dest[i] = src[i].toPlainString(); @@ -225,6 +217,18 @@ public static void copy(BigDecimal[] src, byte[][] dest, int length) { } } + public static void copyFromBoolean(int[] src, String[] dest, int length) { + for (int i = 0; i < length; i++) { + dest[i] = Boolean.toString(src[i] == 1); + } + } + + public static void copyFromTimestamp(long[] src, String[] dest, int length) { + for (int i = 0; i < length; i++) { + dest[i] = new Timestamp(src[i]).toString(); + } + } + public static void copy(String[] src, int[] dest, int length) { for (int i = 0; i < length; i++) { dest[i] = Double.valueOf(src[i]).intValue(); @@ -255,21 +259,21 @@ public static void copy(String[] src, BigDecimal[] dest, int length) { } } - public static void copy(String[] src, byte[][] dest, int length) { + public static void copyToBoolean(String[] src, int[] dest, int length) { for (int i = 0; i < length; i++) { - dest[i] = BytesUtils.toBytes(src[i]); + dest[i] = BooleanUtils.toInt(src[i]); } } - public static void copy(String[] src, String[] dest, int length) { + public static void copyToTimestamp(String[] src, long[] dest, int length) { for (int i = 0; i < length; i++) { - dest[i] = src[i]; + dest[i] = TimestampUtils.toMillisSinceEpoch(src[i]); } } - public static void copy(byte[][] src, String[] dest, int length) { + public static void copy(String[] src, byte[][] dest, int length) { for (int i = 0; i < length; i++) { - dest[i] = BytesUtils.toHexString(src[i]); + dest[i] = BytesUtils.toBytes(src[i]); } } @@ -278,11 +282,10 @@ public static void copy(byte[][] src, BigDecimal[] dest, int length) { dest[i] = BigDecimalUtils.deserialize(src[i]); } } - public static void copy(int[][] src, int[][] dest, int length) { + + public static void copy(byte[][] src, String[] dest, int length) { for (int i = 0; i < length; i++) { - int rowLength = src[i].length; - dest[i] = new int[rowLength]; - copy(src[i], dest[i], rowLength); + dest[i] = BytesUtils.toHexString(src[i]); } } @@ -310,39 +313,39 @@ public static void copy(int[][] src, double[][] dest, int length) { } } - public static void copy(int[][] src, String[][] dest, int length) { + public static void copy(int[][] src, BigDecimal[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new String[rowLength]; + dest[i] = new BigDecimal[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(int[][] src, BigDecimal[][] dest, int length) { + public static void copyToBoolean(int[][] src, int[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new BigDecimal[rowLength]; - copy(src[i], dest[i], rowLength); + dest[i] = new int[rowLength]; + copyToBoolean(src[i], dest[i], rowLength); } } - public static void copy(double[][] src, int[][] dest, int length) { + public static void copy(int[][] src, String[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new int[rowLength]; + dest[i] = new String[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(double[][] src, long[][] dest, int length) { + public static void copy(long[][] src, int[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new long[rowLength]; + dest[i] = new int[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(double[][] src, float[][] dest, int length) { + public static void copy(long[][] src, float[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; dest[i] = new float[rowLength]; @@ -350,7 +353,7 @@ public static void copy(double[][] src, float[][] dest, int length) { } } - public static void copy(double[][] src, double[][] dest, int length) { + public static void copy(long[][] src, double[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; dest[i] = new double[rowLength]; @@ -358,23 +361,31 @@ public static void copy(double[][] src, double[][] dest, int length) { } } - public static void copy(double[][] src, String[][] dest, int length) { + public static void copy(long[][] src, BigDecimal[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new String[rowLength]; + dest[i] = new BigDecimal[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(double[][] src, BigDecimal[][] dest, int length) { + public static void copyToBoolean(long[][] src, int[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new BigDecimal[rowLength]; + dest[i] = new int[rowLength]; + copyToBoolean(src[i], dest[i], rowLength); + } + } + + public static void copy(long[][] src, String[][] dest, int length) { + for (int i = 0; i < length; i++) { + int rowLength = src[i].length; + dest[i] = new String[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(String[][] src, int[][] dest, int length) { + public static void copy(float[][] src, int[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; dest[i] = new int[rowLength]; @@ -382,7 +393,7 @@ public static void copy(String[][] src, int[][] dest, int length) { } } - public static void copy(String[][] src, long[][] dest, int length) { + public static void copy(float[][] src, long[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; dest[i] = new long[rowLength]; @@ -390,39 +401,39 @@ public static void copy(String[][] src, long[][] dest, int length) { } } - public static void copy(String[][] src, float[][] dest, int length) { + public static void copy(float[][] src, double[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new float[rowLength]; + dest[i] = new double[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(String[][] src, double[][] dest, int length) { + public static void copy(float[][] src, BigDecimal[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new double[rowLength]; + dest[i] = new BigDecimal[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(String[][] src, String[][] dest, int length) { + public static void copyToBoolean(float[][] src, int[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new String[rowLength]; - copy(src[i], dest[i], rowLength); + dest[i] = new int[rowLength]; + copyToBoolean(src[i], dest[i], rowLength); } } - public static void copy(String[][] src, BigDecimal[][] dest, int length) { + public static void copy(float[][] src, String[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new BigDecimal[rowLength]; + dest[i] = new String[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(long[][] src, int[][] dest, int length) { + public static void copy(double[][] src, int[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; dest[i] = new int[rowLength]; @@ -430,7 +441,7 @@ public static void copy(long[][] src, int[][] dest, int length) { } } - public static void copy(long[][] src, long[][] dest, int length) { + public static void copy(double[][] src, long[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; dest[i] = new long[rowLength]; @@ -438,7 +449,7 @@ public static void copy(long[][] src, long[][] dest, int length) { } } - public static void copy(long[][] src, float[][] dest, int length) { + public static void copy(double[][] src, float[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; dest[i] = new float[rowLength]; @@ -446,15 +457,23 @@ public static void copy(long[][] src, float[][] dest, int length) { } } - public static void copy(long[][] src, double[][] dest, int length) { + public static void copy(double[][] src, BigDecimal[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new double[rowLength]; + dest[i] = new BigDecimal[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(long[][] src, String[][] dest, int length) { + public static void copyToBoolean(double[][] src, int[][] dest, int length) { + for (int i = 0; i < length; i++) { + int rowLength = src[i].length; + dest[i] = new int[rowLength]; + copyToBoolean(src[i], dest[i], rowLength); + } + } + + public static void copy(double[][] src, String[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; dest[i] = new String[rowLength]; @@ -462,55 +481,55 @@ public static void copy(long[][] src, String[][] dest, int length) { } } - public static void copy(long[][] src, BigDecimal[][] dest, int length) { + public static void copyFromBoolean(int[][] src, String[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new BigDecimal[rowLength]; - copy(src[i], dest[i], rowLength); + dest[i] = new String[rowLength]; + copyFromBoolean(src[i], dest[i], rowLength); } } - public static void copy(float[][] src, int[][] dest, int length) { + public static void copyFromTimestamp(long[][] src, String[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new int[rowLength]; - copy(src[i], dest[i], rowLength); + dest[i] = new String[rowLength]; + copyFromTimestamp(src[i], dest[i], rowLength); } } - public static void copy(float[][] src, long[][] dest, int length) { + public static void copy(String[][] src, int[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new long[rowLength]; + dest[i] = new int[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(float[][] src, float[][] dest, int length) { + public static void copy(String[][] src, long[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new float[rowLength]; + dest[i] = new long[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(float[][] src, double[][] dest, int length) { + public static void copy(String[][] src, float[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new double[rowLength]; + dest[i] = new float[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(float[][] src, String[][] dest, int length) { + public static void copy(String[][] src, double[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new String[rowLength]; + dest[i] = new double[rowLength]; copy(src[i], dest[i], rowLength); } } - public static void copy(float[][] src, BigDecimal[][] dest, int length) { + public static void copy(String[][] src, BigDecimal[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; dest[i] = new BigDecimal[rowLength]; @@ -518,19 +537,19 @@ public static void copy(float[][] src, BigDecimal[][] dest, int length) { } } - public static void copyTimestamp(long[][] src, String[][] dest, int length) { + public static void copyToBoolean(String[][] src, int[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new String[rowLength]; - copyTimestamp(src[i], dest[i], rowLength); + dest[i] = new int[rowLength]; + copyToBoolean(src[i], dest[i], rowLength); } } - public static void copyBoolean(int[][] src, String[][] dest, int length) { + public static void copyToTimestamp(String[][] src, long[][] dest, int length) { for (int i = 0; i < length; i++) { int rowLength = src[i].length; - dest[i] = new String[rowLength]; - copyBoolean(src[i], dest[i], rowLength); + dest[i] = new long[rowLength]; + copyToTimestamp(src[i], dest[i], rowLength); } } }