Skip to content

Commit

Permalink
[multistage] m0ar tests! (binary, boolean, char, time) (#9818)
Browse files Browse the repository at this point in the history
* [multistage] m0ar tests! (binary, boolean, char, time)

Co-authored-by: Rong Rong <[email protected]>
  • Loading branch information
agavra and walterddr authored Nov 28, 2022
1 parent 837aff9 commit 6ef4dfc
Show file tree
Hide file tree
Showing 9 changed files with 324 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,11 @@ public static RowDataBlock buildFromRows(List<Object[]> rows, DataSchema dataSch
setColumn(rowBuilder, byteBuffer, (String) value);
break;
case BYTES:
setColumn(rowBuilder, byteBuffer, (ByteArray) value);
if (value instanceof byte[]) {
setColumn(rowBuilder, byteBuffer, new ByteArray((byte[]) value));
} else {
setColumn(rowBuilder, byteBuffer, (ByteArray) value);
}
break;
case OBJECT:
setColumn(rowBuilder, byteBuffer, value);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.apache.calcite.rel.rules.PinotQueryRuleSets;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.runtime.CalciteContextException;
import org.apache.calcite.sql.SqlExplain;
import org.apache.calcite.sql.SqlExplainFormat;
import org.apache.calcite.sql.SqlExplainLevel;
Expand Down Expand Up @@ -129,6 +130,9 @@ public QueryPlan planQuery(String sqlQuery, SqlNodeAndOptions sqlNodeAndOptions)
plannerContext.setOptions(sqlNodeAndOptions.getOptions());
RelRoot relRoot = compileQuery(sqlNodeAndOptions.getSqlNode(), plannerContext);
return toDispatchablePlan(relRoot, plannerContext);
} catch (CalciteContextException e) {
throw new RuntimeException("Error composing query plan for '" + sqlQuery
+ "': " + e.getMessage() + "'", e);
} catch (Exception e) {
throw new RuntimeException("Error composing query plan for: " + sqlQuery, e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,14 @@ public PinotCatalog(TableCache tableCache) {
@Override
public Table getTable(String name) {
String tableName = TableNameBuilder.extractRawTableName(name);
return new PinotTable(_tableCache.getSchema(tableName));
org.apache.pinot.spi.data.Schema schema = _tableCache.getSchema(tableName);
if (schema == null) {
throw new IllegalArgumentException("Could not find schema for table: '" + tableName
+ "'. This is likely indicative of some kind of corruption and should not happen! "
+ "If you are running this via the a test environment, check to make sure you're "
+ "specifying the correct tables.");
}
return new PinotTable(schema);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -51,6 +52,7 @@
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.StringUtil;
import org.testng.Assert;

Expand Down Expand Up @@ -142,6 +144,10 @@ protected void compareRowEquals(List<Object[]> resultRows, List<Object[]> expect
} else {
return ((BigDecimal) l).compareTo(new BigDecimal((String) r));
}
} else if (l instanceof ByteArray) {
return ((ByteArray) l).compareTo(new ByteArray((byte[]) r));
} else if (l instanceof Timestamp) {
return ((Timestamp) l).compareTo((Timestamp) r);
} else {
throw new RuntimeException("non supported type " + l.getClass());
}
Expand Down Expand Up @@ -268,11 +274,17 @@ private static List<String> toH2FieldNamesAndTypes(org.apache.pinot.spi.data.Sch
case DOUBLE:
fieldType = "double";
break;
case BOOLEAN:
fieldType = "BOOLEAN";
break;
case BIG_DECIMAL:
fieldType = "NUMERIC";
break;
case BOOLEAN:
fieldType = "BOOLEAN";
case BYTES:
fieldType = "BYTEA";
break;
case TIMESTAMP:
fieldType = "TIMESTAMP";
break;
default:
throw new UnsupportedOperationException("Unsupported type conversion to h2 type: " + dataType);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,11 @@ public class ResourceBasedQueriesTest extends QueryRunnerTestBase {
"SelectExpressions.json",
"ValueExpressions.json",
"NumericTypes.json",
"CharacterTypes.json",
"BinaryTypes.json",
"TimeTypes.json",
"BooleanLogic.json",
"Comparisons.json",
"ValueExpressions.json",
"Aggregates.json"
);

Expand Down Expand Up @@ -170,7 +173,7 @@ public void testQueryTestCasesWithH2(String testCaseName, String sql, String exp
try {
compareRowEquals(rows, queryH2(sql));
} catch (Exception e) {
Assert.fail(e.getMessage());
Assert.fail(e.getMessage(), e);
}
});
}
Expand All @@ -197,7 +200,7 @@ private Optional<List<Object[]>> runQuery(String sql, final String except) {
} else {
Pattern pattern = Pattern.compile(except);
Assert.assertTrue(pattern.matcher(e.getMessage()).matches(),
String.format("Caught exception %s, but it did not match the expected pattern %s.",
String.format("Caught exception '%s', but it did not match the expected pattern '%s'.",
e.getMessage(), except));
}
}
Expand Down
42 changes: 42 additions & 0 deletions pinot-query-runtime/src/test/resources/queries/BinaryTypes.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
{
"bytea": {
"tables": {
"bytea": {
"schema": [
{"name": "data", "type": "BYTES"}
],
"inputs": [
["DEADBEEF"]
]
}
},
"queries": [
{
"psql": "8.4.1",
"description": "should select binary data",
"sql": "SELECT * FROM {bytea}"
},
{
"psql": "8.4.1",
"ignored": true,
"comments": [
"looks like we don't support constants, this is treated as a normal string",
"we also require using calcite syntax for byte strings to parse, which is x'deadbeef' instead",
"of postgres syntax which is '\\xdeadbeaf'"
],
"description": "bytea hex constants",
"sql": "SELECT x'DEADBEEF', data from {bytea}"
},
{
"psql": "8.4.2",
"ignored": true,
"comments": [
"in order to specify bytea constants using escape syntax, we need type casting",
"which we don't support. also, calcite doesn't support the escape syntax for bytes"
],
"description": "bytea escape constants",
"sql": "SELECT CAST('\\046' AS BINARY), data from {bytea}"
}
]
}
}
130 changes: 130 additions & 0 deletions pinot-query-runtime/src/test/resources/queries/BooleanLogic.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
{
"boolean_type": {
"tables": {
"bools": {
"schema": [{"name": "b", "type": "BOOLEAN"}],
"inputs": [[true], [false]]
}
},
"queries": [
{
"description": "should support booleans",
"sql": "select b FROM {bools}"
},
{
"description": "should support booleans",
"sql": "select true, false FROM {bools}"
},
{
"description": "should support booleans",
"sql": "select b FROM {bools} WHERE b"
},
{
"description": "should support booleans",
"sql": "select b FROM {bools} WHERE b = false"
},
{
"description": "should support boolean literals",
"sql": "select b = true, b = false FROM {bools}"
},
{
"description": "should support boolean inequality ",
"sql": "select b != true, b <> false FROM {bools}"
}
]
},
"boolean_implicit_casting": {
"tables": {
"tbl": {
"schema": [
{"name": "b", "type": "BOOLEAN"},
{"name": "i", "type": "INT"},
{"name": "d", "type": "DOUBLE"},
{"name": "s", "type": "STRING"},
{"name": "n", "type": "BIG_DECIMAL"},
{"name": "t", "type": "TIMESTAMP"},
{"name": "by", "type": "BYTES"}
],
"inputs": [
[true, 1, 1.2, "foo", "1.234", "2022-01-02 03:45:00", "DEADBEEF"],
[true, 0, 0.0, "", "0.0", "2022-01-02 03:45:00", "00"],
[false, 1, 1.2, "foo", "1.234", "2022-01-02 03:45:00", "DEADBEEF"],
[false, 0, 0.0, "", "0.0", "2022-01-02 03:45:00", "00"]
]
}
},
"queries": [
{
"note": "to be postgres compatible, this should throw an exception - but various other systems (like H2 and MySQL) support this",
"description": "check implicit cast between boolean and int",
"sql": "select b = i FROM {tbl}"
},
{
"description": "check implicit cast between boolean and double",
"sql": "select b = d FROM {tbl}"
},
{
"ignored": true,
"comment": "H2 doesn't support this, and behavior is somewhat unclear",
"description": "check implicit cast between boolean and string",
"sql": "select b = s FROM {tbl}"
},
{
"description": "check implicit cast between boolean and numeric",
"sql": "select b = n FROM {tbl}"
},
{
"description": "implicit cast should fail between boolean and timestamp",
"sql": "select b = t FROM {tbl}",
"expectedException": ".*Cannot apply '=' to arguments.*"
},
{
"description": "implicit should fail between boolean and bytes",
"sql": "select b = by FROM {tbl}",
"expectedException": ".*Cannot apply '=' to arguments.*"
}
]
},
"boolean_logic": {
"tables": {
"bools": {
"schema": [
{"name": "b1", "type": "BOOLEAN"},
{"name": "b2", "type": "BOOLEAN"}
],
"inputs": [
[true, true],
[true, false],
[false, false],
[false, true]
]
}
},
"queries": [
{
"description": "should support AND",
"sql": "select b1 AND b2 FROM {bools}"
},
{
"description": "should support OR",
"sql": "select b1 OR b2 FROM {bools}"
},
{
"description": "should support NOT",
"sql": "select NOT b1, NOT b2 FROM {bools}"
},
{
"description": "should support NOT with order of precedence",
"sql": "select NOT b1 AND b2, NOT b1 OR b2 FROM {bools}"
},
{
"description": "should support cascade",
"sql": "select (b1 AND b2) = true FROM {bools}"
},
{
"description": "should support combine",
"sql": "select (b1 AND b2) OR (b1 AND b1) FROM {bools}"
}
]
}
}
43 changes: 43 additions & 0 deletions pinot-query-runtime/src/test/resources/queries/CharacterTypes.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
{
"fixed_char": {
"ignored": true,
"comment": "we don't support fixed length string types",
"psql": "8.3"
},
"varchar": {
"tables": {
"varchar": {
"schema": [
{"name": "str", "type": "STRING"}
],
"inputs": [
["foo"],
["value with spaces"],
["Οὐχὶ (greek)"],
["แสน (thai)"],
["верстке (russian)"],
["∀x∈ℝ (mathematics)"]
]
}
},
"queries": [
{
"psql": "8.3",
"description": "test different UTF8 string values",
"sql": "SELECT * FROM {varchar}"
},
{
"psql": "8.3",
"description": "test inline constants",
"sql": "SELECT 'value with spaces' FROM {varchar}"
},
{
"ignored": true,
"comment": "calcite doesn't support parsing non ISO-8859-1 characters as constants",
"psql": "8.3",
"description": "test inline UTF8 constants",
"sql": "SELECT 'Οὐχὶ (greek)' FROM {varchar}"
}
]
}
}
Loading

0 comments on commit 6ef4dfc

Please sign in to comment.