Updates to latest conformance tests

test/partiql-tests-runner/src/main/kotlin/org/partiql/runner/ReportAnalyzer.kt

@@ -18,6 +18,7 @@ class ReportAnalyzer(
         const val ICON_X = ":x:"
         const val ICON_CHECK = ":white_check_mark:"
         const val ICON_CIRCLE_RED = ":o:"
+        const val ICON_DIAMOND_ORANGE = ":large_orange_diamond:"
         const val BASE = "BASE"
         const val TARGET = "TARGET"
     }
@@ -63,31 +64,31 @@ class ReportAnalyzer(
         out.appendLine(tableRow("% Passing", firstPassingPercent, secondPassingPercent))
         out.appendLine(tableRow("Passing", firstPassingSize, secondPassingSize, true))
         out.appendLine(tableRow("Failing", firstFailingSize, secondFailingSize, false))
-        out.appendLine(tableRow("Ignored", firstIgnoreSize, secondIgnoreSize, false))
+        out.appendLine(tableRow("Ignored", firstIgnoreSize, secondIgnoreSize, false, badIcon = ICON_DIAMOND_ORANGE))
         out.appendLine(tableRow("Total Tests", firstTotalSize, secondTotalSize, true))
     }
 
     private fun tableRow(name: String, first: Double, second: Double): String {
         val firstString = "%.2f".format(first)
         val secondString = "%.2f".format(second)
         val delta = second - first
-        val deltaIcon = getIconForComparison(first, second, true)
+        val deltaIcon = getIconForComparison(first, second, true, ICON_CHECK, ICON_CIRCLE_RED)
         val deltaString = "%.2f".format(delta)
         return "| $name | $firstString% | $secondString% | $deltaString% $deltaIcon |"
     }
 
-    private fun tableRow(name: String, first: Int, second: Int, positiveDeltaGood: Boolean): String {
+    private fun tableRow(name: String, first: Int, second: Int, positiveDeltaGood: Boolean, goodIcon: String = ICON_CHECK, badIcon: String = ICON_CIRCLE_RED): String {
         val delta = second - first
-        val deltaIcon = getIconForComparison(first, second, positiveDeltaGood)
+        val deltaIcon = getIconForComparison(first, second, positiveDeltaGood, goodIcon, badIcon)
         return "| $name | $first | $second | $delta $deltaIcon |"
     }
 
-    private fun <T> getIconForComparison(first: Comparable<T>, second: T, positiveDeltaGood: Boolean): String {
+    private fun <T> getIconForComparison(first: Comparable<T>, second: T, positiveDeltaGood: Boolean, goodIcon: String, badIcon: String): String {
         val comparison = first.compareTo(second)
         return when {
-            comparison < 0 -> if (positiveDeltaGood) ICON_CHECK else ICON_CIRCLE_RED
-            comparison == 0 -> ICON_CHECK
-            else -> if (positiveDeltaGood) ICON_CIRCLE_RED else ICON_CHECK
+            comparison < 0 -> if (positiveDeltaGood) goodIcon else badIcon
+            comparison == 0 -> goodIcon
+            else -> if (positiveDeltaGood) badIcon else goodIcon
         }
     }
 

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/ConformanceTestEval.kt

@@ -60,7 +60,207 @@ class ConformanceTestEval : ConformanceTestBase<PartiQLStatement<*>, PartiQLResu
         Pair("subtractionOutOfAllowedPrecision", ERROR_EVAL_MODE_COMPILE_OPTIONS),
     )
 
-    private val testsToFix: List<Pair<String, CompileOptions>> = aliasTests + arithmeticCases
+    /**
+     * TODO: Fix these once https://github.com/partiql/partiql-tests/pull/124 is decided upon.
+     */
+    private val undefinedVariableCases = listOf(
+        Pair("GROUP BY binding referenced in FROM clause", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("GROUP BY binding referenced in WHERE clause", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("GROUP AS binding referenced in FROM clause", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("GROUP AS binding referenced in WHERE clause", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair(
+            "GROUP BY with JOIN : SELECT supplierName, COUNT(*) as the_count FROM suppliers AS s INNER JOIN products AS p ON s.supplierId = p.supplierId GROUP BY supplierName",
+            COERCE_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "GROUP BY with JOIN : SELECT supplierName, COUNT(*) as the_count FROM suppliers AS s INNER JOIN products AS p ON s.supplierId = p.supplierId GROUP BY supplierName",
+            ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "SELECT col1, g FROM simple_1_col_1_group, join_me GROUP BY col1 GROUP AS g",
+            COERCE_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "SELECT col1, g FROM simple_1_col_1_group, join_me GROUP BY col1 GROUP AS g",
+            ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "SELECT VALUE { 'col1': col1, 'g': g } FROM simple_1_col_1_group, join_me GROUP BY col1 GROUP AS g",
+            COERCE_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "SELECT VALUE { 'col1': col1, 'g': g } FROM simple_1_col_1_group, join_me GROUP BY col1 GROUP AS g",
+            ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "SELECT col1, g FROM simple_1_col_1_group, different_types_per_row GROUP BY col1 GROUP AS g",
+            COERCE_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "SELECT col1, g FROM simple_1_col_1_group, different_types_per_row GROUP BY col1 GROUP AS g",
+            ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "SELECT VALUE { 'col1': col1, 'g': g } FROM simple_1_col_1_group, different_types_per_row GROUP BY col1 GROUP AS g",
+            COERCE_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "SELECT VALUE { 'col1': col1, 'g': g } FROM simple_1_col_1_group, different_types_per_row GROUP BY col1 GROUP AS g",
+            ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair("select fld3,period from t1,t2 where fld1 = 011401", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("select fld3,period from t2,t1 where companynr*10 = 37*10", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("undefinedUnqualifiedVariableWithUndefinedVariableBehaviorMissing", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair(
+            "undefinedUnqualifiedVariableIsNullExprWithUndefinedVariableBehaviorMissing",
+            COERCE_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "undefinedUnqualifiedVariableIsMissingExprWithUndefinedVariableBehaviorMissing",
+            COERCE_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair("MYSQL_SELECT_20", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("MYSQL_SELECT_20", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("MYSQL_SELECT_21", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("MYSQL_SELECT_21", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+    )
+
+    /**
+     * TODO
+     */
+    private val negativeOffset = listOf(
+        Pair("offset -1", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("offset -1", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("offset 1-2", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+    )
+
+    /**
+     * TODO
+     */
+    private val mistypedCollAggs = listOf(
+        Pair("COLL_MAX non-collection", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("COLL_AVG non-collection", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("COLL_COUNT non-collection", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("COLL_SUM non-collection", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("COLL_MIN non-collection", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("COLL_ANY non-collection", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("COLL_SOME non-collection", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("COLL_EVERY non-collection", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+    )
+
+    /**
+     * TODO
+     */
+    private val miscellaneousCases = listOf(
+        Pair("subscript with non-existent variable in lowercase", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("subscript with non-existent variable in uppercase", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair(
+            "repeated field on struct is ambiguous{identifier:\"REPEATED\",cn:9,bn:\"REPEATED\"}",
+            ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "repeated field on struct is ambiguous{identifier:\" \\\"repeated\\\" \",cn:10,bn:\"repeated\"}",
+            ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair("invalid extract year from time", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("invalid extract month from time", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("invalid extract day from time", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("invalid extract month from time with time zone", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("invalid extract day from time with time zone", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+    )
+
+    /**
+     * TODO
+     */
+    private val mistypedFunctions = listOf(
+        Pair("MOD(MISSING, 'some string')", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("MOD('some string', MISSING)", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("MOD(NULL, 'some string')", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("MOD('some string', NULL)", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("MOD(3, 'some string')", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("MOD('some string', 3)", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("CARDINALITY('foo') type mismatch", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("OCTET_LENGTH invalid type", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("CHARACTER_LENGTH invalid type", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("ABS('foo')", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("POSITION invalid type in string", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("POSITION string in invalid type", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("BIT_LENGTH invalid type", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+    )
+
+    /**
+     * TODO
+     */
+    private val invalidCasts = listOf(
+        Pair(
+            "cast to int invalid target type{value:\"`2017T`\",target:\"TIMESTAMP\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "cast to int invalid target type{value:\" `{{\\\"\\\"}}` \",target:\"CLOB\"}",
+            ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair(
+            "cast to int invalid target type{value:\" `{{\\\"1\\\"}}` \",target:\"CLOB\"}",
+            ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair("cast to int invalid target type{value:\"`{{}}`\",target:\"BLOB\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("cast to int invalid target type{value:\"[1, 2]\",target:\"LIST\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("cast to int invalid target type{value:\"[1]\",target:\"LIST\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("cast to int invalid target type{value:\"[]\",target:\"LIST\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("cast to int invalid target type{value:\"`(1 2)`\",target:\"SEXP\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("cast to int invalid target type{value:\"`(1)`\",target:\"SEXP\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("cast to int invalid target type{value:\"`()`\",target:\"SEXP\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("cast to int invalid target type{value:\"{'a': 1}\",target:\"STRUCT\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair(
+            "cast to int invalid target type{value:\"{'a': '12'}\",target:\"STRUCT\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS
+        ),
+        Pair("cast to int invalid target type{value:\"{}\",target:\"STRUCT\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("cast to int invalid target type{value:\"<<1, 2>>\",target:\"BAG\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("cast to int invalid target type{value:\"<<1>>\",target:\"BAG\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("cast to int invalid target type{value:\"<<>>\",target:\"BAG\"}", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+    )
+
+    /**
+     * TODO
+     */
+    private val specialForms = listOf(
+        Pair("More than one character given for ESCAPE", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("LIKE bad value type", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("LIKE bad pattern type", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("LIKE bad escape type", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+    )
+
+    /**
+     * TODO
+     */
+    private val operators = listOf(
+        Pair("array navigation with wrongly typed array index", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("data type mismatch in comparison expression", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("data type mismatch in logical expression", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+    )
+
+    /**
+     * TODO
+     */
+    private val bagOps = listOf(
+        Pair("Example 6 - Value Coercion not union-compatible", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("Example 6 - Value Coercion not union-compatible", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+    )
+
+    /**
+     * TODO: Determine whether SEXP should be treated as a first-class collection.
+     *  See: https://github.com/partiql/partiql-tests/pull/126#discussion_r1710271573
+     */
+    private val sexpTests = listOf(
+        Pair("projectOfSexp", COERCE_EVAL_MODE_COMPILE_OPTIONS),
+        Pair("projectOfSexp", ERROR_EVAL_MODE_COMPILE_OPTIONS),
+    )
+
+    /**
+     * TODO
+     */
+    private val testsToFix: List<Pair<String, CompileOptions>> =
+        aliasTests + arithmeticCases + undefinedVariableCases + sexpTests + negativeOffset + mistypedFunctions + invalidCasts + miscellaneousCases + mistypedCollAggs + specialForms + operators + bagOps
 
     /**
      * This holds all of the Graph Pattern Matching Language conformance tests. The new evaluator does not yet support

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/report/ReportGenerator.kt

@@ -9,15 +9,24 @@ import java.io.File
 import java.nio.file.Files
 import kotlin.io.path.Path
 
-class ReportGenerator(val engine: String) : TestWatcher, AfterAllCallback {
-    var failingTests = emptySet<String>()
-    var passingTests = emptySet<String>()
-    var ignoredTests = emptySet<String>()
+class ReportGenerator(
+    private val engine: String
+) : TestWatcher, AfterAllCallback {
+
+    private var failingTests = emptySet<String>()
+    private var passingTests = emptySet<String>()
+    private var ignoredTests = emptySet<String>()
+
     override fun testFailed(context: ExtensionContext?, cause: Throwable?) {
         failingTests += context?.displayName ?: ""
         super.testFailed(context, cause)
     }
 
+    override fun testAborted(context: ExtensionContext?, cause: Throwable?) {
+        ignoredTests += context?.displayName ?: ""
+        super.testAborted(context, cause)
+    }
+
     override fun testSuccessful(context: ExtensionContext?) {
         passingTests += context?.displayName ?: ""
         super.testSuccessful(context)

test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/test/TestRunner.kt

@@ -1,5 +1,6 @@
 package org.partiql.runner.test
 
+import org.junit.jupiter.api.Assumptions
 import org.partiql.lang.eval.CompileOptions
 import org.partiql.runner.schema.Assertion
 import org.partiql.runner.schema.TestCase
@@ -10,18 +11,14 @@ import org.partiql.runner.schema.TestCase
 class TestRunner<T, V>(private val factory: TestExecutor.Factory<T, V>) {
 
     fun test(case: TestCase.Eval, skipList: List<Pair<String, CompileOptions>>) {
-        if (skipList.contains((Pair(case.name, case.compileOptions)))) {
-            return
-        }
+        Assumptions.assumeFalse(skipList.contains((Pair(case.name, case.compileOptions))))
         val executor = factory.create(case.env, case.compileOptions)
         val input = case.statement
         run(input, case, executor)
     }
 
     fun test(case: TestCase.Equiv, skipList: List<Pair<String, CompileOptions>>) {
-        if (skipList.contains((Pair(case.name, case.compileOptions)))) {
-            return
-        }
+        Assumptions.assumeFalse(skipList.contains((Pair(case.name, case.compileOptions))))
         val executor = factory.create(case.env, case.compileOptions)
         case.statements.forEach { run(it, case, executor) }
     }