From 25e38dd31d93509e186d57ef75e798a279e9a5fb Mon Sep 17 00:00:00 2001 From: Jordan Lewis Date: Fri, 10 Sep 2021 14:19:54 -0300 Subject: [PATCH] sql: support in-memory JSON arrays This commit adds support for SQL arrays of JSON in memory. This allows functions like `array_agg` to work against JSON objects, which is useful for compatibility. Notably, this commit does not add support for storing JSON arrays in table columns. Release note (sql change): add support for SQL arrays containing JSON for in-memory processing. This does not add support for storing SQL arrays of JSON in tables. --- docs/generated/sql/aggregates.md | 2 ++ docs/generated/sql/functions.md | 14 +++++++++ pkg/sql/catalog/colinfo/BUILD.bazel | 1 + pkg/sql/catalog/colinfo/col_type_info.go | 6 ++++ pkg/sql/logictest/testdata/logic_test/json | 36 ++++++++++++++-------- pkg/sql/opt/optbuilder/testdata/aggregate | 1 + pkg/sql/opt/optbuilder/testdata/scalar | 16 ++++++++-- pkg/sql/rowenc/column_type_encoding.go | 8 +++++ pkg/sql/types/types.go | 2 -- 9 files changed, 69 insertions(+), 17 deletions(-) diff --git a/docs/generated/sql/aggregates.md b/docs/generated/sql/aggregates.md index a57ae3a2059d..902eb806bdee 100644 --- a/docs/generated/sql/aggregates.md +++ b/docs/generated/sql/aggregates.md @@ -33,6 +33,8 @@ array_agg(arg1: geometry) → geometry[]

Aggregates the selected values into an array.

 +array_agg(arg1: jsonb) → jsonb[]

Aggregates the selected values into an array.

+ array_agg(arg1: oid) → oid[]

Aggregates the selected values into an array.

 array_agg(arg1: timetz) → timetz[]

Aggregates the selected values into an array.

diff --git a/docs/generated/sql/functions.md b/docs/generated/sql/functions.md index 049dcdc89c97..d4e5a8756389 100644 --- a/docs/generated/sql/functions.md +++ b/docs/generated/sql/functions.md @@ -35,6 +35,8 @@ array_append(array: geometry[], elem: geometry) → geometry[]

Appends elem to array, returning the result.

 +array_append(array: jsonb[], elem: jsonb) → jsonb[]

Appends elem to array, returning the result.

+ array_append(array: oid[], elem: oid) → oid[]

Appends elem to array, returning the result.

 array_append(array: timetz[], elem: timetz) → timetz[]

Appends elem to array, returning the result.

@@ -73,6 +75,8 @@ array_cat(left: geometry[], right: geometry[]) → geometry[]

Appends two arrays.

 +array_cat(left: jsonb[], right: jsonb[]) → jsonb[]

Appends two arrays.

+ array_cat(left: oid[], right: oid[]) → oid[]

Appends two arrays.

 array_cat(left: timetz[], right: timetz[]) → timetz[]

Appends two arrays.

@@ -115,6 +119,8 @@ array_position(array: geometry[], elem: geometry) → int

Return the index of the first occurrence of elem in array.

 +array_position(array: jsonb[], elem: jsonb) → int

Return the index of the first occurrence of elem in array.

+ array_position(array: oid[], elem: oid) → int

Return the index of the first occurrence of elem in array.

 array_position(array: timetz[], elem: timetz) → int

Return the index of the first occurrence of elem in array.

@@ -153,6 +159,8 @@ array_positions(array: geometry[], elem: geometry) → int[]

Returns and array of indexes of all occurrences of elem in array.

 +array_positions(array: jsonb[], elem: jsonb) → int[]

Returns and array of indexes of all occurrences of elem in array.

+ array_positions(array: oid[], elem: oid) → int[]

Returns and array of indexes of all occurrences of elem in array.

 array_positions(array: timetz[], elem: timetz) → int[]

Returns and array of indexes of all occurrences of elem in array.

@@ -191,6 +199,8 @@ array_prepend(elem: geometry, array: geometry[]) → geometry[]

Prepends elem to array, returning the result.

 +array_prepend(elem: jsonb, array: jsonb[]) → jsonb[]

Prepends elem to array, returning the result.

+ array_prepend(elem: oid, array: oid[]) → oid[]

Prepends elem to array, returning the result.

 array_prepend(elem: timetz, array: timetz[]) → timetz[]

Prepends elem to array, returning the result.

@@ -229,6 +239,8 @@ array_remove(array: geometry[], elem: geometry) → geometry[]

Remove from array all elements equal to elem.

 +array_remove(array: jsonb[], elem: jsonb) → jsonb[]

Remove from array all elements equal to elem.

+ array_remove(array: oid[], elem: oid) → oid[]

Remove from array all elements equal to elem.

 array_remove(array: timetz[], elem: timetz) → timetz[]

Remove from array all elements equal to elem.

@@ -267,6 +279,8 @@ array_replace(array: geometry[], toreplace: geometry, replacewith: geometry) → geometry[]

Replace all occurrences of toreplace in array with replacewith.

 +array_replace(array: jsonb[], toreplace: jsonb, replacewith: jsonb) → jsonb[]

Replace all occurrences of toreplace in array with replacewith.

+ array_replace(array: oid[], toreplace: oid, replacewith: oid) → oid[]

Replace all occurrences of toreplace in array with replacewith.

 array_replace(array: timetz[], toreplace: timetz, replacewith: timetz) → timetz[]

Replace all occurrences of toreplace in array with replacewith.

diff --git a/pkg/sql/catalog/colinfo/BUILD.bazel b/pkg/sql/catalog/colinfo/BUILD.bazel index b85f599c121f..8fb6c4ae4925 100644 --- a/pkg/sql/catalog/colinfo/BUILD.bazel +++ b/pkg/sql/catalog/colinfo/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//pkg/sql/sem/tree", "//pkg/sql/types", "//pkg/util/encoding", + "//pkg/util/errorutil/unimplemented", "@com_github_cockroachdb_errors//:errors", "@com_github_lib_pq//oid", "@org_golang_x_text//language", diff --git a/pkg/sql/catalog/colinfo/col_type_info.go b/pkg/sql/catalog/colinfo/col_type_info.go index d1808fb090a7..f188587c2309 100644 --- a/pkg/sql/catalog/colinfo/col_type_info.go +++ b/pkg/sql/catalog/colinfo/col_type_info.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/util/errorutil/unimplemented" "github.com/cockroachdb/errors" "github.com/lib/pq/oid" "golang.org/x/text/language" @@ -92,6 +93,11 @@ func ValidateColumnDefType(t *types.T) error { // Nested arrays are not supported as a column type. return errors.Errorf("nested array unsupported as column type: %s", t.String()) } + if t.ArrayContents().Family() == types.JsonFamily { + // JSON arrays are not supported as a column type. + return unimplemented.NewWithIssueDetailf(23468, t.String(), + "arrays of JSON unsupported as column type") + } if err := types.CheckArrayElementType(t.ArrayContents()); err != nil { return err } diff --git a/pkg/sql/logictest/testdata/logic_test/json b/pkg/sql/logictest/testdata/logic_test/json index c9908a10f134..199f1391ad4b 100644 --- a/pkg/sql/logictest/testdata/logic_test/json +++ b/pkg/sql/logictest/testdata/logic_test/json @@ -84,20 +84,17 @@ SELECT NULL::JSON ---- NULL -statement error arrays of jsonb not allowed.*\nHINT:.*\n.*23468 -SELECT ARRAY['"hello"'::JSON] - -statement error arrays of jsonb not allowed.*\nHINT:.*\n.*23468 -SELECT '{}'::JSONB[] - -statement error arrays of jsonb not allowed.*\nHINT:.*\n.*23468 +statement error arrays of JSON unsupported as column type.*\nHINT:.*\n.*23468 CREATE TABLE x (y JSONB[]) statement ok -CREATE TABLE foo (bar JSON) +CREATE TABLE foo (pk INT DEFAULT unique_rowid(), bar JSON) + +statement error arrays of JSON unsupported as column type.*\nHINT:.*\n.*23468 +CREATE VIEW x AS SELECT array_agg(bar) FROM foo statement ok -INSERT INTO foo VALUES +INSERT INTO foo(bar) VALUES ('{"a": "b"}'), ('[1, 2, 3]'), ('"hello"'), @@ -201,19 +198,24 @@ NULL NULL NULL -query T +query IT SELECT * from foo where bar->'x' = '[1]' ---- -query T +query IT SELECT * from foo where bar->'x' = '{}' ---- +query T +SELECT array_agg(bar ORDER BY pk) FROM foo +---- +{"'{\"a\": \"b\"}'","'[1, 2, 3]'","\"hello\"",1.000,true,false,NULL,"'{\"x\": [1, 2, 3]}'","'{\"x\": {\"y\": \"z\"}}'"} + statement ok DELETE FROM foo statement ok -INSERT INTO foo VALUES ('{"a": {"c": "d"}}'); +INSERT INTO foo(bar) VALUES ('{"a": {"c": "d"}}'); query TT SELECT bar->'a'->'c', bar->'a'->>'c' FROM foo @@ -840,3 +842,13 @@ SELECT j - s FROM t57165 ---- {} {} + +query T +SELECT ARRAY['"hello"'::JSON] +---- +{"\"hello\""} + +query T +SELECT '{}'::JSONB[] +---- +{} diff --git a/pkg/sql/opt/optbuilder/testdata/aggregate b/pkg/sql/opt/optbuilder/testdata/aggregate index f03555b53ac0..f53e5cdd8979 100644 --- a/pkg/sql/opt/optbuilder/testdata/aggregate +++ b/pkg/sql/opt/optbuilder/testdata/aggregate @@ -225,6 +225,7 @@ array_agg(uuid) -> uuid[] array_agg(inet) -> inet[] array_agg(time) -> time[] array_agg(timetz) -> timetz[] +array_agg(jsonb) -> jsonb[] array_agg(varbit) -> varbit[] array_agg(bool) -> bool[] diff --git a/pkg/sql/opt/optbuilder/testdata/scalar b/pkg/sql/opt/optbuilder/testdata/scalar index a394c2c0bdfb..d514ed820404 100644 --- a/pkg/sql/opt/optbuilder/testdata/scalar +++ b/pkg/sql/opt/optbuilder/testdata/scalar @@ -896,12 +896,14 @@ concat [type=int[]] build-scalar ARRAY['"foo"'::jsonb] ---- -error: unimplemented: arrays of jsonb not allowed +array: [type=jsonb[]] + └── const: '"foo"' [type=jsonb] build-scalar ARRAY['"foo"'::json] ---- -error: unimplemented: arrays of jsonb not allowed +array: [type=jsonb[]] + └── const: '"foo"' [type=jsonb] opt SELECT -((-9223372036854775808):::int) @@ -1030,7 +1032,15 @@ project build SELECT ARRAY(VALUES ('{}'::JSONB)) ---- -error (0A000): unimplemented: arrays of jsonb not allowed +project + ├── columns: array:2 + ├── values + │ └── () + └── projections + └── array-flatten [as=array:2] + └── values + ├── columns: column1:1!null + └── ('{}',) build SELECT ARRAY(SELECT 1, 2) diff --git a/pkg/sql/rowenc/column_type_encoding.go b/pkg/sql/rowenc/column_type_encoding.go index 8d0c5484695a..44471641ced4 100644 --- a/pkg/sql/rowenc/column_type_encoding.go +++ b/pkg/sql/rowenc/column_type_encoding.go @@ -1356,6 +1356,8 @@ func DatumTypeToArrayElementEncodingType(t *types.T) (encoding.Type, error) { return encoding.UUID, nil case types.INetFamily: return encoding.IPAddr, nil + case types.JsonFamily: + return encoding.JSON, nil default: return 0, errors.AssertionFailedf("no known encoding type for %s", t) } @@ -1427,6 +1429,12 @@ func encodeArrayElement(b []byte, d tree.Datum) ([]byte, error) { return encodeArrayElement(b, t.Wrapped) case *tree.DEnum: return encoding.EncodeUntaggedBytesValue(b, t.PhysicalRep), nil + case *tree.DJSON: + encoded, err := json.EncodeJSON(nil, t.JSON) + if err != nil { + return nil, err + } + return encoding.EncodeUntaggedBytesValue(b, encoded), nil default: return nil, errors.Errorf("don't know how to encode %s (%T)", d, d) } diff --git a/pkg/sql/types/types.go b/pkg/sql/types/types.go index 8bdd7947807e..ed991c7ff5ba 100644 --- a/pkg/sql/types/types.go +++ b/pkg/sql/types/types.go @@ -2452,8 +2452,6 @@ func IsStringType(t *T) bool { // the issue number should be included in the error report to inform the user. func IsValidArrayElementType(t *T) (valid bool, issueNum int) { switch t.Family() { - case JsonFamily: - return false, 23468 default: return true, 0 }