From c61e322a282b6f836bf75db5f9fa10367396b4d6 Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Mon, 11 Jul 2022 15:18:49 -0400 Subject: [PATCH] Add support for inline array aggregate filters --- query/graphql/mapper/mapper.go | 13 +- query/graphql/mapper/targetable.go | 4 + query/graphql/planner/count.go | 50 ++++++- query/graphql/planner/sum.go | 14 ++ query/graphql/schema/generate.go | 21 ++- query/graphql/schema/manager.go | 4 + query/graphql/schema/root.go | 100 ++++++++++++++ .../inline_array/with_average_filter_test.go | 71 ++++++++++ .../inline_array/with_count_filter_test.go | 125 ++++++++++++++++++ .../inline_array/with_sum_filter_test.go | 71 ++++++++++ 10 files changed, 461 insertions(+), 12 deletions(-) create mode 100644 tests/integration/query/inline_array/with_average_filter_test.go create mode 100644 tests/integration/query/inline_array/with_count_filter_test.go create mode 100644 tests/integration/query/inline_array/with_sum_filter_test.go diff --git a/query/graphql/mapper/mapper.go b/query/graphql/mapper/mapper.go index 9a6bdebc7e..3a818081ff 100644 --- a/query/graphql/mapper/mapper.go +++ b/query/graphql/mapper/mapper.go @@ -134,12 +134,15 @@ func resolveAggregates( fieldDesc, isField := desc.GetField(target.hostExternalName) if isField && !fieldDesc.IsObject() { // If the hostExternalName matches a non-object field - // we can just take it as a field-requestable as only - // objects are targetable-requestables. + // we don't have to search for it and can just construct the + // targeting info here. hasHost = true - host = &Field{ - Index: int(fieldDesc.ID), - Name: target.hostExternalName, + host = &Targetable{ + Field: Field{ + Index: int(fieldDesc.ID), + Name: target.hostExternalName, + }, + Filter: ToFilter(target.filter, mapping), } } else { childObjectIndex := mapping.FirstIndexOfName(target.hostExternalName) diff --git a/query/graphql/mapper/targetable.go b/query/graphql/mapper/targetable.go index 3d0b679a22..e5e00a3942 100644 --- a/query/graphql/mapper/targetable.go +++ b/query/graphql/mapper/targetable.go @@ -158,3 +158,7 @@ func (t *Targetable) cloneTo(index int) *Targetable { OrderBy: t.OrderBy, } } + +func (t *Targetable) AsTargetable() (*Targetable, bool) { + return t, true +} diff --git a/query/graphql/planner/count.go b/query/graphql/planner/count.go index 0ca855e780..77a1b5a3bf 100644 --- a/query/graphql/planner/count.go +++ b/query/graphql/planner/count.go @@ -101,9 +101,53 @@ func (n *countNode) Next() (bool, error) { length := v.Len() if source.Filter != nil { - docArray, isDocArray := property.([]core.Doc) - if isDocArray { - for _, doc := range docArray { + switch array := property.(type) { + case []core.Doc: + for _, doc := range array { + passed, err := mapper.RunFilter(doc, source.Filter) + if err != nil { + return false, err + } + if passed { + count += 1 + } + } + + case []bool: + for _, doc := range array { + passed, err := mapper.RunFilter(doc, source.Filter) + if err != nil { + return false, err + } + if passed { + count += 1 + } + } + + case []int64: + for _, doc := range array { + passed, err := mapper.RunFilter(doc, source.Filter) + if err != nil { + return false, err + } + if passed { + count += 1 + } + } + + case []float64: + for _, doc := range array { + passed, err := mapper.RunFilter(doc, source.Filter) + if err != nil { + return false, err + } + if passed { + count += 1 + } + } + + case []string: + for _, doc := range array { passed, err := mapper.RunFilter(doc, source.Filter) if err != nil { return false, err diff --git a/query/graphql/planner/sum.go b/query/graphql/planner/sum.go index 13d144ed1c..7b1112212b 100644 --- a/query/graphql/planner/sum.go +++ b/query/graphql/planner/sum.go @@ -223,10 +223,24 @@ func (n *sumNode) Next() (bool, error) { } case []int64: for _, childItem := range childCollection { + passed, err := mapper.RunFilter(childItem, source.Filter) + if err != nil { + return false, err + } + if !passed { + continue + } sum += float64(childItem) } case []float64: for _, childItem := range childCollection { + passed, err := mapper.RunFilter(childItem, source.Filter) + if err != nil { + return false, err + } + if !passed { + continue + } sum += childItem } } diff --git a/query/graphql/schema/generate.go b/query/graphql/schema/generate.go index 9ce3f4e031..99c282ecfd 100644 --- a/query/graphql/schema/generate.go +++ b/query/graphql/schema/generate.go @@ -291,15 +291,28 @@ func (g *Generator) createExpandedFieldAggregate( ) { for _, aggregateTarget := range f.Args { target := aggregateTarget.Name() - var targetType string + var filterTypeName string if target == parserTypes.GroupFieldName { - targetType = obj.Name() + filterTypeName = obj.Name() + "FilterArg" } else { - targetType = obj.Fields()[target].Type.Name() + filterType := obj.Fields()[target].Type + if list, isList := filterType.(*gql.List); isList && gql.IsLeafType(list.OfType) { + // If it is a list of leaf types - the filter is just the set of OperatorBlocks + // that are supported by this type - there can be no field selections. + if notNull, isNotNull := list.OfType.(*gql.NonNull); isNotNull { + // GQL does not support '!' in type names, and so we have to manipulate the + // underlying name like this if it is a nullable type. + filterTypeName = fmt.Sprintf("NotNull%sOperatorBlock", notNull.OfType.Name()) + } else { + filterTypeName = genTypeName(list.OfType, "OperatorBlock") + } + } else { + filterTypeName = filterType.Name() + "FilterArg" + } } expandedField := &gql.InputObjectFieldConfig{ - Type: g.manager.schema.TypeMap()[targetType+"FilterArg"], + Type: g.manager.schema.TypeMap()[filterTypeName], } aggregateTarget.Type.(*gql.InputObject).AddFieldConfig("filter", expandedField) } diff --git a/query/graphql/schema/manager.go b/query/graphql/schema/manager.go index f0d35e16a2..a9911b0777 100644 --- a/query/graphql/schema/manager.go +++ b/query/graphql/schema/manager.go @@ -146,11 +146,15 @@ func defaultTypes() []gql.Type { // Filter scalar blocks booleanOperatorBlock, + notNullBooleanOperatorBlock, dateTimeOperatorBlock, floatOperatorBlock, + notNullFloatOperatorBlock, idOperatorBlock, intOperatorBlock, + notNullIntOperatorBlock, stringOperatorBlock, + notNullstringOperatorBlock, schemaTypes.CommitLinkObject, schemaTypes.CommitObject, diff --git a/query/graphql/schema/root.go b/query/graphql/schema/root.go index e1d44a32e8..fc844f4722 100644 --- a/query/graphql/schema/root.go +++ b/query/graphql/schema/root.go @@ -49,6 +49,25 @@ var booleanOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ }, }) +// notNullBooleanOperatorBlock filter block for boolean! types. +var notNullBooleanOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ + Name: "NotNullBooleanOperatorBlock", + Fields: gql.InputObjectConfigFieldMap{ + "_eq": &gql.InputObjectFieldConfig{ + Type: gql.Boolean, + }, + "_ne": &gql.InputObjectFieldConfig{ + Type: gql.Boolean, + }, + "_in": &gql.InputObjectFieldConfig{ + Type: gql.NewList(gql.NewNonNull(gql.Boolean)), + }, + "_nin": &gql.InputObjectFieldConfig{ + Type: gql.NewList(gql.NewNonNull(gql.Boolean)), + }, + }, +}) + // dateTimeOperatorBlock filter block for DateTime types. var dateTimeOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ Name: "DateTimeOperatorBlock", @@ -111,6 +130,37 @@ var floatOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ }, }) +// notNullFloatOperatorBlock filter block for Float! types. +var notNullFloatOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ + Name: "NotNullFloatOperatorBlock", + Fields: gql.InputObjectConfigFieldMap{ + "_eq": &gql.InputObjectFieldConfig{ + Type: gql.Float, + }, + "_ne": &gql.InputObjectFieldConfig{ + Type: gql.Float, + }, + "_gt": &gql.InputObjectFieldConfig{ + Type: gql.Float, + }, + "_ge": &gql.InputObjectFieldConfig{ + Type: gql.Float, + }, + "_lt": &gql.InputObjectFieldConfig{ + Type: gql.Float, + }, + "_le": &gql.InputObjectFieldConfig{ + Type: gql.Float, + }, + "_in": &gql.InputObjectFieldConfig{ + Type: gql.NewList(gql.NewNonNull(gql.Float)), + }, + "_nin": &gql.InputObjectFieldConfig{ + Type: gql.NewList(gql.NewNonNull(gql.Float)), + }, + }, +}) + // intOperatorBlock filter block for Int types. var intOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ Name: "IntOperatorBlock", @@ -142,6 +192,37 @@ var intOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ }, }) +// notNullIntOperatorBlock filter block for Int! types. +var notNullIntOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ + Name: "NotNullIntOperatorBlock", + Fields: gql.InputObjectConfigFieldMap{ + "_eq": &gql.InputObjectFieldConfig{ + Type: gql.Int, + }, + "_ne": &gql.InputObjectFieldConfig{ + Type: gql.Int, + }, + "_gt": &gql.InputObjectFieldConfig{ + Type: gql.Int, + }, + "_ge": &gql.InputObjectFieldConfig{ + Type: gql.Int, + }, + "_lt": &gql.InputObjectFieldConfig{ + Type: gql.Int, + }, + "_le": &gql.InputObjectFieldConfig{ + Type: gql.Int, + }, + "_in": &gql.InputObjectFieldConfig{ + Type: gql.NewList(gql.NewNonNull(gql.Int)), + }, + "_nin": &gql.InputObjectFieldConfig{ + Type: gql.NewList(gql.NewNonNull(gql.Int)), + }, + }, +}) + // stringOperatorBlock filter block for string types. var stringOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ Name: "StringOperatorBlock", @@ -164,6 +245,25 @@ var stringOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ }, }) +// notNullstringOperatorBlock filter block for string! types. +var notNullstringOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ + Name: "NotNullStringOperatorBlock", + Fields: gql.InputObjectConfigFieldMap{ + "_eq": &gql.InputObjectFieldConfig{ + Type: gql.String, + }, + "_ne": &gql.InputObjectFieldConfig{ + Type: gql.String, + }, + "_in": &gql.InputObjectFieldConfig{ + Type: gql.NewList(gql.NewNonNull(gql.String)), + }, + "_nin": &gql.InputObjectFieldConfig{ + Type: gql.NewList(gql.NewNonNull(gql.String)), + }, + }, +}) + // idOperatorBlock filter block for ID types. var idOperatorBlock = gql.NewInputObject(gql.InputObjectConfig{ Name: "IDOperatorBlock", diff --git a/tests/integration/query/inline_array/with_average_filter_test.go b/tests/integration/query/inline_array/with_average_filter_test.go new file mode 100644 index 0000000000..52a7c6176b --- /dev/null +++ b/tests/integration/query/inline_array/with_average_filter_test.go @@ -0,0 +1,71 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package inline_array + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestQueryInlineIntegerArrayWithsWithAverageWithFilter(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple inline array, filtered average of integer array", + Query: `query { + users { + Name + _avg(FavouriteIntegers: {filter: {_gt: 0}}) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "Shahzad", + "FavouriteIntegers": [-1, 2, -1, 1, 0] + }`)}, + }, + Results: []map[string]interface{}{ + { + "Name": "Shahzad", + "_avg": float64(1.5), + }, + }, + } + + executeTestCase(t, test) +} + +func TestQueryInlineFloatArrayWithsWithAverageWithFilter(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple inline array, filtered average of float array", + Query: `query { + users { + Name + _avg(FavouriteFloats: {filter: {_lt: 9}}) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "Shahzad", + "FavouriteFloats": [3.4, 3.6, 10] + }`)}, + }, + Results: []map[string]interface{}{ + { + "Name": "Shahzad", + "_avg": 3.5, + }, + }, + } + + executeTestCase(t, test) +} diff --git a/tests/integration/query/inline_array/with_count_filter_test.go b/tests/integration/query/inline_array/with_count_filter_test.go new file mode 100644 index 0000000000..2b2b5f2d55 --- /dev/null +++ b/tests/integration/query/inline_array/with_count_filter_test.go @@ -0,0 +1,125 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package inline_array + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestQueryInlineBoolArrayWithsWithCountWithFilter(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple inline array, filtered count of bool array", + Query: `query { + users { + Name + _count(LikedIndexes: {filter: {_eq: true}}) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "Shahzad", + "LikedIndexes": [true, true, false, true] + }`)}, + }, + Results: []map[string]interface{}{ + { + "Name": "Shahzad", + "_count": 3, + }, + }, + } + + executeTestCase(t, test) +} + +func TestQueryInlineIntegerArrayWithsWithCountWithFilter(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple inline array, filtered count of integer array", + Query: `query { + users { + Name + _count(FavouriteIntegers: {filter: {_gt: 0}}) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "Shahzad", + "FavouriteIntegers": [-1, 2, -1, 1, 0] + }`)}, + }, + Results: []map[string]interface{}{ + { + "Name": "Shahzad", + "_count": 2, + }, + }, + } + + executeTestCase(t, test) +} + +func TestQueryInlineFloatArrayWithsWithCountWithFilter(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple inline array, filtered count of float array", + Query: `query { + users { + Name + _count(FavouriteFloats: {filter: {_lt: 9}}) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "Shahzad", + "FavouriteFloats": [3.1425, 0.00000000001, 10] + }`)}, + }, + Results: []map[string]interface{}{ + { + "Name": "Shahzad", + "_count": 2, + }, + }, + } + + executeTestCase(t, test) +} + +func TestQueryInlineStringArrayWithsWithCountWithFilter(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple inline array, filtered count of string array", + Query: `query { + users { + Name + _count(PreferredStrings: {filter: {_in: ["", "the first"]}}) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "Shahzad", + "PreferredStrings": ["", "the previous", "the first", "empty string"] + }`)}, + }, + Results: []map[string]interface{}{ + { + "Name": "Shahzad", + "_count": 2, + }, + }, + } + + executeTestCase(t, test) +} diff --git a/tests/integration/query/inline_array/with_sum_filter_test.go b/tests/integration/query/inline_array/with_sum_filter_test.go new file mode 100644 index 0000000000..98d1efd867 --- /dev/null +++ b/tests/integration/query/inline_array/with_sum_filter_test.go @@ -0,0 +1,71 @@ +// Copyright 2022 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package inline_array + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestQueryInlineIntegerArrayWithsWithSumWithFilter(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple inline array, filtered sum of integer array", + Query: `query { + users { + Name + _sum(FavouriteIntegers: {filter: {_gt: 0}}) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "Shahzad", + "FavouriteIntegers": [-1, 2, -1, 1, 0] + }`)}, + }, + Results: []map[string]interface{}{ + { + "Name": "Shahzad", + "_sum": int64(3), + }, + }, + } + + executeTestCase(t, test) +} + +func TestQueryInlineFloatArrayWithsWithSumWithFilter(t *testing.T) { + test := testUtils.QueryTestCase{ + Description: "Simple inline array, filtered sum of float array", + Query: `query { + users { + Name + _sum(FavouriteFloats: {filter: {_lt: 9}}) + } + }`, + Docs: map[int][]string{ + 0: { + (`{ + "Name": "Shahzad", + "FavouriteFloats": [3.1425, 0.00000000001, 10] + }`)}, + }, + Results: []map[string]interface{}{ + { + "Name": "Shahzad", + "_sum": 3.14250000001, + }, + }, + } + + executeTestCase(t, test) +}