feat: Enable secondary index for compound filter conditions

docs/data_format_changes/i3417-sec-index-compound-filter-conditions.md

@@ -0,0 +1,3 @@
+# Enabled compound filter conditions for secondary indexes
+
+Changed the way fieldFetched are counted for explain directive

internal/connor/all.go

@@ -43,7 +43,7 @@ func all(condition, data any) (bool, error) {
 
 func allSlice[T any](condition any, data []T) (bool, error) {
 	for _, c := range data {
-		m, err := eq(condition, c)
+		m, err := eq(condition, c, true)
 		if err != nil {
 			return false, err
 		} else if !m {

internal/connor/and.go

@@ -8,7 +8,7 @@ func and(condition, data any) (bool, error) {
 	switch cn := condition.(type) {
 	case []any:
 		for _, c := range cn {
-			if m, err := eq(c, data); err != nil {
+			if m, err := eq(c, data, true); err != nil {
 				return false, err
 			} else if !m {
 				return false, nil

internal/connor/any.go

@@ -43,7 +43,7 @@ func anyOp(condition, data any) (bool, error) {
 
 func anySlice[T any](condition any, data []T) (bool, error) {
 	for _, c := range data {
-		m, err := eq(condition, c)
+		m, err := eq(condition, c, true)
 		if err != nil {
 			return false, err
 		} else if m {

internal/connor/connor.go

@@ -49,13 +49,13 @@ func IsOpSimple(op string) bool {
 // Match is the default method used in Connor to match some data to a
 // set of conditions.
 func Match(conditions map[FilterKey]any, data any) (bool, error) {
-	return eq(conditions, data)
+	return eq(conditions, data, true)
 }
 
 // matchWith can be used to specify the exact operator to use when performing
 // a match operation. This is primarily used when building custom operators or
 // if you wish to override the behavior of another operator.
-func matchWith(op string, conditions, data any) (bool, error) {
+func matchWith(op string, conditions, data any, propExists bool) (bool, error) {
 	switch op {
 	case AndOp:
 		return and(conditions, data)
@@ -64,7 +64,7 @@ func matchWith(op string, conditions, data any) (bool, error) {
 	case AllOp:
 		return all(conditions, data)
 	case EqualOp, AliasOp:
-		return eq(conditions, data)
+		return eq(conditions, data, propExists)
 	case GreaterOrEqualOp:
 		return ge(conditions, data)
 	case GreaterOp:
@@ -76,7 +76,7 @@ func matchWith(op string, conditions, data any) (bool, error) {
 	case LesserOp:
 		return lt(conditions, data)
 	case NotEqualOp:
-		return ne(conditions, data)
+		return ne(conditions, data, propExists)
 	case NotInOp:
 		return nin(conditions, data)
 	case OrOp:

internal/connor/eq.go

@@ -13,7 +13,7 @@ import (
 
 // eq is an operator which performs object equality
 // tests.
-func eq(condition, data any) (bool, error) {
+func eq(condition, data any, propExists bool) (bool, error) {
 	switch arr := data.(type) {
 	case []core.Doc:
 		return anySlice(condition, arr)
@@ -34,11 +34,11 @@ func eq(condition, data any) (bool, error) {
 	switch cn := condition.(type) {
 	case map[FilterKey]any:
 		for prop, cond := range cn {
-			d, op, err := prop.PropertyAndOperator(data, EqualOp)
-			if err != nil {
-				return false, err
+			res := prop.PropertyAndOperator(data, EqualOp)
+			if res.Err != nil {
+				return false, res.Err
 			}
-			m, err := matchWith(op, cond, d)
+			m, err := matchWith(res.Operator, cond, res.Data, !res.MissProp && propExists)
 			if err != nil {
 				return false, err
 			}
@@ -85,7 +85,7 @@ func objectsEqual(condition map[string]any, data any) (bool, error) {
 		return false, nil
 	}
 	for k, v := range d {
-		m, err := eq(condition[k], v)
+		m, err := eq(condition[k], v, true)
 		if err != nil {
 			return false, err
 		} else if !m {

internal/connor/in.go

@@ -8,7 +8,7 @@ func in(conditions, data any) (bool, error) {
 	switch cn := conditions.(type) {
 	case []any:
 		for _, ce := range cn {
-			if m, err := eq(ce, data); err != nil {
+			if m, err := eq(ce, data, true); err != nil {
 				return false, err
 			} else if m {
 				return true, nil

internal/connor/key.go

@@ -1,13 +1,25 @@
 package connor
 
+// KeyResult represents the result of a filter key operation.
+type KeyResult struct {
+	// Data is the data that should be used to filter the value matching the key.
+	Data any
+	// MissProp is true if the key is missing a property, otherwise false.
+	// It's relevant for object of dynamic type, like JSON.
+	MissProp bool
+	// Operator is the operator that should be used to filter the value matching the key.
+	// If the key does not have an operator the given defaultOp will be returned.
+	Operator string
+	// Err is the error that occurred while filtering the value matching the key.
+	Err error
+}
+
 // FilterKey represents a type that may be used as a map key
 // in a filter.
 type FilterKey interface {
-	// PropertyAndOperator returns the data and operator that should be used
-	// to filter the value matching this key.
-	//
-	// If the key does not have an operator the given defaultOp will be returned.
-	PropertyAndOperator(data any, defaultOp string) (any, string, error)
+	// PropertyAndOperator returns [KeyResult] that contains data and operator that should be
+	// used to filter the value matching this key.
+	PropertyAndOperator(data any, defaultOp string) KeyResult
 	// Equal returns true if other is equal, otherwise returns false.
 	Equal(other FilterKey) bool
 }

internal/connor/ne.go

@@ -2,8 +2,13 @@ package connor
 
 // ne performs object inequality comparisons by inverting
 // the result of the EqualOperator for non-error cases.
-func ne(conditions, data any) (bool, error) {
-	m, err := eq(conditions, data)
+func ne(conditions, data any, propExists bool) (bool, error) {
+	// _ne operator should return false if the property does not exist.
+	if !propExists {
+		return false, nil
+	}
+
+	m, err := eq(conditions, data, propExists)
 
 	if err != nil {
 		return false, err

internal/connor/none.go

@@ -41,7 +41,7 @@ func none(condition, data any) (bool, error) {
 
 func noneSlice[T any](condition any, data []T) (bool, error) {
 	for _, c := range data {
-		m, err := eq(condition, c)
+		m, err := eq(condition, c, true)
 		if err != nil {
 			return false, err
 		} else if m {

internal/connor/not.go

@@ -3,7 +3,7 @@ package connor
 // not is an operator which performs object equality test
 // and returns the inverse of the result.
 func not(condition, data any) (bool, error) {
-	m, err := eq(condition, data)
+	m, err := eq(condition, data, true)
 	if err != nil {
 		return false, err
 	}

internal/connor/not_test.go

@@ -34,8 +34,8 @@ type operator struct {
 	Operation string
 }
 
-func (k *operator) PropertyAndOperator(data any, defaultOp string) (any, string, error) {
-	return data, k.Operation, nil
+func (k *operator) PropertyAndOperator(data any, defaultOp string) KeyResult {
+	return KeyResult{Data: data, Operator: k.Operation}
 }
 
 func (k *operator) Equal(other FilterKey) bool {

internal/connor/or.go

@@ -8,7 +8,7 @@ func or(condition, data any) (bool, error) {
 	switch cn := condition.(type) {
 	case []any:
 		for _, c := range cn {
-			if m, err := eq(c, data); err != nil {
+			if m, err := eq(c, data, true); err != nil {
 				return false, err
 			} else if m {
 				return true, nil

internal/db/collection.go

@@ -71,7 +71,7 @@ func (c *collection) newFetcher() fetcher.Fetcher {
 	if c.fetcherFactory != nil {
 		innerFetcher = c.fetcherFactory()
 	} else {
-		innerFetcher = fetcher.NewDocumentFetcher()
+		innerFetcher = fetcher.NewDocumentFetcher(immutable.Option[client.IndexDescription]{})
 	}
 
 	return lens.NewFetcher(innerFetcher, c.db.LensRegistry())

internal/db/fetcher/document.go

@@ -135,6 +135,9 @@
 
 	for {
 		res, ok := f.kvResultsIter.NextSync()
+		if res.Error != nil {
+			return immutable.None[EncodedDocument](), res.Error
+		}
 		if !ok {
 			break
 		}
@@ -179,13 +182,16 @@
 		return err
 	}
 
+	// we count the fields fetched here instead of after checking if the field was requested
+	// because we need to count all fields fetched to see more accurate picture of the performance
+	// of the query
+	f.execInfo.FieldsFetched++
+
 	fieldDesc, ok := f.fieldsByID[fieldID]
 	if !ok {
 		return nil
 	}
 
-	f.execInfo.FieldsFetched++
-
 	doc.properties[fieldDesc] = &encProperty{
 		Desc: fieldDesc,
 		Raw:  kv.Value,