Emit a warning if the scanner encounters json indexing metadata writt…

…en by a newer version of Dolt that it can't read.

go/store/prolly/tree/json_cursor.go

@@ -36,10 +36,16 @@ func getPreviousKey(ctx context.Context, cur *cursor) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	if cur2.Valid() {
-		return cur2.parent.CurrentKey(), nil
+	// If we're at the start of the tree, return nil.
+	if !cur2.Valid() {
+		return nil, nil
 	}
-	return nil, nil
+	key := cur2.parent.CurrentKey()
+	err = errorIfNotSupportedLocation(key)
+	if err != nil {
+		return nil, err
+	}
+	return key, nil
 }
 
 // newJsonCursor takes the root node of a prolly tree representing a JSON document, and creates a new JsonCursor for reading

go/store/prolly/tree/json_indexed_document.go

@@ -119,6 +119,22 @@ func (i IndexedJsonDocument) Lookup(ctx context.Context, pathString string) (sql
 		nonIndexedDoc := types.JSONDocument{Val: val}
 		return nonIndexedDoc.Lookup(ctx, pathString)
 	}
+	result, err := i.tryLookup(ctx, pathString)
+	if err == unknownLocationKeyError {
+		if sqlCtx, ok := ctx.(*sql.Context); ok {
+			sqlCtx.GetLogger().Warn(err)
+		}
+		v, err := i.ToInterface()
+		if err != nil {
+			return nil, err
+		}
+		return types.JSONDocument{Val: v}.Lookup(ctx, pathString)
+	}
+	return result, err
+}
+
+func (i IndexedJsonDocument) tryLookup(ctx context.Context, pathString string) (sql.JSONWrapper, error) {
+
 	path, err := jsonPathElementsFromMySQLJsonPath([]byte(pathString))
 	if err != nil {
 		return nil, err
@@ -145,6 +161,21 @@ func (i IndexedJsonDocument) Lookup(ctx context.Context, pathString string) (sql
 
 // Insert implements types.MutableJSON
 func (i IndexedJsonDocument) Insert(ctx context.Context, path string, val sql.JSONWrapper) (types.MutableJSON, bool, error) {
+	result, changed, err := i.tryInsert(ctx, path, val)
+	if err == unknownLocationKeyError {
+		if sqlCtx, ok := ctx.(*sql.Context); ok {
+			sqlCtx.GetLogger().Warn(err)
+		}
+		v, err := i.ToInterface()
+		if err != nil {
+			return nil, false, err
+		}
+		return types.JSONDocument{Val: v}.Insert(ctx, path, val)
+	}
+	return result, changed, err
+}
+
+func (i IndexedJsonDocument) tryInsert(ctx context.Context, path string, val sql.JSONWrapper) (types.MutableJSON, bool, error) {
 	keyPath, err := jsonPathElementsFromMySQLJsonPath([]byte(path))
 	if err != nil {
 		return nil, false, err

go/store/prolly/tree/json_location.go

@@ -41,6 +41,9 @@ import (
 //	             - 2 / arrayInitialElement  - The location points to where a new value would be inserted at the start of an array.
 //	                                          This is always one byte past the start of the array.
 //	             - 3 / startOfValue         - The location points one byte past the end of the value.
+//	        In the event this format changes, the upper bits can be used to indicate the version. Currently, if these bits are not
+//	        all 0, the engine should emit a warning, and then fall back on a naive implementation of the current operation that
+//	        does not require these keys.
 //
 //				The remainder of |key| is a sequence of encoded path elements, each of which is either an object key or array index:
 //				    <path-element> ::= <object-key> | <array-index>
@@ -68,6 +71,8 @@ const (
 	endOfValue
 )
 
+var unknownLocationKeyError = fmt.Errorf("A JSON document was written with a future version of Dolt, and the index metadata cannot be read. This will impact performance for large documents.")
+
 const (
 	beginObjectKey byte = 0xFF
 	beginArrayKey  byte = 0xFE
@@ -148,6 +153,13 @@ func isValidJsonPathKey(key []byte) bool {
 	return true
 }
 
+func errorIfNotSupportedLocation(key []byte) error {
+	if jsonPathType(key[0]) > endOfValue {
+		return unknownLocationKeyError
+	}
+	return nil
+}
+
 type lexState int
 
 const (

go/store/prolly/tree/json_scanner.go

@@ -19,9 +19,6 @@ import (
 	"io"
 )
 
-// TODO: JsonScanner currently assumes that there are no escaped characters in strings. This can be fixed by escaping/unescaping
-// strings as they are read from the document, before they are used in any comparisons.
-
 // JsonScanner is a state machine that parses already-normalized JSON while keeping track of the path to the current value.
 // It is not a general-purpose JSON parser. In particular, it makes the following assumptions about the JSON:
 // - All whitespace has been removed