Add LazyJSONDocument, which wraps a JSON string and only deserializes it if needed.

enginetest/evaluation.go

@@ -819,7 +819,10 @@ func widenJSONValues(val interface{}) sql.JSONWrapper {
 		js = types.MustJSON(str)
 	}
 
-	doc := js.ToInterface()
+	doc, err := js.ToInterface()
+	if err != nil {
+		panic(err)
+	}
 
 	if _, ok := js.(sql.Statistic); ok {
 		// avoid comparing time values in statistics

sql/expression/function/aggregation/json_agg.go

@@ -159,7 +159,10 @@ func (j *jsonObjectBuffer) Update(ctx *sql.Context, row sql.Row) error {
 
 	// unwrap JSON values
 	if js, ok := val.(sql.JSONWrapper); ok {
-		val = js.ToInterface()
+		val, err = js.ToInterface()
+		if err != nil {
+			return err
+		}
 	}
 
 	// Update the map.

sql/expression/function/aggregation/unary_agg_buffers.go

@@ -647,7 +647,10 @@ func (j *jsonArrayBuffer) Update(ctx *sql.Context, row sql.Row) error {
 
 	// unwrap JSON values
 	if js, ok := v.(sql.JSONWrapper); ok {
-		v = js.ToInterface()
+		v, err = js.ToInterface()
+		if err != nil {
+			return err
+		}
 	}
 
 	j.vals = append(j.vals, v)

sql/expression/function/aggregation/window_functions.go

@@ -1050,7 +1050,10 @@ func (a *WindowedJSONArrayAgg) aggregateVals(ctx *sql.Context, interval sql.Wind
 
 		// unwrap JSON values
 		if js, ok := v.(sql.JSONWrapper); ok {
-			v = js.ToInterface()
+			v, err = js.ToInterface()
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		vals = append(vals, v)
@@ -1134,7 +1137,10 @@ func (a *WindowedJSONObjectAgg) aggregateVals(ctx *sql.Context, interval sql.Win
 
 		// unwrap JSON values
 		if js, ok := val.(sql.JSONWrapper); ok {
-			val = js.ToInterface()
+			val, err = js.ToInterface()
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		// Update the map.

sql/expression/function/json/json_array.go

@@ -113,7 +113,10 @@ func (j *JSONArray) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
 
 		switch v := val.(type) {
 		case sql.JSONWrapper:
-			val = v.ToInterface()
+			val, err = v.ToInterface()
+			if err != nil {
+				return nil, err
+			}
 		case []byte:
 			val = string(v)
 		}

sql/expression/function/json/json_common.go

@@ -71,7 +71,11 @@ func getJSONDocumentFromRow(ctx *sql.Context, row sql.Row, json sql.Expression)
 	doc, ok := converted.(types.JSONDocument)
 	if !ok {
 		// This should never happen, but just in case.
-		doc = types.JSONDocument{Val: js.(sql.JSONWrapper).ToInterface()}
+		val, err := js.(sql.JSONWrapper).ToInterface()
+		if err != nil {
+			return nil, err
+		}
+		doc = types.JSONDocument{Val: val}
 	}
 
 	return &doc, nil

sql/expression/function/json/json_contains.go

@@ -162,7 +162,15 @@ func (j *JSONContains) Eval(ctx *sql.Context, row sql.Row) (interface{}, error)
 	}
 
 	// Now determine whether the candidate value exists in the target
-	return types.ContainsJSON(target.ToInterface(), candidate.ToInterface())
+	targetVal, err := target.ToInterface()
+	if err != nil {
+		return nil, err
+	}
+	candidateVal, err := candidate.ToInterface()
+	if err != nil {
+		return nil, err
+	}
+	return types.ContainsJSON(targetVal, candidateVal)
 }
 
 func (j *JSONContains) Children() []sql.Expression {

sql/expression/function/json/json_object.go

@@ -109,7 +109,10 @@ func (j JSONObject) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
 			key = val.(string)
 		} else {
 			if json, ok := val.(sql.JSONWrapper); ok {
-				val = json.ToInterface()
+				val, err = json.ToInterface()
+				if err != nil {
+					return nil, err
+				}
 			}
 			obj[key] = val
 		}

sql/expression/function/json/json_value.go

@@ -182,7 +182,7 @@ func GetJSONFromWrapperOrCoercibleString(js interface{}) (jsonData interface{},
 		}
 		return jsonData, nil
 	case sql.JSONWrapper:
-		return jsType.ToInterface(), nil
+		return jsType.ToInterface()
 	default:
 		return nil, InvalidJsonArgument.New()
 	}

sql/fulltext/fulltext.go

@@ -175,6 +175,14 @@ func writeHashedValue(h hash.Hash, val interface{}) (valIsNull bool, err error)
 		if _, err := h.Write([]byte(str)); err != nil {
 			return false, err
 		}
+	case *types.LazyJSONDocument:
+		str, err := types.StringifyJSON(val)
+		if err != nil {
+			return false, err
+		}
+		if _, err := h.Write([]byte(str)); err != nil {
+			return false, err
+		}
 	case nil:
 		return true, nil
 	default:

sql/plan/histogram.go

@@ -1,10 +1,11 @@
 package plan
 
 import (
-	"encoding/json"
 	"fmt"
 	"strings"
 
+	"github.com/dolthub/go-mysql-server/sql/types"
+
 	"github.com/dolthub/go-mysql-server/sql"
 )
 
@@ -58,8 +59,7 @@ func (u *UpdateHistogram) Resolved() bool {
 }
 
 func (u *UpdateHistogram) String() string {
-	statMap := u.stats.ToInterface()
-	statBytes, _ := json.Marshal(statMap)
+	statBytes, _ := types.MarshallJson(u.stats)
 	return fmt.Sprintf("update histogram  %s.(%s) using %s", u.table, strings.Join(u.cols, ","), statBytes)
 }
 

sql/statistics.go

@@ -142,7 +142,7 @@ func (h Histogram) IsEmpty() bool {
 	return len(h) == 0
 }
 
-func (h Histogram) ToInterface() interface{} {
+func (h Histogram) ToInterface() (interface{}, error) {
 	ret := make([]interface{}, len(h))
 	for i, b := range h {
 		var upperBound Row
@@ -167,7 +167,7 @@ func (h Histogram) ToInterface() interface{} {
 			"upper_bound":    upperBound,
 		}
 	}
-	return ret
+	return ret, nil
 }
 
 func (h Histogram) DebugString() string {
@@ -216,5 +216,5 @@ type HistogramBucket interface {
 // by minimizing the need to unmarshall a JSONWrapper into a JSONDocument.
 type JSONWrapper interface {
 	// ToInterface converts a JSONWrapper to an interface{} of simple types
-	ToInterface() interface{}
+	ToInterface() (interface{}, error)
 }

sql/stats/statistic.go

@@ -209,12 +209,17 @@ func (s *Statistic) IndexClass() sql.IndexClass {
 	return sql.IndexClass(s.IdxClass)
 }
 
-func (s *Statistic) ToInterface() interface{} {
+func (s *Statistic) ToInterface() (interface{}, error) {
 	typs := make([]string, len(s.Typs))
 	for i, t := range s.Typs {
 		typs[i] = t.String()
 	}
 
+	buckets, err := s.Histogram().ToInterface()
+	if err != nil {
+		return nil, err
+	}
+
 	return map[string]interface{}{
 		"statistic": map[string]interface{}{
 			"row_count":      s.RowCount(),
@@ -225,9 +230,9 @@ func (s *Statistic) ToInterface() interface{} {
 			"qualifier":      s.Qualifier().String(),
 			"columns":        s.Columns(),
 			"types:":         typs,
-			"buckets":        s.Histogram().ToInterface(),
+			"buckets":        buckets,
 		},
-	}
+	}, nil
 }
 
 func ParseTypeStrings(typs []string) ([]sql.Type, error) {

sql/types/json.go

@@ -15,7 +15,6 @@
 package types
 
 import (
-	"bytes"
 	"encoding/json"
 	"reflect"
 
@@ -131,30 +130,30 @@ func (t JsonType) SQL(ctx *sql.Context, dest []byte, v interface{}) (sqltypes.Va
 		return sqltypes.NULL, nil
 	}
 
-	// Convert to jsonType
-	jsVal, _, err := t.Convert(v)
-	if err != nil {
-		return sqltypes.NULL, err
-	}
-	js := jsVal.(sql.JSONWrapper)
-
 	var val []byte
-	switch j := js.(type) {
-	case JSONStringer:
-		str, err := j.JSONString()
+
+	// If we read the JSON from a table, pass through the bytes to avoid a deserialization and reserialization round-trip.
+	// This is kind of a hack, and it means that reading JSON from tables no longer matches MySQL byte-for-byte.
+	// But its worth it to avoid the round-trip, which can be very slow.
+	if j, ok := v.(*LazyJSONDocument); ok {
+		str, err := MarshallJson(j)
 		if err != nil {
 			return sqltypes.NULL, err
 		}
-		val = AppendAndSliceString(dest, str)
-	default:
-		jsonBytes, err := json.Marshal(js.ToInterface())
+		val = AppendAndSliceBytes(dest, str)
+	} else {
+		// Convert to jsonType
+		jsVal, _, err := t.Convert(v)
 		if err != nil {
 			return sqltypes.NULL, err
 		}
+		js := jsVal.(sql.JSONWrapper)
 
-		jsonBytes = bytes.ReplaceAll(jsonBytes, []byte(",\""), []byte(", \""))
-		jsonBytes = bytes.ReplaceAll(jsonBytes, []byte("\":"), []byte("\": "))
-		val = AppendAndSliceBytes(dest, jsonBytes)
+		str, err := StringifyJSON(js)
+		if err != nil {
+			return sqltypes.NULL, err
+		}
+		val = AppendAndSliceString(dest, str)
 	}
 
 	return sqltypes.MakeTrusted(sqltypes.TypeJSON, val), nil

sql/types/json_test.go

@@ -212,6 +212,29 @@ func TestValuer(t *testing.T) {
 	require.Equal(t, `{"a": "one"}`, res)
 }
 
+func TestLazyJsonDocument(t *testing.T) {
+	testCases := []struct {
+		s    string
+		json interface{}
+	}{
+		{`"1"`, "1"},
+		{`{"a": [1.0, null]}`, map[string]any{"a": []any{1.0, nil}}},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.s, func(t *testing.T) {
+			doc := NewLazyJSONDocument([]byte(testCase.s))
+			val, err := doc.ToInterface()
+			require.NoError(t, err)
+			require.Equal(t, testCase.json, val)
+		})
+	}
+	t.Run("lazy docs only error when deserialized", func(t *testing.T) {
+		doc := NewLazyJSONDocument([]byte("not valid json"))
+		_, err := doc.ToInterface()
+		require.Error(t, err)
+	})
+}
+
 type JsonRoundtripTest struct {
 	desc     string
 	input    string