settings: add functionality for settings override

This commit adds override functionality to the `SettingsWatcher`.

We define an `OverrideMonitor` interface that is used to discover the
overrides and listen for updates. The logic in `SettingsWatcher` is
updated to return the override whenever there is one set, otherwise
return the current value as normal.

On non-system tenants, an `OverrideMonitor` implementation will use
new tenant connector APIs to allow the host cluster to override
settings (see the separate RFC).

Release note: None

pkg/server/settingswatcher/BUILD.bazel

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "settingswatcher",
     srcs = [
+        "overrides.go",
         "row_decoder.go",
         "settings_watcher.go",
     ],
@@ -28,6 +29,7 @@ go_library(
         "//pkg/util/log",
         "//pkg/util/protoutil",
         "//pkg/util/stop",
+        "//pkg/util/syncutil",
         "@com_github_cockroachdb_errors//:errors",
     ],
 )
@@ -43,6 +45,7 @@ go_test(
         ":settingswatcher",
         "//pkg/base",
         "//pkg/keys",
+        "//pkg/kv/kvclient/rangefeed:with-mocks",
         "//pkg/roachpb:with-mocks",
         "//pkg/security",
         "//pkg/security/securitytest",
@@ -56,6 +59,7 @@ go_test(
         "//pkg/testutils/testcluster",
         "//pkg/util/hlc",
         "//pkg/util/leaktest",
+        "//pkg/util/syncutil",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_stretchr_testify//require",
     ],

pkg/server/settingswatcher/overrides.go

@@ -0,0 +1,28 @@
+// Copyright 2021 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package settingswatcher
+
+// OverridesMonitor is an interface through which the settings watcher can
+// receive setting overrides. Used for non-system tenants.
+//
+// The expected usage is to listen for a message on NotifyCh(), and use
+// Current() to retrieve the updated list of overrides when a message is
+// received.
+type OverridesMonitor interface {
+	// NotifyCh returns a channel that receives a message any time the current set
+	// of overrides changes.
+	NotifyCh() <-chan struct{}
+
+	// Overrides retrieves the current set of setting overrides, as a map from
+	// setting key to RawValue. Any settings that are present must be set to the
+	// overridden value.
+	Overrides() map[string]RawValue
+}

pkg/server/settingswatcher/row_decoder.go

@@ -31,6 +31,13 @@ type RowDecoder struct {
 	colIdxMap catalog.TableColMap
 }
 
+// RawValue contains a raw-value / value-type pair, corresponding to the value
+// and valueType columns of the settings table.
+type RawValue struct {
+	Value string
+	Type  string
+}
+
 // MakeRowDecoder makes a new RowDecoder for the settings table.
 func MakeRowDecoder(codec keys.SQLCodec) RowDecoder {
 	return RowDecoder{
@@ -42,69 +49,69 @@ func MakeRowDecoder(codec keys.SQLCodec) RowDecoder {
 }
 
 // DecodeRow decodes a row of the system.settings table. If the value is not
-// present, the setting key will be returned but the other two fields will be
-// zero and the tombstone bool will be set.
+// present, the setting key will be returned but the value will be zero and the
+// tombstone bool will be set.
 func (d *RowDecoder) DecodeRow(
 	kv roachpb.KeyValue,
-) (setting, val, valType string, tombstone bool, _ error) {
+) (setting string, val RawValue, tombstone bool, _ error) {
 	tbl := systemschema.SettingsTable
 	// First we need to decode the setting name field from the index key.
 	{
 		types := []*types.T{tbl.PublicColumns()[0].GetType()}
 		nameRow := make([]rowenc.EncDatum, 1)
 		_, matches, _, err := rowenc.DecodeIndexKey(d.codec, types, nameRow, nil, kv.Key)
 		if err != nil {
-			return "", "", "", false, errors.Wrap(err, "failed to decode key")
+			return "", RawValue{}, false, errors.Wrap(err, "failed to decode key")
 		}
 		if !matches {
-			return "", "", "", false, errors.Errorf("unexpected non-settings KV with settings prefix: %v", kv.Key)
+			return "", RawValue{}, false, errors.Errorf("unexpected non-settings KV with settings prefix: %v", kv.Key)
 		}
 		if err := nameRow[0].EnsureDecoded(types[0], &d.alloc); err != nil {
-			return "", "", "", false, err
+			return "", RawValue{}, false, err
 		}
 		setting = string(tree.MustBeDString(nameRow[0].Datum))
 	}
 	if !kv.Value.IsPresent() {
-		return setting, "", "", true, nil
+		return setting, RawValue{}, true, nil
 	}
 
 	// The rest of the columns are stored as a family, packed with diff-encoded
 	// column IDs followed by their values.
 	{
 		// column valueType can be null (missing) so we default it to "s".
-		valType = "s"
+		val.Type = "s"
 		bytes, err := kv.Value.GetTuple()
 		if err != nil {
-			return "", "", "", false, err
+			return "", RawValue{}, false, err
 		}
 		var colIDDiff uint32
 		var lastColID descpb.ColumnID
 		var res tree.Datum
 		for len(bytes) > 0 {
 			_, _, colIDDiff, _, err = encoding.DecodeValueTag(bytes)
 			if err != nil {
-				return "", "", "", false, err
+				return "", RawValue{}, false, err
 			}
 			colID := lastColID + descpb.ColumnID(colIDDiff)
 			lastColID = colID
 			if idx, ok := d.colIdxMap.Get(colID); ok {
 				res, bytes, err = rowenc.DecodeTableValue(&d.alloc, tbl.PublicColumns()[idx].GetType(), bytes)
 				if err != nil {
-					return "", "", "", false, err
+					return "", RawValue{}, false, err
 				}
 				switch colID {
 				case tbl.PublicColumns()[1].GetID(): // value
-					val = string(tree.MustBeDString(res))
+					val.Value = string(tree.MustBeDString(res))
 				case tbl.PublicColumns()[3].GetID(): // valueType
-					valType = string(tree.MustBeDString(res))
+					val.Type = string(tree.MustBeDString(res))
 				case tbl.PublicColumns()[2].GetID(): // lastUpdated
 					// TODO(dt): we could decode just the len and then seek `bytes` past
 					// it, without allocating/decoding the unused timestamp.
 				default:
-					return "", "", "", false, errors.Errorf("unknown column: %v", colID)
+					return "", RawValue{}, false, errors.Errorf("unknown column: %v", colID)
 				}
 			}
 		}
 	}
-	return setting, val, valType, false, nil
+	return setting, val, false, nil
 }

pkg/server/settingswatcher/row_decoder_external_test.go

@@ -76,24 +76,24 @@ func TestRowDecoder(t *testing.T) {
 			Value: *row.Value,
 		}
 
-		k, val, valType, tombstone, err := dec.DecodeRow(kv)
+		k, val, tombstone, err := dec.DecodeRow(kv)
 		require.NoError(t, err)
 		require.False(t, tombstone)
 		if exp, ok := toSet[k]; ok {
-			require.Equal(t, exp.expStr, val)
-			require.Equal(t, exp.expValType, valType)
+			require.Equal(t, exp.expStr, val.Value)
+			require.Equal(t, exp.expValType, val.Type)
 			delete(toSet, k)
 		}
 
 		// Test the tombstone logic while we're here.
 		{
 			kv.Value.Reset()
-			tombstoneK, val, valType, tombstone, err := dec.DecodeRow(kv)
+			tombstoneK, val, tombstone, err := dec.DecodeRow(kv)
 			require.NoError(t, err)
 			require.True(t, tombstone)
 			require.Equal(t, k, tombstoneK)
-			require.Zero(t, val)
-			require.Zero(t, valType)
+			require.Zero(t, val.Value)
+			require.Zero(t, val.Type)
 		}
 	}
 	require.Len(t, toSet, 0)