tenantrate: relax host-side tenant rate limit

pkg/kv/kvserver/replica_init.go

@@ -341,7 +341,7 @@ func (r *Replica) setDescLockedRaftMuLocked(ctx context.Context, desc *roachpb.R
 		r.mu.tenantID = tenantID
 		r.store.metrics.acquireTenant(tenantID)
 		if tenantID != roachpb.SystemTenantID {
-			r.tenantLimiter = r.store.tenantRateLimiters.GetTenant(tenantID, r.store.stopper.ShouldQuiesce())
+			r.tenantLimiter = r.store.tenantRateLimiters.GetTenant(ctx, tenantID, r.store.stopper.ShouldQuiesce())
 		}
 	}
 

pkg/kv/kvserver/tenantrate/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//pkg/multitenant/tenantcostmodel",
         "//pkg/roachpb:with-mocks",
         "//pkg/settings",
+        "//pkg/util/log",
         "//pkg/util/metric",
         "//pkg/util/metric/aggmetric",
         "//pkg/util/quotapool",

pkg/kv/kvserver/tenantrate/factory.go

@@ -16,6 +16,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcostmodel"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/settings"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/quotapool"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
@@ -73,7 +74,9 @@ func NewLimiterFactory(sv *settings.Values, knobs *TestingKnobs) *LimiterFactory
 // reference counted; call Destroy on the returned limiter when it is no longer
 // in use. If the closer channel is non-nil, closing it will lead to any blocked
 // requests becoming unblocked.
-func (rl *LimiterFactory) GetTenant(tenantID roachpb.TenantID, closer <-chan struct{}) Limiter {
+func (rl *LimiterFactory) GetTenant(
+	ctx context.Context, tenantID roachpb.TenantID, closer <-chan struct{},
+) Limiter {
 
 	if tenantID == roachpb.SystemTenantID {
 		return &rl.systemLimiter
@@ -94,6 +97,10 @@ func (rl *LimiterFactory) GetTenant(tenantID roachpb.TenantID, closer <-chan str
 		rcLim = new(refCountedLimiter)
 		rcLim.lim.init(rl, tenantID, rl.mu.config, rl.metrics.tenantMetrics(tenantID), options...)
 		rl.mu.tenants[tenantID] = rcLim
+		log.Infof(
+			ctx, "tenant %s rate limiter initialized (rate: %g RU/s; burst: %g RU)",
+			tenantID, rl.mu.config.Rate, rl.mu.config.Burst,
+		)
 	}
 	rcLim.refCount++
 	return &rcLim.lim

pkg/kv/kvserver/tenantrate/limiter.go

@@ -23,6 +23,10 @@ import (
 
 // Limiter is used to rate-limit KV requests for a given tenant.
 //
+// The intention is to limit a single tenant from using a large percentage of a
+// KV machine, which could lead to very significant variation in observed
+// performance depending how many other tenants are using the node.
+//
 // The use of an interface permits a different implementation for the system
 // tenant and other tenants. The remaining commentary will pertain to the
 // implementation used for non-system tenants. The limiter is implemented as

pkg/kv/kvserver/tenantrate/limiter_test.go

@@ -14,6 +14,7 @@ import (
 	"context"
 	"fmt"
 	"regexp"
+	"runtime"
 	"sort"
 	"strings"
 	"testing"
@@ -46,8 +47,8 @@ func TestCloser(t *testing.T) {
 	})
 	tenant := roachpb.MakeTenantID(2)
 	closer := make(chan struct{})
-	limiter := factory.GetTenant(tenant, closer)
 	ctx := context.Background()
+	limiter := factory.GetTenant(ctx, tenant, closer)
 	// First Wait call will not block.
 	require.NoError(t, limiter.Wait(ctx, tenantcostmodel.TestingRequestInfo(true, 1)))
 	errCh := make(chan error, 1)
@@ -436,10 +437,11 @@ func timesToStrings(times []time.Time) []string {
 //  [2#2, 3#1]
 //
 func (ts *testState) getTenants(t *testing.T, d *datadriven.TestData) string {
+	ctx := context.Background()
 	tenantIDs := parseTenantIDs(t, d)
 	for i := range tenantIDs {
 		id := roachpb.MakeTenantID(tenantIDs[i])
-		ts.tenants[id] = append(ts.tenants[id], ts.rl.GetTenant(id, nil /* closer */))
+		ts.tenants[id] = append(ts.tenants[id], ts.rl.GetTenant(ctx, id, nil /* closer */))
 	}
 	return ts.FormatTenants()
 }
@@ -509,6 +511,12 @@ func (ts *testState) estimateIOPS(t *testing.T, d *datadriven.TestData) string {
 
 	sustained := calculateIOPS(config.Rate)
 	burst := calculateIOPS(config.Burst)
+
+	// By default, the rate scales with GOMAXPROCS.
+	numProcs := float64(runtime.GOMAXPROCS(0))
+	sustained /= numProcs
+	burst /= numProcs
+
 	fmtFloat := func(val float64) string {
 		if val < 10 {
 			return fmt.Sprintf("%.1f", val)
@@ -518,17 +526,17 @@ func (ts *testState) estimateIOPS(t *testing.T, d *datadriven.TestData) string {
 	switch workload.ReadPercentage {
 	case 0:
 		return fmt.Sprintf(
-			"Write-only workload (%s writes): %s sustained IOPS, %s burst.",
+			"Write-only workload (%s writes): %s sustained IOPS/CPU, %s burst.",
 			humanize.IBytes(uint64(workload.WriteSize)), fmtFloat(sustained), fmtFloat(burst),
 		)
 	case 100:
 		return fmt.Sprintf(
-			"Read-only workload (%s reads): %s sustained IOPS, %s burst.",
+			"Read-only workload (%s reads): %s sustained IOPS/CPU, %s burst.",
 			humanize.IBytes(uint64(workload.ReadSize)), fmtFloat(sustained), fmtFloat(burst),
 		)
 	default:
 		return fmt.Sprintf(
-			"Mixed workload (%d%% reads; %s reads; %s writes): %s sustained IOPS, %s burst.",
+			"Mixed workload (%d%% reads; %s reads; %s writes): %s sustained IOPS/CPU, %s burst.",
 			workload.ReadPercentage,
 			humanize.IBytes(uint64(workload.ReadSize)), humanize.IBytes(uint64(workload.WriteSize)),
 			fmtFloat(sustained), fmtFloat(burst),

pkg/kv/kvserver/tenantrate/settings.go

@@ -11,8 +11,11 @@
 package tenantrate
 
 import (
+	"runtime"
+
 	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcostmodel"
 	"github.com/cockroachdb/cockroach/pkg/settings"
+	"github.com/cockroachdb/errors"
 )
 
 // Config contains the configuration of the rate limiter.
@@ -38,11 +41,26 @@ type Config struct {
 // The burst limit setting is defined as a multiplier of the rate (i.e. in
 // seconds), so it doesn't need to be adjusted in concert with the rate.
 var (
+	// ruRateLimit was initially set to an absolute value in RU/s, with the
+	// intention of throttling free tier tenants.
+	//
+	// We now use it to disallow a single tenant from harnessing a large fraction
+	// of a KV node, in order to avoid very significant fluctuations in
+	// performance depending on what other tenants are using the same KV node.
+	// In this mode, a value of -200 means that we allow 200 RU/s per CPU, or
+	// roughly
+	// 20% of the machine (by design 1 RU roughly maps to 1 CPU-millisecond).
 	ruRateLimit = settings.RegisterFloatSetting(
 		"kv.tenant_rate_limiter.rate_limit",
-		"per-tenant rate limit in Request Units per second",
-		200,
-		settings.PositiveFloat,
+		"per-tenant rate limit in Request Units per second if positive, "+
+			"or Request Units per second per CPU if negative",
+		-200,
+		func(v float64) error {
+			if v == 0 {
+				return errors.New("cannot set to zero value")
+			}
+			return nil
+		},
 	)
 
 	ruBurstLimitSeconds = settings.RegisterFloatSetting(
@@ -59,9 +77,20 @@ var (
 	}
 )
 
+// absoluteRateFromConfigValue returns an absolute rate (in RU/s) from a value
+// of the kv.tenant_rate_limiter.rate_limit setting.
+func absoluteRateFromConfigValue(value float64) float64 {
+	if value < 0 {
+		// We use GOMAXPROCS instead of NumCPU because the former could be adjusted
+		// based on cgroup limits (see cgroups.AdjustMaxProcs).
+		return -value * float64(runtime.GOMAXPROCS(0))
+	}
+	return value
+}
+
 // ConfigFromSettings constructs a Config using the cluster setting values.
 func ConfigFromSettings(sv *settings.Values) Config {
-	rate := ruRateLimit.Get(sv)
+	rate := absoluteRateFromConfigValue(ruRateLimit.Get(sv))
 	return Config{
 		Rate:      rate,
 		Burst:     rate * ruBurstLimitSeconds.Get(sv),
@@ -72,7 +101,7 @@ func ConfigFromSettings(sv *settings.Values) Config {
 // DefaultConfig returns the configuration that corresponds to the default
 // setting values.
 func DefaultConfig() Config {
-	rate := ruRateLimit.Default()
+	rate := absoluteRateFromConfigValue(ruRateLimit.Default())
 	return Config{
 		Rate:      rate,
 		Burst:     rate * ruBurstLimitSeconds.Default(),

pkg/kv/kvserver/tenantrate/testdata/estimate_iops

@@ -2,48 +2,48 @@ estimate_iops
 readpercentage: 100
 readsize: 4096
 ----
-Read-only workload (4.0 KiB reads): 271 sustained IOPS, 2706 burst.
+Read-only workload (4.0 KiB reads): 271 sustained IOPS/CPU, 2706 burst.
 
 estimate_iops
 readpercentage: 100
 readsize: 65536
 ----
-Read-only workload (64 KiB reads): 151 sustained IOPS, 1509 burst.
+Read-only workload (64 KiB reads): 151 sustained IOPS/CPU, 1509 burst.
 
 estimate_iops
 readpercentage: 100
 readsize: 1048576
 ----
-Read-only workload (1.0 MiB reads): 19 sustained IOPS, 187 burst.
+Read-only workload (1.0 MiB reads): 19 sustained IOPS/CPU, 187 burst.
 
 estimate_iops
 readpercentage: 0
 writesize: 4096
 ----
-Write-only workload (4.0 KiB writes): 78 sustained IOPS, 780 burst.
+Write-only workload (4.0 KiB writes): 78 sustained IOPS/CPU, 780 burst.
 
 estimate_iops
 readpercentage: 0
 writesize: 65536
 ----
-Write-only workload (64 KiB writes): 7.7 sustained IOPS, 77 burst.
+Write-only workload (64 KiB writes): 7.7 sustained IOPS/CPU, 77 burst.
 
 estimate_iops
 readpercentage: 0
 writesize: 1048576
 ----
-Write-only workload (1.0 MiB writes): 0.5 sustained IOPS, 5.0 burst.
+Write-only workload (1.0 MiB writes): 0.5 sustained IOPS/CPU, 5.0 burst.
 
 estimate_iops
 readpercentage: 50
 readsize: 4096
 writesize: 4096
 ----
-Mixed workload (50% reads; 4.0 KiB reads; 4.0 KiB writes): 121 sustained IOPS, 1212 burst.
+Mixed workload (50% reads; 4.0 KiB reads; 4.0 KiB writes): 121 sustained IOPS/CPU, 1212 burst.
 
 estimate_iops
 readpercentage: 90
 readsize: 4096
 writesize: 4096
 ----
-Mixed workload (90% reads; 4.0 KiB reads; 4.0 KiB writes): 217 sustained IOPS, 2171 burst.
+Mixed workload (90% reads; 4.0 KiB reads; 4.0 KiB writes): 217 sustained IOPS/CPU, 2171 burst.

pkg/kv/kvserver/tenantrate/testdata/reads

@@ -81,5 +81,3 @@ await
 ----
 []
 
-
-