[r326] frontend: Allow blocking raw http requests

CHANGELOG.md

@@ -25,6 +25,7 @@
 * [ENHANCEMENT] Query-frontend: include more information about read consistency in trace spans produced when using experimental ingest storage. #10412
 * [ENHANCEMENT] Ingester: Hide tokens in ingester ring status page when ingest storage is enabled #10399
 * [ENHANCEMENT] Ingester: add `active_series_additional_custom_trackers` configuration, in addition to the already existing `active_series_custom_trackers`. The `active_series_additional_custom_trackers` configuration allows you to configure additional custom trackers that get merged with `active_series_custom_trackers` at runtime. #10428
+* [ENHANCEMENT] Query-frontend: Allow blocking raw http requests with the `blocked_requests` configuration. Requests can be blocked based on their path, method
 * [BUGFIX] Distributor: Use a boolean to track changes while merging the ReplicaDesc components, rather than comparing the objects directly. #10185
 * [BUGFIX] Querier: fix timeout responding to query-frontend when response size is very close to `-querier.frontend-client.grpc-max-send-msg-size`. #10154
 * [BUGFIX] Query-frontend and querier: show warning/info annotations in some cases where they were missing (if a lazy querier was used). #10277

cmd/mimir/config-descriptor.json

@@ -4327,6 +4327,16 @@
           "fieldType": "blocked_queries_config...",
           "fieldCategory": "experimental"
         },
+        {
+          "kind": "field",
+          "name": "blocked_requests",
+          "required": false,
+          "desc": "List of http requests to block.",
+          "fieldValue": null,
+          "fieldDefaultValue": null,
+          "fieldType": "blocked_requests_config...",
+          "fieldCategory": "experimental"
+        },
         {
           "kind": "field",
           "name": "align_queries_with_step",

docs/sources/mimir/configure/configuration-parameters/index.md

@@ -3578,6 +3578,9 @@ The `limits` block configures default and per-tenant limits imposed by component
 # (experimental) List of queries to block.
 [blocked_queries: <blocked_queries_config...> | default = ]
 
+# (experimental) List of http requests to block.
+[blocked_requests: <blocked_requests_config...> | default = ]
+
 # Mutate incoming queries to align their start and end with their step to
 # improve result caching.
 # CLI flag: -query-frontend.align-queries-with-step

docs/sources/mimir/manage/mimir-runbooks/_index.md

@@ -2550,6 +2550,19 @@ How to **fix** it:
 
 This error only occurs when an administrator has explicitly define a blocked list for a given tenant. After assessing whether or not the reason for blocking one or multiple queries you can update the tenant's limits and remove the pattern.
 
+### err-mimir-request-blocked
+
+This error occurs when a query-frontend blocks a HTTP request because the request matches at least one of the rules defined in the limits.
+
+How it **works**:
+
+- The query-frontend implements a checker responsible for assessing whether the request is blocked or not.
+- To configure the limit, set the block `blocked_requests` in the `limits`.
+
+How to **fix** it:
+
+This error only occurs when an administrator has explicitly define a blocked list for a given tenant. After assessing whether or not the reason for blocking one or multiple requests you can update the tenant's limits and remove the configuration.
+
 ### err-mimir-alertmanager-max-grafana-config-size
 
 This non-critical error occurs when the Alertmanager receives a Grafana Alertmanager configuration larger than the configured size limit.

pkg/frontend/querymiddleware/limits.go

@@ -105,6 +105,9 @@ type Limits interface {
 	// BlockedQueries returns the blocked queries.
 	BlockedQueries(userID string) []*validation.BlockedQuery
 
+	// BlockedRequests returns the blocked http requests.
+	BlockedRequests(userID string) []*validation.BlockedRequest
+
 	// AlignQueriesWithStep returns if queries should be adjusted to be step-aligned
 	AlignQueriesWithStep(userID string) bool
 

pkg/frontend/querymiddleware/limits_test.go

@@ -601,6 +601,10 @@ func (m multiTenantMockLimits) IngestStorageReadConsistency(userID string) strin
 	return m.byTenant[userID].ingestStorageReadConsistency
 }
 
+func (m multiTenantMockLimits) BlockedRequests(userID string) []*validation.BlockedRequest {
+	return m.byTenant[userID].blockedRequests
+}
+
 type mockLimits struct {
 	maxQueryLookback                     time.Duration
 	maxQueryLength                       time.Duration
@@ -626,6 +630,7 @@ type mockLimits struct {
 	enabledPromQLExperimentalFunctions   []string
 	prom2RangeCompat                     bool
 	blockedQueries                       []*validation.BlockedQuery
+	blockedRequests                      []*validation.BlockedRequest
 	alignQueriesWithStep                 bool
 	queryIngestersWithin                 time.Duration
 	ingestStorageReadConsistency         string
@@ -741,6 +746,10 @@ func (m mockLimits) IngestStorageReadConsistency(string) string {
 	return m.ingestStorageReadConsistency
 }
 
+func (m mockLimits) BlockedRequests(string) []*validation.BlockedRequest {
+	return m.blockedRequests
+}
+
 type mockHandler struct {
 	mock.Mock
 }

pkg/frontend/querymiddleware/request_blocker.go

@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package querymiddleware
+
+import (
+	"net/http"
+
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/grafana/dskit/tenant"
+	"github.com/grafana/regexp"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	apierror "github.com/grafana/mimir/pkg/api/error"
+	"github.com/grafana/mimir/pkg/util/globalerror"
+)
+
+func newRequestBlockedError() error {
+	return apierror.New(apierror.TypeBadData, globalerror.RequestBlocked.Message("the request has been blocked by the cluster administrator"))
+}
+
+type requestBlocker struct {
+	limits                 Limits
+	logger                 log.Logger
+	blockedRequestsCounter *prometheus.CounterVec
+}
+
+func newRequestBlocker(
+	limits Limits,
+	logger log.Logger,
+	registerer prometheus.Registerer,
+) *requestBlocker {
+	blockedRequestsCounter := promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
+		Name: "cortex_query_frontend_rejected_requests_total",
+		Help: "Number of HTTP requests that were rejected by the cluster administrator.",
+	}, []string{"user"})
+	return &requestBlocker{
+		limits:                 limits,
+		logger:                 logger,
+		blockedRequestsCounter: blockedRequestsCounter,
+	}
+}
+
+func (rb *requestBlocker) isBlocked(r *http.Request) error {
+	tenants, err := tenant.TenantIDs(r.Context())
+	if err != nil {
+		return nil
+	}
+
+	for _, tenant := range tenants {
+		blockedRequests := rb.limits.BlockedRequests(tenant)
+
+		for _, blockedRequest := range blockedRequests {
+			if blockedPath := blockedRequest.Path; blockedPath != "" && blockedPath != r.URL.Path {
+				continue
+			}
+
+			if blockedMethod := blockedRequest.Method; blockedMethod != "" && blockedMethod != r.Method {
+				continue
+			}
+
+			if blockedParams := blockedRequest.QueryParams; len(blockedParams) > 0 {
+				query := r.URL.Query()
+				blockedByParams := false
+				for key, blocked := range blockedParams {
+					if blocked.IsRegexp {
+						blockedRegexp, err := regexp.Compile(blocked.Value)
+						if err != nil {
+							level.Error(rb.logger).Log("msg", "failed to compile regexp. Not blocking", "regexp", blocked.Value, "err", err)
+							continue
+						}
+
+						if blockedRegexp.MatchString(query.Get(key)) {
+							blockedByParams = true
+							break
+						}
+					} else if query.Get(key) == blocked.Value {
+						blockedByParams = true
+						break
+					}
+				}
+
+				if !blockedByParams {
+					continue
+				}
+			}
+
+			level.Info(rb.logger).Log("msg", "request blocked", "user", tenant, "url", r.URL.String(), "method", r.Method)
+			rb.blockedRequestsCounter.WithLabelValues(tenant).Inc()
+			return newRequestBlockedError()
+		}
+	}
+	return nil
+
+}

pkg/frontend/querymiddleware/request_blocker_test.go

@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package querymiddleware
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/go-kit/log"
+	"github.com/grafana/dskit/user"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/mimir/pkg/util/validation"
+)
+
+func TestRequestBlocker_IsBlocked(t *testing.T) {
+	const userID = "user-1"
+	// Mock the limits.
+	limits := multiTenantMockLimits{
+		byTenant: map[string]mockLimits{
+			userID: {
+				blockedRequests: []*validation.BlockedRequest{
+					{Path: "/blocked-by-path"},
+					{Method: "POST"},
+					{
+						QueryParams: map[string]validation.BlockedRequestQueryParam{"foo": {Value: "bar"}},
+					},
+					{
+						Path: "/blocked-by-path2", Method: "GET",
+						QueryParams: map[string]validation.BlockedRequestQueryParam{"foo": {Value: "bar2"}},
+					},
+					{
+						Path: "/block-by-query-regexp", Method: "GET",
+						QueryParams: map[string]validation.BlockedRequestQueryParam{"foo": {Value: ".*hello.*", IsRegexp: true}},
+					},
+					// Invalid regexp should not block anything.
+					{
+						QueryParams: map[string]validation.BlockedRequestQueryParam{"foo": {Value: "\bar", IsRegexp: true}},
+					},
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		name     string
+		request  func() *http.Request
+		expected error
+	}{
+		{
+			name: "request is not blocked",
+			request: func() *http.Request {
+				req, err := http.NewRequest(http.MethodGet, "/not-blocked", nil)
+				require.NoError(t, err)
+				return req
+			},
+			expected: nil,
+		},
+		{
+			name: "request is blocked by path",
+			request: func() *http.Request {
+				req, err := http.NewRequest(http.MethodGet, "/blocked-by-path", nil)
+				require.NoError(t, err)
+				return req
+			},
+			expected: newRequestBlockedError(),
+		},
+		{
+			name: "request is blocked by method",
+			request: func() *http.Request {
+				req, err := http.NewRequest(http.MethodPost, "/not-blocked", nil)
+				require.NoError(t, err)
+				return req
+			},
+			expected: newRequestBlockedError(),
+		},
+		{
+			name: "request is blocked by query params",
+			request: func() *http.Request {
+				req, err := http.NewRequest(http.MethodGet, "/not-blocked?foo=bar", nil)
+				require.NoError(t, err)
+				return req
+			},
+			expected: newRequestBlockedError(),
+		},
+		{
+			name: "request is blocked by path, method and query params",
+			request: func() *http.Request {
+				req, err := http.NewRequest(http.MethodGet, "/blocked-by-path2?foo=bar2", nil)
+				require.NoError(t, err)
+				return req
+			},
+			expected: newRequestBlockedError(),
+		},
+		{
+			name: "request does not fully match blocked request (different method)",
+			request: func() *http.Request {
+				req, err := http.NewRequest(http.MethodDelete, "/blocked-by-path2?foo=bar2", nil)
+				require.NoError(t, err)
+				return req
+			},
+			expected: nil,
+		},
+		{
+			name: "request does not fully match blocked request (different query params)",
+			request: func() *http.Request {
+				req, err := http.NewRequest(http.MethodGet, "/blocked-by-path2?foo=bar3", nil)
+				require.NoError(t, err)
+				return req
+			},
+			expected: nil,
+		},
+		{
+			name: "request does not fully match blocked request (different path)",
+			request: func() *http.Request {
+				req, err := http.NewRequest(http.MethodGet, "/blocked-by-path3?foo=bar2", nil)
+				require.NoError(t, err)
+				return req
+			},
+			expected: nil,
+		},
+		{
+			name: "regexp does not match",
+			request: func() *http.Request {
+				req, err := http.NewRequest(http.MethodGet, "/block-by-query-regexp?foo=test", nil)
+				require.NoError(t, err)
+				return req
+			},
+			expected: nil,
+		},
+		{
+			name: "regexp matches",
+			request: func() *http.Request {
+				req, err := http.NewRequest(http.MethodGet, "/block-by-query-regexp?foo=my-value-hello-test", nil)
+				require.NoError(t, err)
+				return req
+			},
+			expected: newRequestBlockedError(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			blocker := newRequestBlocker(limits, log.NewNopLogger(), prometheus.NewRegistry())
+
+			req := tt.request()
+			ctx := user.InjectOrgID(context.Background(), userID)
+			req = req.WithContext(ctx)
+
+			err := blocker.isBlocked(req)
+			assert.Equal(t, err, tt.expected)
+		})
+	}
+}

pkg/frontend/querymiddleware/roundtrip.go

@@ -257,6 +257,7 @@ func newQueryTripperware(
 	}
 
 	queryRangeMiddleware, queryInstantMiddleware, remoteReadMiddleware := newQueryMiddlewares(cfg, log, limits, codec, c, cacheKeyGenerator, cacheExtractor, engine, registerer)
+	requestBlocker := newRequestBlocker(limits, log, registerer)
 
 	return func(next http.RoundTripper) http.RoundTripper {
 		// IMPORTANT: roundtrippers are executed in *reverse* order because they are wrappers.
@@ -311,6 +312,10 @@ func newQueryTripperware(
 		cardinality = NewCardinalityQueryRequestValidationRoundTripper(cardinality)
 
 		return RoundTripFunc(func(r *http.Request) (*http.Response, error) {
+			if err := requestBlocker.isBlocked(r); err != nil {
+				return nil, err
+			}
+
 			switch {
 			case IsRangeQuery(r.URL.Path):
 				return queryrange.RoundTrip(r)