From 73f1fb663cdef72d7e53882e1ceba90b33b2e04f Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 23 Nov 2023 10:14:55 +0000 Subject: [PATCH] query-frontend worker: cancel is not an error (#4648) Could be the user closed the tab, or it's a sharded query and some other shard errored. Whatever the cause, cancellation is not an error. I expect two effects: the worker won't log "context canceled", and it will process the next request immediately. --- CHANGELOG.md | 1 + pkg/querier/worker/frontend_processor.go | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fc8d89522f..6d3c9a3605f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ * [FEATURE] Add the experimental `-.s3.send-content-md5` flag (defaults to `false`) to configure S3 Put Object requests to send a `Content-MD5` header. Setting this flag is not recommended unless your object storage does not support checksums. #6622 * [FEATURE] Distributor: add an experimental flag `-distributor.reusable-ingester-push-worker` that can be used to pre-allocate a pool of workers to be used to send push requests to the ingesters. #6660 * [FEATURE] Distributor: Support enabling of automatically generated name suffixes for metrics ingested via OTLP, through the flag `-distributor.otel-metric-suffixes-enabled`. #6542 +* [ENHANCEMENT] Query-frontend: don't treat cancel as an error. #4648 * [ENHANCEMENT] Ingester: exported summary `cortex_ingester_inflight_push_requests_summary` tracking total number of inflight requests in percentile buckets. #5845 * [ENHANCEMENT] Query-scheduler: add `cortex_query_scheduler_enqueue_duration_seconds` metric that records the time taken to enqueue or reject a query request. #5879 * [ENHANCEMENT] Query-frontend: add `cortex_query_frontend_enqueue_duration_seconds` metric that records the time taken to enqueue or reject a query request. When query-scheduler is in use, the metric has the `scheduler_address` label to differentiate the enqueue duration by query-scheduler backend. #5879 #6087 #6120 diff --git a/pkg/querier/worker/frontend_processor.go b/pkg/querier/worker/frontend_processor.go index fa841b46a58..47b74ee3874 100644 --- a/pkg/querier/worker/frontend_processor.go +++ b/pkg/querier/worker/frontend_processor.go @@ -14,6 +14,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/dskit/backoff" + "github.com/grafana/dskit/grpcutil" "github.com/grafana/dskit/httpgrpc" "go.uber.org/atomic" "google.golang.org/grpc" @@ -84,9 +85,10 @@ func (fp *frontendProcessor) processQueriesOnSingleStream(workerCtx context.Cont } if err := fp.process(c, inflightQuery); err != nil { - level.Error(fp.log).Log("msg", "error processing requests", "address", address, "err", err) - backoff.Wait() - continue + if !grpcutil.IsCanceled(err) { + level.Error(fp.log).Log("msg", "error processing requests", "address", address, "err", err) + backoff.Wait() + } } backoff.Reset()