From fc8e64cc0114ed9d019de096470649cb8151db8a Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 27 Sep 2021 16:11:36 +0200 Subject: [PATCH] .*: Removed limit for received message size through gRPC. (#4708) Error log: observatorium-thanos-receive-default-9observatorium-thanos-receive-default-0 thanos-receive level=error ts=2021-09-24T18:13:03.444687849Z caller=handler.go:366 component=receive component=receive-handler err="10 errors: replicate write request for endpoint observatorium-thanos-receive-default-1.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: quorum not reached: 3 errors: forwarding request to endpoint observatorium-thanos-receive-default-2.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6319597 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-1.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6319597 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-3.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6319597 vs. 4194304); replicate write request for endpoint observatorium-thanos-receive-default-7.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: quorum not reached: 3 errors: forwarding request to endpoint observatorium-thanos-receive-default-8.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6197131 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-7.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6197131 vs. 4194304); store locally for endpoint observatorium-thanos-receive-default-9.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: conflict; replicate write request for endpoint observatorium-thanos-receive-default-0.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: quorum not reached: 3 errors: forwarding request to endpoint observatorium-thanos-receive-default-2.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6119023 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-1.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6119023 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-0.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6119023 vs. 4194304); replicate write request for endpoint observatorium-thanos-receive-default-8.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: quorum not reached: 3 errors: forwarding request to endpoint observatorium-thanos-receive-default-8.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6255490 vs. 4194304); store locally for endpoint observatorium-thanos-receive-default-9.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: conflict; forwarding request to endpoint observatorium-thanos-receive-default-0.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6255490 vs. 4194304); replicate write request for endpoint observatorium-thanos-receive-default-9.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: quorum not reached: 3 errors: forwarding request to endpoint observatorium-thanos-receive-default-1.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6118562 vs. 4194304); store locally for endpoint observatorium-thanos-receive-default-9.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: conflict; forwarding request to endpoint observatorium-thanos-receive-default-0.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6118562 vs. 4194304); replicate write request for endpoint observatorium-thanos-receive-default-6.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: quorum not reached: 3 errors: forwarding request to endpoint observatorium-thanos-receive-default-8.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (5955344 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-7.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (5955344 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-6.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (5955344 vs. 4194304); replicate write request for endpoint observatorium-thanos-receive-default-2.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: quorum not reached: 3 errors: forwarding request to endpoint observatorium-thanos-receive-default-2.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (5867179 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-3.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (5867179 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-4.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (5867179 vs. 4194304); replicate write request for endpoint observatorium-thanos-receive-default-3.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: quorum not reached: 3 errors: forwarding request to endpoint observatorium-thanos-receive-default-5.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6014110 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-3.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6014110 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-4.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6014110 vs. 4194304); replicate write request for endpoint observatorium-thanos-receive-default-4.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: quorum not reached: 3 errors: forwarding request to endpoint observatorium-thanos-receive-default-5.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6064467 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-6.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6064467 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-4.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (6064467 vs. 4194304); replicate write request for endpoint observatorium-thanos-receive-default-5.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: quorum not reached: 3 errors: forwarding request to endpoint observatorium-thanos-receive-default-5.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (5796760 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-7.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (5796760 vs. 4194304); forwarding request to endpoint observatorium-thanos-receive-default-6.observatorium-thanos-receive-default.monitoring.svc.cluster.local:10901: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (5796760 vs. 4194304)" msg="internal server error" Signed-off-by: Bartlomiej Plotka --- pkg/server/grpc/grpc.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/server/grpc/grpc.go b/pkg/server/grpc/grpc.go index 10c6f08e4b..6a39ba5afb 100644 --- a/pkg/server/grpc/grpc.go +++ b/pkg/server/grpc/grpc.go @@ -72,7 +72,11 @@ func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer } options.grpcOpts = append(options.grpcOpts, []grpc.ServerOption{ + // NOTE: It is recommended for gRPC messages to not go over 1MB, yet it is typical for remote write requests and store API responses to go over 4MB. + // Remove limits and allow users to use histogram message sizes to detect those situations. + // TODO(bwplotka): https://github.com/grpc-ecosystem/go-grpc-middleware/issues/462 grpc.MaxSendMsgSize(math.MaxInt32), + grpc.MaxRecvMsgSize(math.MaxInt32), grpc_middleware.WithUnaryServerChain( grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), met.UnaryServerInterceptor(),