diff --git a/CHANGELOG.md b/CHANGELOG.md index f5da0bee4c8..ad0f98a67fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ compactor.compaction.flush_size_bytes. [#1696](https://github.com/grafana/tempo/pull/1696) (@joe-elliott) * [CHANGE] Return 200 instead of 206 when blocks failed is < tolerate_failed_blocks. [#1725](https://github.com/grafana/tempo/pull/1725) (@joe-elliott) * [CHANGE] Update Go to 1.19 [#1665](https://github.com/grafana/tempo/pull/1665) (@ie-pham) +* [CHANGE] Remove unsued scheduler frontend code [#1734](https://github.com/grafana/tempo/pull/1734) (@mapno) * [FEATURE] Add capability to configure the used S3 Storage Class [#1697](https://github.com/grafana/tempo/pull/1714) (@amitsetty) * [ENHANCEMENT] cache: expose username and sentinel_username redis configuration options for ACL-based Redis Auth support [#1708](https://github.com/grafana/tempo/pull/1708) (@jsievenpiper) * [ENHANCEMENT] metrics-generator: expose span size as a metric [#1662](https://github.com/grafana/tempo/pull/1662) (@ie-pham) diff --git a/Makefile b/Makefile index 319031f7f0f..c6078022194 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ GOPATH := $(shell go env GOPATH) GORELEASER := $(GOPATH)/bin/goreleaser # Build Images -DOCKER_PROTOBUF_IMAGE ?= otel/build-protobuf:0.2.1 +DOCKER_PROTOBUF_IMAGE ?= otel/build-protobuf:0.14.0 FLATBUFFERS_IMAGE ?= neomantra/flatbuffers LOKI_BUILD_IMAGE ?= grafana/loki-build-image:0.21.0 DOCS_IMAGE ?= grafana/docs-base:latest @@ -184,7 +184,6 @@ gen-proto: $(call PROTO_GEN,$(PROTO_INTERMEDIATE_DIR)/trace/v1/trace.proto,./pkg/tempopb/) $(call PROTO_GEN,pkg/tempopb/tempo.proto,./) $(call PROTO_GEN_WITH_VENDOR,modules/frontend/v1/frontendv1pb/frontend.proto,./) - $(call PROTO_GEN_WITH_VENDOR,modules/frontend/v2/frontendv2pb/frontend.proto,./) rm -rf $(PROTO_INTERMEDIATE_DIR) diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go index 8012eabd38f..147b42cf275 100644 --- a/cmd/tempo/app/modules.go +++ b/cmd/tempo/app/modules.go @@ -216,9 +216,9 @@ func (t *App) initQuerier() (services.Service, error) { } func (t *App) initQueryFrontend() (services.Service, error) { - // cortexTripper is a bridge between http and httpgrpc. it does the job of passing data to the cortex - // frontend code - cortexTripper, v1, _, err := frontend.InitFrontend(t.cfg.Frontend.Config, frontend.CortexNoQuerierLimits{}, 0, log.Logger, prometheus.DefaultRegisterer) + // cortexTripper is a bridge between http and httpgrpc. + // It does the job of passing data to the cortex frontend code. + cortexTripper, v1, err := frontend.InitFrontend(t.cfg.Frontend.Config, frontend.CortexNoQuerierLimits{}, log.Logger, prometheus.DefaultRegisterer) if err != nil { return nil, err } diff --git a/modules/frontend/config.go b/modules/frontend/config.go index 603861366ab..5ed86074a5c 100644 --- a/modules/frontend/config.go +++ b/modules/frontend/config.go @@ -7,37 +7,31 @@ import ( "net/http" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/grafana/tempo/modules/frontend/transport" v1 "github.com/grafana/tempo/modules/frontend/v1" - v2 "github.com/grafana/tempo/modules/frontend/v2" "github.com/grafana/tempo/pkg/usagestats" - "github.com/grafana/tempo/pkg/util" ) var ( - statWorkerConcurrency = usagestats.NewInt("frontend_worker_concurrency") - statVersion = usagestats.NewString("frontend_version") + statVersion = usagestats.NewString("frontend_version") ) type Config struct { - Config CombinedFrontendConfig `yaml:",inline"` - MaxRetries int `yaml:"max_retries,omitempty"` - QueryShards int `yaml:"query_shards,omitempty"` - TolerateFailedBlocks int `yaml:"tolerate_failed_blocks,omitempty"` - Search SearchConfig `yaml:"search"` + Config v1.Config `yaml:",inline"` + MaxRetries int `yaml:"max_retries,omitempty"` + QueryShards int `yaml:"query_shards,omitempty"` + TolerateFailedBlocks int `yaml:"tolerate_failed_blocks,omitempty"` + Search SearchConfig `yaml:"search"` } type SearchConfig struct { Sharder SearchSharderConfig `yaml:",inline"` } -func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) { - cfg.Config.DownstreamURL = "" - cfg.Config.Handler.LogQueriesLongerThan = 0 - cfg.Config.FrontendV1.MaxOutstandingPerTenant = 100 +func (cfg *Config) RegisterFlagsAndApplyDefaults(string, *flag.FlagSet) { + cfg.Config.MaxOutstandingPerTenant = 100 cfg.MaxRetries = 2 cfg.QueryShards = 20 cfg.TolerateFailedBlocks = 0 @@ -58,66 +52,19 @@ type CortexNoQuerierLimits struct{} var _ v1.Limits = (*CortexNoQuerierLimits)(nil) -func (CortexNoQuerierLimits) MaxQueriersPerUser(user string) int { return 0 } +func (CortexNoQuerierLimits) MaxQueriersPerUser(string) int { return 0 } -// This struct combines several configuration options together to preserve backwards compatibility. -type CombinedFrontendConfig struct { - Handler transport.HandlerConfig `yaml:",inline"` - FrontendV1 v1.Config `yaml:",inline"` - FrontendV2 v2.Config `yaml:",inline"` - - DownstreamURL string `yaml:"downstream_url"` -} - -func (cfg *CombinedFrontendConfig) RegisterFlags(f *flag.FlagSet) { - cfg.Handler.RegisterFlags(f) - cfg.FrontendV1.RegisterFlags(f) - cfg.FrontendV2.RegisterFlags(f) - - f.StringVar(&cfg.DownstreamURL, "frontend.downstream-url", "", "URL of downstream Prometheus.") -} - -// InitFrontend initializes frontend (either V1 -- without scheduler, or V2 -- with scheduler) or no frontend at -// all if downstream Prometheus URL is used instead. +// InitFrontend initializes V1 frontend // // Returned RoundTripper can be wrapped in more round-tripper middlewares, and then eventually registered // into HTTP server using the Handler from this package. Returned RoundTripper is always non-nil // (if there are no errors), and it uses the returned frontend (if any). -func InitFrontend(cfg CombinedFrontendConfig, limits v1.Limits, grpcListenPort int, log log.Logger, reg prometheus.Registerer) (http.RoundTripper, *v1.Frontend, *v2.Frontend, error) { - switch { - case cfg.DownstreamURL != "": - // If the user has specified a downstream Prometheus, then we should use that. - rt, err := NewDownstreamRoundTripper(cfg.DownstreamURL, http.DefaultTransport) - return rt, nil, nil, err - - case cfg.FrontendV2.SchedulerAddress != "": - statVersion.Set("v2") - statWorkerConcurrency.Set(int64(cfg.FrontendV2.WorkerConcurrency)) - - // If query-scheduler address is configured, use Frontend. - if cfg.FrontendV2.Addr == "" { - addr, err := util.GetFirstAddressOf(cfg.FrontendV2.InfNames) - if err != nil { - return nil, nil, nil, errors.Wrap(err, "failed to get frontend address") - } - - cfg.FrontendV2.Addr = addr - } - - if cfg.FrontendV2.Port == 0 { - cfg.FrontendV2.Port = grpcListenPort - } - - fr, err := v2.NewFrontend(cfg.FrontendV2, log, reg) - return transport.AdaptGrpcRoundTripperToHTTPRoundTripper(fr), nil, fr, err - - default: - statVersion.Set("v1") - // No scheduler = use original frontend. - fr, err := v1.New(cfg.FrontendV1, limits, log, reg) - if err != nil { - return nil, nil, nil, err - } - return transport.AdaptGrpcRoundTripperToHTTPRoundTripper(fr), fr, nil, nil +func InitFrontend(cfg v1.Config, limits v1.Limits, log log.Logger, reg prometheus.Registerer) (http.RoundTripper, *v1.Frontend, error) { + statVersion.Set("v1") + // No scheduler = use original frontend. + fr, err := v1.New(cfg, limits, log, reg) + if err != nil { + return nil, nil, err } + return transport.AdaptGrpcRoundTripperToHTTPRoundTripper(fr), fr, nil } diff --git a/modules/frontend/frontend.go b/modules/frontend/frontend.go index d050722440c..c0d9e7d7e62 100644 --- a/modules/frontend/frontend.go +++ b/modules/frontend/frontend.go @@ -11,8 +11,8 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/golang/protobuf/jsonpb" //nolint:all //deprecated - "github.com/golang/protobuf/proto" //nolint:all //deprecated + "github.com/golang/protobuf/jsonpb" //nolint:all //deprecated + "github.com/golang/protobuf/proto" //nolint:all //deprecated "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" diff --git a/modules/frontend/tracebyidsharding.go b/modules/frontend/tracebyidsharding.go index cd66fb942ec..3decb5a13cc 100644 --- a/modules/frontend/tracebyidsharding.go +++ b/modules/frontend/tracebyidsharding.go @@ -13,7 +13,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/golang/protobuf/proto" //nolint:all //deprecated + "github.com/golang/protobuf/proto" //nolint:all //deprecated "github.com/grafana/tempo/modules/querier" "github.com/grafana/tempo/pkg/api" "github.com/grafana/tempo/pkg/model/trace" diff --git a/modules/frontend/v2/frontend.go b/modules/frontend/v2/frontend.go deleted file mode 100644 index 0da04dc6b7c..00000000000 --- a/modules/frontend/v2/frontend.go +++ /dev/null @@ -1,319 +0,0 @@ -package v2 - -import ( - "context" - "flag" - "fmt" - "math/rand" - "net/http" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/grafana/dskit/flagext" - "github.com/grafana/dskit/grpcclient" - "github.com/grafana/dskit/services" - "github.com/grafana/dskit/tenant" - "github.com/opentracing/opentracing-go" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/weaveworks/common/httpgrpc" - "go.uber.org/atomic" - - "github.com/grafana/tempo/modules/frontend/v2/frontendv2pb" - "github.com/grafana/tempo/modules/querier/stats" - "github.com/grafana/tempo/pkg/util/httpgrpcutil" -) - -// Config for a Frontend. -type Config struct { - SchedulerAddress string `yaml:"scheduler_address"` - DNSLookupPeriod time.Duration `yaml:"scheduler_dns_lookup_period"` - WorkerConcurrency int `yaml:"scheduler_worker_concurrency"` - GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"` - - // Used to find local IP address, that is sent to scheduler and querier-worker. - InfNames []string `yaml:"instance_interface_names"` - - // If set, address is not computed from interfaces. - Addr string `yaml:"address" doc:"hidden"` - Port int `doc:"hidden"` -} - -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&cfg.SchedulerAddress, "frontend.scheduler-address", "", "DNS hostname used for finding query-schedulers.") - f.DurationVar(&cfg.DNSLookupPeriod, "frontend.scheduler-dns-lookup-period", 10*time.Second, "How often to resolve the scheduler-address, in order to look for new query-scheduler instances.") - f.IntVar(&cfg.WorkerConcurrency, "frontend.scheduler-worker-concurrency", 5, "Number of concurrent workers forwarding queries to single query-scheduler.") - - cfg.InfNames = []string{"eth0", "en0"} - f.Var((*flagext.StringSlice)(&cfg.InfNames), "frontend.instance-interface-names", "Name of network interface to read address from. This address is sent to query-scheduler and querier, which uses it to send the query response back to query-frontend.") - f.StringVar(&cfg.Addr, "frontend.instance-addr", "", "IP address to advertise to querier (via scheduler) (resolved via interfaces by default).") - f.IntVar(&cfg.Port, "frontend.instance-port", 0, "Port to advertise to querier (via scheduler) (defaults to server.grpc-listen-port).") - - cfg.GRPCClientConfig.RegisterFlagsWithPrefix("frontend.grpc-client-config", f) -} - -// Frontend implements GrpcRoundTripper. It queues HTTP requests, -// dispatches them to backends via gRPC, and handles retries for requests which failed. -type Frontend struct { - services.Service - - cfg Config - log log.Logger - - lastQueryID atomic.Uint64 - - // frontend workers will read from this channel, and send request to scheduler. - requestsCh chan *frontendRequest - - schedulerWorkers *frontendSchedulerWorkers - requests *requestsInProgress -} - -type frontendRequest struct { - queryID uint64 - request *httpgrpc.HTTPRequest - userID string - statsEnabled bool - - cancel context.CancelFunc - - enqueue chan enqueueResult - response chan *frontendv2pb.QueryResultRequest -} - -type enqueueStatus int - -const ( - // Sent to scheduler successfully, and frontend should wait for response now. - waitForResponse enqueueStatus = iota - - // Failed to forward request to scheduler, frontend will try again. - failed -) - -type enqueueResult struct { - status enqueueStatus - - cancelCh chan<- uint64 // Channel that can be used for request cancellation. If nil, cancellation is not possible. -} - -// NewFrontend creates a new frontend. -func NewFrontend(cfg Config, log log.Logger, reg prometheus.Registerer) (*Frontend, error) { - requestsCh := make(chan *frontendRequest) - - schedulerWorkers, err := newFrontendSchedulerWorkers(cfg, fmt.Sprintf("%s:%d", cfg.Addr, cfg.Port), requestsCh, log) - if err != nil { - return nil, err - } - - f := &Frontend{ - cfg: cfg, - log: log, - requestsCh: requestsCh, - schedulerWorkers: schedulerWorkers, - requests: newRequestsInProgress(), - } - // Randomize to avoid getting responses from queries sent before restart, which could lead to mixing results - // between different queries. Note that frontend verifies the user, so it cannot leak results between tenants. - // This isn't perfect, but better than nothing. - f.lastQueryID.Store(rand.Uint64()) - - promauto.With(reg).NewGaugeFunc(prometheus.GaugeOpts{ - Name: "cortex_query_frontend_queries_in_progress", - Help: "Number of queries in progress handled by this frontend.", - }, func() float64 { - return float64(f.requests.count()) - }) - - promauto.With(reg).NewGaugeFunc(prometheus.GaugeOpts{ - Name: "cortex_query_frontend_connected_schedulers", - Help: "Number of schedulers this frontend is connected to.", - }, func() float64 { - return float64(f.schedulerWorkers.getWorkersCount()) - }) - - f.Service = services.NewIdleService(f.starting, f.stopping) - return f, nil -} - -func (f *Frontend) starting(ctx context.Context) error { - return errors.Wrap(services.StartAndAwaitRunning(ctx, f.schedulerWorkers), "failed to start frontend scheduler workers") -} - -func (f *Frontend) stopping(_ error) error { - return errors.Wrap(services.StopAndAwaitTerminated(context.Background(), f.schedulerWorkers), "failed to stop frontend scheduler workers") -} - -// RoundTripGRPC round trips a proto (instead of a HTTP request). -func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { - if s := f.State(); s != services.Running { - return nil, fmt.Errorf("frontend not running: %v", s) - } - - tenantIDs, err := tenant.TenantIDs(ctx) - if err != nil { - return nil, err - } - userID := tenant.JoinTenantIDs(tenantIDs) - - // Propagate trace context in gRPC too - this will be ignored if using HTTP. - tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx) - if tracer != nil && span != nil { - carrier := (*httpgrpcutil.HttpgrpcHeadersCarrier)(req) - if err := tracer.Inject(span.Context(), opentracing.HTTPHeaders, carrier); err != nil { - return nil, err - } - } - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - freq := &frontendRequest{ - queryID: f.lastQueryID.Inc(), - request: req, - userID: userID, - statsEnabled: stats.IsEnabled(ctx), - - cancel: cancel, - - // Buffer of 1 to ensure response or error can be written to the channel - // even if this goroutine goes away due to client context cancellation. - enqueue: make(chan enqueueResult, 1), - response: make(chan *frontendv2pb.QueryResultRequest, 1), - } - - f.requests.put(freq) - defer f.requests.delete(freq.queryID) - - retries := f.cfg.WorkerConcurrency + 1 // To make sure we hit at least two different schedulers. - -enqueueAgain: - select { - case <-ctx.Done(): - return nil, ctx.Err() - - case f.requestsCh <- freq: - // Enqueued, let's wait for response. - } - - var cancelCh chan<- uint64 - - select { - case <-ctx.Done(): - return nil, ctx.Err() - - case enqRes := <-freq.enqueue: - if enqRes.status == waitForResponse { - cancelCh = enqRes.cancelCh - break // go wait for response. - } else if enqRes.status == failed { - retries-- - if retries > 0 { - goto enqueueAgain - } - } - - return nil, httpgrpc.Errorf(http.StatusInternalServerError, "failed to enqueue request") - } - - select { - case <-ctx.Done(): - if cancelCh != nil { - select { - case cancelCh <- freq.queryID: - // cancellation sent. - default: - // failed to cancel, ignore. - } - } - return nil, ctx.Err() - - case resp := <-freq.response: - if stats.ShouldTrackHTTPGRPCResponse(resp.HttpResponse) { - stats := stats.FromContext(ctx) - stats.Merge(resp.Stats) // Safe if stats is nil. - } - - return resp.HttpResponse, nil - } -} - -func (f *Frontend) QueryResult(ctx context.Context, qrReq *frontendv2pb.QueryResultRequest) (*frontendv2pb.QueryResultResponse, error) { - tenantIDs, err := tenant.TenantIDs(ctx) - if err != nil { - return nil, err - } - userID := tenant.JoinTenantIDs(tenantIDs) - - req := f.requests.get(qrReq.QueryID) - // It is possible that some old response belonging to different user was received, if frontend has restarted. - // To avoid leaking query results between users, we verify the user here. - // To avoid mixing results from different queries, we randomize queryID counter on start. - if req != nil && req.userID == userID { - select { - case req.response <- qrReq: - // Should always be possible, unless QueryResult is called multiple times with the same queryID. - default: - level.Warn(f.log).Log("msg", "failed to write query result to the response channel", "queryID", qrReq.QueryID, "user", userID) - } - } - - return &frontendv2pb.QueryResultResponse{}, nil -} - -// CheckReady determines if the query frontend is ready. Function parameters/return -// chosen to match the same method in the ingester -func (f *Frontend) CheckReady(_ context.Context) error { - workers := f.schedulerWorkers.getWorkersCount() - - // If frontend is connected to at least one scheduler, we are ready. - if workers > 0 { - return nil - } - - msg := fmt.Sprintf("not ready: number of schedulers this worker is connected to is %d", workers) - level.Info(f.log).Log("msg", msg) - return errors.New(msg) -} - -type requestsInProgress struct { - mu sync.Mutex - requests map[uint64]*frontendRequest -} - -func newRequestsInProgress() *requestsInProgress { - return &requestsInProgress{ - requests: map[uint64]*frontendRequest{}, - } -} - -func (r *requestsInProgress) count() int { - r.mu.Lock() - defer r.mu.Unlock() - - return len(r.requests) -} - -func (r *requestsInProgress) put(req *frontendRequest) { - r.mu.Lock() - defer r.mu.Unlock() - - r.requests[req.queryID] = req -} - -func (r *requestsInProgress) delete(queryID uint64) { - r.mu.Lock() - defer r.mu.Unlock() - - delete(r.requests, queryID) -} - -func (r *requestsInProgress) get(queryID uint64) *frontendRequest { - r.mu.Lock() - defer r.mu.Unlock() - - return r.requests[queryID] -} diff --git a/modules/frontend/v2/frontend_scheduler_worker.go b/modules/frontend/v2/frontend_scheduler_worker.go deleted file mode 100644 index 7b151ed8758..00000000000 --- a/modules/frontend/v2/frontend_scheduler_worker.go +++ /dev/null @@ -1,329 +0,0 @@ -package v2 - -import ( - "context" - "net/http" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/grafana/dskit/backoff" - "github.com/grafana/dskit/services" - "github.com/pkg/errors" - "github.com/weaveworks/common/httpgrpc" - "google.golang.org/grpc" - - "github.com/grafana/tempo/modules/frontend/v2/frontendv2pb" - "github.com/grafana/tempo/pkg/scheduler/schedulerpb" - "github.com/grafana/tempo/pkg/util" -) - -type frontendSchedulerWorkers struct { - services.Service - - cfg Config - log log.Logger - frontendAddress string - - // Channel with requests that should be forwarded to the scheduler. - requestsCh <-chan *frontendRequest - - watcher services.Service - - mu sync.Mutex - // Set to nil when stop is called... no more workers are created afterwards. - workers map[string]*frontendSchedulerWorker -} - -func newFrontendSchedulerWorkers(cfg Config, frontendAddress string, requestsCh <-chan *frontendRequest, log log.Logger) (*frontendSchedulerWorkers, error) { - f := &frontendSchedulerWorkers{ - cfg: cfg, - log: log, - frontendAddress: frontendAddress, - requestsCh: requestsCh, - workers: map[string]*frontendSchedulerWorker{}, - } - - w, err := util.NewDNSWatcher(cfg.SchedulerAddress, cfg.DNSLookupPeriod, f) - if err != nil { - return nil, err - } - - f.watcher = w - f.Service = services.NewIdleService(f.starting, f.stopping) - return f, nil -} - -func (f *frontendSchedulerWorkers) starting(ctx context.Context) error { - return services.StartAndAwaitRunning(ctx, f.watcher) -} - -func (f *frontendSchedulerWorkers) stopping(_ error) error { - err := services.StopAndAwaitTerminated(context.Background(), f.watcher) - - f.mu.Lock() - defer f.mu.Unlock() - - for _, w := range f.workers { - w.stop() - } - f.workers = nil - - return err -} - -func (f *frontendSchedulerWorkers) AddressAdded(address string) { - f.mu.Lock() - ws := f.workers - w := f.workers[address] - f.mu.Unlock() - - // Already stopped or we already have worker for this address. - if ws == nil || w != nil { - return - } - - level.Info(f.log).Log("msg", "adding connection to scheduler", "addr", address) - conn, err := f.connectToScheduler(context.Background(), address) - if err != nil { - level.Error(f.log).Log("msg", "error connecting to scheduler", "addr", address, "err", err) - return - } - - // No worker for this address yet, start a new one. - w = newFrontendSchedulerWorker(conn, address, f.frontendAddress, f.requestsCh, f.cfg.WorkerConcurrency, f.log) - - f.mu.Lock() - defer f.mu.Unlock() - - // Can be nil if stopping has been called already. - if f.workers != nil { - f.workers[address] = w - w.start() - } -} - -func (f *frontendSchedulerWorkers) AddressRemoved(address string) { - level.Info(f.log).Log("msg", "removing connection to scheduler", "addr", address) - - f.mu.Lock() - // This works fine if f.workers is nil already. - w := f.workers[address] - delete(f.workers, address) - f.mu.Unlock() - - if w != nil { - w.stop() - } -} - -// Get number of workers. -func (f *frontendSchedulerWorkers) getWorkersCount() int { - f.mu.Lock() - defer f.mu.Unlock() - - return len(f.workers) -} - -func (f *frontendSchedulerWorkers) connectToScheduler(ctx context.Context, address string) (*grpc.ClientConn, error) { - // Because we only use single long-running method, it doesn't make sense to inject user ID, send over tracing or add metrics. - opts, err := f.cfg.GRPCClientConfig.DialOption(nil, nil) - if err != nil { - return nil, err - } - - conn, err := grpc.DialContext(ctx, address, opts...) - if err != nil { - return nil, err - } - return conn, nil -} - -// Worker managing single gRPC connection to Scheduler. Each worker starts multiple goroutines for forwarding -// requests and cancellations to scheduler. -type frontendSchedulerWorker struct { - log log.Logger - - conn *grpc.ClientConn - concurrency int - schedulerAddr string - frontendAddr string - - // Context and cancellation used by individual goroutines. - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup - - // Shared between all frontend workers. - requestCh <-chan *frontendRequest - - // Cancellation requests for this scheduler are received via this channel. It is passed to frontend after - // query has been enqueued to scheduler. - cancelCh chan uint64 -} - -func newFrontendSchedulerWorker(conn *grpc.ClientConn, schedulerAddr string, frontendAddr string, requestCh <-chan *frontendRequest, concurrency int, log log.Logger) *frontendSchedulerWorker { - w := &frontendSchedulerWorker{ - log: log, - conn: conn, - concurrency: concurrency, - schedulerAddr: schedulerAddr, - frontendAddr: frontendAddr, - requestCh: requestCh, - cancelCh: make(chan uint64), - } - w.ctx, w.cancel = context.WithCancel(context.Background()) - - return w -} - -func (w *frontendSchedulerWorker) start() { - client := schedulerpb.NewSchedulerForFrontendClient(w.conn) - for i := 0; i < w.concurrency; i++ { - w.wg.Add(1) - go func() { - defer w.wg.Done() - w.runOne(w.ctx, client) - }() - } -} - -func (w *frontendSchedulerWorker) stop() { - w.cancel() - w.wg.Wait() - if err := w.conn.Close(); err != nil { - level.Error(w.log).Log("msg", "error while closing connection to scheduler", "err", err) - } -} - -func (w *frontendSchedulerWorker) runOne(ctx context.Context, client schedulerpb.SchedulerForFrontendClient) { - backoffConfig := backoff.Config{ - MinBackoff: 50 * time.Millisecond, - MaxBackoff: 1 * time.Second, - } - - backoff := backoff.New(ctx, backoffConfig) - for backoff.Ongoing() { - loop, loopErr := client.FrontendLoop(ctx) - if loopErr != nil { - level.Error(w.log).Log("msg", "error contacting scheduler", "err", loopErr, "addr", w.schedulerAddr) - backoff.Wait() - continue - } - - loopErr = w.schedulerLoop(loop) - if closeErr := loop.CloseSend(); closeErr != nil { - level.Debug(w.log).Log("msg", "failed to close frontend loop", "err", loopErr, "addr", w.schedulerAddr) - } - - if loopErr != nil { - level.Error(w.log).Log("msg", "error sending requests to scheduler", "err", loopErr, "addr", w.schedulerAddr) - backoff.Wait() - continue - } - - backoff.Reset() - } -} - -func (w *frontendSchedulerWorker) schedulerLoop(loop schedulerpb.SchedulerForFrontend_FrontendLoopClient) error { - if err := loop.Send(&schedulerpb.FrontendToScheduler{ - Type: schedulerpb.INIT, - FrontendAddress: w.frontendAddr, - }); err != nil { - return err - } - - if resp, err := loop.Recv(); err != nil || resp.Status != schedulerpb.OK { - if err != nil { - return err - } - return errors.Errorf("unexpected status received for init: %v", resp.Status) - } - - ctx := loop.Context() - - for { - select { - case <-ctx.Done(): - // No need to report error if our internal context is canceled. This can happen during shutdown, - // or when scheduler is no longer resolvable. (It would be nice if this context reported "done" also when - // connection scheduler stops the call, but that doesn't seem to be the case). - // - // Reporting error here would delay reopening the stream (if the worker context is not done yet). - level.Debug(w.log).Log("msg", "stream context finished", "err", ctx.Err()) - return nil - - case req := <-w.requestCh: - err := loop.Send(&schedulerpb.FrontendToScheduler{ - Type: schedulerpb.ENQUEUE, - QueryID: req.queryID, - UserID: req.userID, - HttpRequest: req.request, - FrontendAddress: w.frontendAddr, - StatsEnabled: req.statsEnabled, - }) - - if err != nil { - req.enqueue <- enqueueResult{status: failed} - return err - } - - resp, err := loop.Recv() - if err != nil { - req.enqueue <- enqueueResult{status: failed} - return err - } - - switch resp.Status { - case schedulerpb.OK: - req.enqueue <- enqueueResult{status: waitForResponse, cancelCh: w.cancelCh} - // Response will come from querier. - - case schedulerpb.SHUTTING_DOWN: - // Scheduler is shutting down, report failure to enqueue and stop this loop. - req.enqueue <- enqueueResult{status: failed} - return errors.New("scheduler is shutting down") - - case schedulerpb.ERROR: - req.enqueue <- enqueueResult{status: waitForResponse} - req.response <- &frontendv2pb.QueryResultRequest{ - HttpResponse: &httpgrpc.HTTPResponse{ - Code: http.StatusInternalServerError, - Body: []byte(err.Error()), - }, - } - - case schedulerpb.TOO_MANY_REQUESTS_PER_TENANT: - req.enqueue <- enqueueResult{status: waitForResponse} - req.response <- &frontendv2pb.QueryResultRequest{ - HttpResponse: &httpgrpc.HTTPResponse{ - Code: http.StatusTooManyRequests, - Body: []byte("too many outstanding requests"), - }, - } - } - - case reqID := <-w.cancelCh: - err := loop.Send(&schedulerpb.FrontendToScheduler{ - Type: schedulerpb.CANCEL, - QueryID: reqID, - }) - - if err != nil { - return err - } - - resp, err := loop.Recv() - if err != nil { - return err - } - - // Scheduler may be shutting down, report that. - if resp.Status != schedulerpb.OK { - return errors.Errorf("unexpected status received for cancellation: %v", resp.Status) - } - } - } -} diff --git a/modules/frontend/v2/frontendv2pb/frontend.pb.go b/modules/frontend/v2/frontendv2pb/frontend.pb.go deleted file mode 100644 index 324aa78525d..00000000000 --- a/modules/frontend/v2/frontendv2pb/frontend.pb.go +++ /dev/null @@ -1,636 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: modules/frontend/v2/frontendv2pb/frontend.proto - -package frontendv2pb - -import ( - context "context" - fmt "fmt" - _ "github.com/gogo/protobuf/gogoproto" - proto "github.com/gogo/protobuf/proto" - stats "github.com/grafana/tempo/modules/querier/stats" - httpgrpc "github.com/weaveworks/common/httpgrpc" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - io "io" - math "math" - math_bits "math/bits" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -type QueryResultRequest struct { - QueryID uint64 `protobuf:"varint,1,opt,name=queryID,proto3" json:"queryID,omitempty"` - HttpResponse *httpgrpc.HTTPResponse `protobuf:"bytes,2,opt,name=httpResponse,proto3" json:"httpResponse,omitempty"` - Stats *stats.Stats `protobuf:"bytes,3,opt,name=stats,proto3" json:"stats,omitempty"` -} - -func (m *QueryResultRequest) Reset() { *m = QueryResultRequest{} } -func (m *QueryResultRequest) String() string { return proto.CompactTextString(m) } -func (*QueryResultRequest) ProtoMessage() {} -func (*QueryResultRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_275b013a45bbea59, []int{0} -} -func (m *QueryResultRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *QueryResultRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_QueryResultRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *QueryResultRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryResultRequest.Merge(m, src) -} -func (m *QueryResultRequest) XXX_Size() int { - return m.Size() -} -func (m *QueryResultRequest) XXX_DiscardUnknown() { - xxx_messageInfo_QueryResultRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryResultRequest proto.InternalMessageInfo - -func (m *QueryResultRequest) GetQueryID() uint64 { - if m != nil { - return m.QueryID - } - return 0 -} - -func (m *QueryResultRequest) GetHttpResponse() *httpgrpc.HTTPResponse { - if m != nil { - return m.HttpResponse - } - return nil -} - -func (m *QueryResultRequest) GetStats() *stats.Stats { - if m != nil { - return m.Stats - } - return nil -} - -type QueryResultResponse struct { -} - -func (m *QueryResultResponse) Reset() { *m = QueryResultResponse{} } -func (m *QueryResultResponse) String() string { return proto.CompactTextString(m) } -func (*QueryResultResponse) ProtoMessage() {} -func (*QueryResultResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_275b013a45bbea59, []int{1} -} -func (m *QueryResultResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *QueryResultResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_QueryResultResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *QueryResultResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryResultResponse.Merge(m, src) -} -func (m *QueryResultResponse) XXX_Size() int { - return m.Size() -} -func (m *QueryResultResponse) XXX_DiscardUnknown() { - xxx_messageInfo_QueryResultResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryResultResponse proto.InternalMessageInfo - -func init() { - proto.RegisterType((*QueryResultRequest)(nil), "frontendv2pb.QueryResultRequest") - proto.RegisterType((*QueryResultResponse)(nil), "frontendv2pb.QueryResultResponse") -} - -func init() { - proto.RegisterFile("modules/frontend/v2/frontendv2pb/frontend.proto", fileDescriptor_275b013a45bbea59) -} - -var fileDescriptor_275b013a45bbea59 = []byte{ - // 320 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0x3d, 0x4f, 0x3a, 0x41, - 0x10, 0xc6, 0x6f, 0xff, 0x7f, 0x5f, 0x92, 0x85, 0x6a, 0x8d, 0xe6, 0x42, 0x71, 0x81, 0x2b, 0x0c, - 0x8d, 0xb7, 0xc9, 0x69, 0x65, 0x69, 0x0c, 0xd1, 0x4e, 0x56, 0x2a, 0x3b, 0x0e, 0x86, 0x03, 0xe5, - 0x6e, 0x8e, 0x7d, 0x81, 0xf0, 0x25, 0x8c, 0x1f, 0xcb, 0x92, 0xd2, 0xd2, 0xc0, 0x17, 0x31, 0xec, - 0xc2, 0xe5, 0x88, 0x89, 0xcd, 0x66, 0x9e, 0xcc, 0xef, 0xc9, 0x33, 0x33, 0x4b, 0x79, 0x86, 0x43, - 0x33, 0x05, 0xc5, 0x47, 0x12, 0x73, 0x0d, 0xf9, 0x90, 0xcf, 0xe3, 0xb2, 0x9e, 0xc7, 0x45, 0x52, - 0x8a, 0xa8, 0x90, 0xa8, 0x91, 0xd5, 0xab, 0xcd, 0xc6, 0x55, 0x3a, 0xd1, 0x63, 0x93, 0x44, 0x03, - 0xcc, 0x78, 0x8a, 0x29, 0x72, 0x0b, 0x25, 0x66, 0x64, 0x95, 0x15, 0xb6, 0x72, 0xe6, 0xc6, 0x4d, - 0x05, 0x5f, 0x40, 0x7f, 0x0e, 0x0b, 0x94, 0x6f, 0x8a, 0x0f, 0x30, 0xcb, 0x30, 0xe7, 0x63, 0xad, - 0x8b, 0x54, 0x16, 0x83, 0xb2, 0xd8, 0xb9, 0x5a, 0xfb, 0x19, 0x67, 0x06, 0xe4, 0x04, 0x24, 0x57, - 0xba, 0xaf, 0x95, 0x7b, 0x1d, 0x12, 0xbe, 0x13, 0xca, 0xba, 0x06, 0xe4, 0x52, 0x80, 0x32, 0x53, - 0x2d, 0x60, 0x66, 0x40, 0x69, 0xe6, 0xd3, 0xd3, 0xad, 0x67, 0xf9, 0x78, 0xef, 0x93, 0x26, 0x69, - 0x1f, 0x89, 0xbd, 0x64, 0xb7, 0xb4, 0xbe, 0x4d, 0x11, 0xa0, 0x0a, 0xcc, 0x15, 0xf8, 0xff, 0x9a, - 0xa4, 0x5d, 0x8b, 0x2f, 0xa2, 0x32, 0xfa, 0xa1, 0xd7, 0x7b, 0xda, 0x77, 0xc5, 0x01, 0xcb, 0x42, - 0x7a, 0x6c, 0xb3, 0xfd, 0xff, 0xd6, 0x54, 0x8f, 0xdc, 0x24, 0xcf, 0xdb, 0x57, 0xb8, 0x56, 0x78, - 0x4e, 0xcf, 0x0e, 0xe6, 0x71, 0xd6, 0xf8, 0x95, 0xb2, 0xce, 0xee, 0x7e, 0x1d, 0x94, 0x5d, 0xb7, - 0x0f, 0xeb, 0xd1, 0x5a, 0x05, 0x66, 0xcd, 0xa8, 0x7a, 0xe3, 0xe8, 0xf7, 0x5e, 0x8d, 0xd6, 0x1f, - 0x84, 0x4b, 0x0a, 0xbd, 0xbb, 0xcb, 0xcf, 0x75, 0x40, 0x56, 0xeb, 0x80, 0x7c, 0xaf, 0x03, 0xf2, - 0xb1, 0x09, 0xbc, 0xd5, 0x26, 0xf0, 0xbe, 0x36, 0x81, 0xf7, 0x72, 0xf0, 0x87, 0xc9, 0x89, 0x3d, - 0xe1, 0xf5, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xf5, 0xbc, 0x57, 0x71, 0x0b, 0x02, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// FrontendForQuerierClient is the client API for FrontendForQuerier service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type FrontendForQuerierClient interface { - QueryResult(ctx context.Context, in *QueryResultRequest, opts ...grpc.CallOption) (*QueryResultResponse, error) -} - -type frontendForQuerierClient struct { - cc *grpc.ClientConn -} - -func NewFrontendForQuerierClient(cc *grpc.ClientConn) FrontendForQuerierClient { - return &frontendForQuerierClient{cc} -} - -func (c *frontendForQuerierClient) QueryResult(ctx context.Context, in *QueryResultRequest, opts ...grpc.CallOption) (*QueryResultResponse, error) { - out := new(QueryResultResponse) - err := c.cc.Invoke(ctx, "/frontendv2pb.FrontendForQuerier/QueryResult", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// FrontendForQuerierServer is the server API for FrontendForQuerier service. -type FrontendForQuerierServer interface { - QueryResult(context.Context, *QueryResultRequest) (*QueryResultResponse, error) -} - -// UnimplementedFrontendForQuerierServer can be embedded to have forward compatible implementations. -type UnimplementedFrontendForQuerierServer struct { -} - -func (*UnimplementedFrontendForQuerierServer) QueryResult(ctx context.Context, req *QueryResultRequest) (*QueryResultResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method QueryResult not implemented") -} - -func RegisterFrontendForQuerierServer(s *grpc.Server, srv FrontendForQuerierServer) { - s.RegisterService(&_FrontendForQuerier_serviceDesc, srv) -} - -func _FrontendForQuerier_QueryResult_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(QueryResultRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(FrontendForQuerierServer).QueryResult(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/frontendv2pb.FrontendForQuerier/QueryResult", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(FrontendForQuerierServer).QueryResult(ctx, req.(*QueryResultRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _FrontendForQuerier_serviceDesc = grpc.ServiceDesc{ - ServiceName: "frontendv2pb.FrontendForQuerier", - HandlerType: (*FrontendForQuerierServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "QueryResult", - Handler: _FrontendForQuerier_QueryResult_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "modules/frontend/v2/frontendv2pb/frontend.proto", -} - -func (m *QueryResultRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *QueryResultRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *QueryResultRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Stats != nil { - { - size, err := m.Stats.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintFrontend(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a - } - if m.HttpResponse != nil { - { - size, err := m.HttpResponse.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintFrontend(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - if m.QueryID != 0 { - i = encodeVarintFrontend(dAtA, i, uint64(m.QueryID)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *QueryResultResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *QueryResultResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *QueryResultResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - return len(dAtA) - i, nil -} - -func encodeVarintFrontend(dAtA []byte, offset int, v uint64) int { - offset -= sovFrontend(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *QueryResultRequest) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.QueryID != 0 { - n += 1 + sovFrontend(uint64(m.QueryID)) - } - if m.HttpResponse != nil { - l = m.HttpResponse.Size() - n += 1 + l + sovFrontend(uint64(l)) - } - if m.Stats != nil { - l = m.Stats.Size() - n += 1 + l + sovFrontend(uint64(l)) - } - return n -} - -func (m *QueryResultResponse) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - return n -} - -func sovFrontend(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozFrontend(x uint64) (n int) { - return sovFrontend(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *QueryResultRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFrontend - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: QueryResultRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: QueryResultRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field QueryID", wireType) - } - m.QueryID = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFrontend - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.QueryID |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field HttpResponse", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFrontend - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthFrontend - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthFrontend - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.HttpResponse == nil { - m.HttpResponse = &httpgrpc.HTTPResponse{} - } - if err := m.HttpResponse.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Stats", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFrontend - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthFrontend - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthFrontend - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Stats == nil { - m.Stats = &stats.Stats{} - } - if err := m.Stats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipFrontend(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthFrontend - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *QueryResultResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFrontend - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: QueryResultResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: QueryResultResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := skipFrontend(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthFrontend - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipFrontend(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - depth := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowFrontend - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowFrontend - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - case 1: - iNdEx += 8 - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowFrontend - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthFrontend - } - iNdEx += length - case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupFrontend - } - depth-- - case 5: - iNdEx += 4 - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - if iNdEx < 0 { - return 0, ErrInvalidLengthFrontend - } - if depth == 0 { - return iNdEx, nil - } - } - return 0, io.ErrUnexpectedEOF -} - -var ( - ErrInvalidLengthFrontend = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowFrontend = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupFrontend = fmt.Errorf("proto: unexpected end of group") -) diff --git a/modules/frontend/v2/frontendv2pb/frontend.proto b/modules/frontend/v2/frontendv2pb/frontend.proto deleted file mode 100644 index 0523b498423..00000000000 --- a/modules/frontend/v2/frontendv2pb/frontend.proto +++ /dev/null @@ -1,28 +0,0 @@ -syntax = "proto3"; - -package frontendv2pb; - -option go_package = "frontendv2pb"; - -import "github.com/gogo/protobuf/gogoproto/gogo.proto"; -import "github.com/weaveworks/common/httpgrpc/httpgrpc.proto"; -import "modules/querier/stats/stats.proto"; - -option (gogoproto.marshaler_all) = true; -option (gogoproto.unmarshaler_all) = true; - -// Frontend interface exposed to Queriers. Used by queriers to report back the result of the query. -service FrontendForQuerier { - rpc QueryResult (QueryResultRequest) returns (QueryResultResponse) { }; -} - -message QueryResultRequest { - uint64 queryID = 1; - httpgrpc.HTTPResponse httpResponse = 2; - stats.Stats stats = 3; - - // There is no userID field here, because Querier puts userID into the context when - // calling QueryResult, and that is where Frontend expects to find it. -} - -message QueryResultResponse { } diff --git a/modules/querier/worker/scheduler_processor.go b/modules/querier/worker/scheduler_processor.go deleted file mode 100644 index 8899dfbe102..00000000000 --- a/modules/querier/worker/scheduler_processor.go +++ /dev/null @@ -1,227 +0,0 @@ -package worker - -import ( - "context" - "fmt" - "net/http" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/grafana/dskit/backoff" - "github.com/grafana/dskit/grpcclient" - "github.com/grafana/dskit/ring/client" - "github.com/grafana/dskit/services" - otgrpc "github.com/opentracing-contrib/go-grpc" - "github.com/opentracing/opentracing-go" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/weaveworks/common/httpgrpc" - "github.com/weaveworks/common/middleware" - "github.com/weaveworks/common/user" - "google.golang.org/grpc" - "google.golang.org/grpc/health/grpc_health_v1" - - "github.com/grafana/tempo/modules/frontend/v2/frontendv2pb" - querier_stats "github.com/grafana/tempo/modules/querier/stats" - "github.com/grafana/tempo/pkg/scheduler/schedulerpb" - "github.com/grafana/tempo/pkg/util/httpgrpcutil" - util_log "github.com/grafana/tempo/pkg/util/log" -) - -func newSchedulerProcessor(cfg Config, handler RequestHandler, log log.Logger, reg prometheus.Registerer) (*schedulerProcessor, []services.Service) { - p := &schedulerProcessor{ - log: log, - handler: handler, - maxMessageSize: cfg.GRPCClientConfig.MaxSendMsgSize, - querierID: cfg.QuerierID, - grpcConfig: cfg.GRPCClientConfig, - - frontendClientRequestDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ - Name: "cortex_querier_query_frontend_request_duration_seconds", - Help: "Time spend doing requests to frontend.", - Buckets: prometheus.ExponentialBuckets(0.001, 4, 6), - }, []string{"operation", "status_code"}), - } - - frontendClientsGauge := promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_querier_query_frontend_clients", - Help: "The current number of clients connected to query-frontend.", - }) - - poolConfig := client.PoolConfig{ - CheckInterval: 5 * time.Second, - HealthCheckEnabled: true, - HealthCheckTimeout: 1 * time.Second, - } - - p.frontendPool = client.NewPool("frontend", poolConfig, nil, p.createFrontendClient, frontendClientsGauge, log) - return p, []services.Service{p.frontendPool} -} - -// Handles incoming queries from query-scheduler. -type schedulerProcessor struct { - log log.Logger - handler RequestHandler - grpcConfig grpcclient.Config - maxMessageSize int - querierID string - - frontendPool *client.Pool - frontendClientRequestDuration *prometheus.HistogramVec -} - -// notifyShutdown implements processor. -func (sp *schedulerProcessor) notifyShutdown(ctx context.Context, conn *grpc.ClientConn, address string) { - client := schedulerpb.NewSchedulerForQuerierClient(conn) - - req := &schedulerpb.NotifyQuerierShutdownRequest{QuerierID: sp.querierID} - if _, err := client.NotifyQuerierShutdown(ctx, req); err != nil { - // Since we're shutting down there's nothing we can do except logging it. - level.Warn(sp.log).Log("msg", "failed to notify querier shutdown to query-scheduler", "address", address, "err", err) - } -} - -func (sp *schedulerProcessor) processQueriesOnSingleStream(ctx context.Context, conn *grpc.ClientConn, address string) { - schedulerClient := schedulerpb.NewSchedulerForQuerierClient(conn) - - backoff := backoff.New(ctx, processorBackoffConfig) - for backoff.Ongoing() { - c, err := schedulerClient.QuerierLoop(ctx) - if err == nil { - err = c.Send(&schedulerpb.QuerierToScheduler{QuerierID: sp.querierID}) - } - - if err != nil { - level.Error(sp.log).Log("msg", "error contacting scheduler", "err", err, "addr", address) - backoff.Wait() - continue - } - - if err := sp.querierLoop(c, address); err != nil { - level.Error(sp.log).Log("msg", "error processing requests from scheduler", "err", err, "addr", address) - backoff.Wait() - continue - } - - backoff.Reset() - } -} - -// process loops processing requests on an established stream. -func (sp *schedulerProcessor) querierLoop(c schedulerpb.SchedulerForQuerier_QuerierLoopClient, address string) error { - // Build a child context so we can cancel a query when the stream is closed. - ctx, cancel := context.WithCancel(c.Context()) - defer cancel() - - for { - request, err := c.Recv() - if err != nil { - return err - } - - // Handle the request on a "background" goroutine, so we go back to - // blocking on c.Recv(). This allows us to detect the stream closing - // and cancel the query. We don't actually handle queries in parallel - // here, as we're running in lock step with the server - each Recv is - // paired with a Send. - go func() { - // We need to inject user into context for sending response back. - ctx := user.InjectOrgID(ctx, request.UserID) - - tracer := opentracing.GlobalTracer() - // Ignore errors here. If we cannot get parent span, we just don't create new one. - parentSpanContext, _ := httpgrpcutil.GetParentSpanForRequest(tracer, request.HttpRequest) - if parentSpanContext != nil { - queueSpan, spanCtx := opentracing.StartSpanFromContextWithTracer(ctx, tracer, "querier_processor_runRequest", opentracing.ChildOf(parentSpanContext)) - defer queueSpan.Finish() - - ctx = spanCtx - } - logger := util_log.WithContext(ctx, sp.log) - - sp.runRequest(ctx, logger, request.QueryID, request.FrontendAddress, request.StatsEnabled, request.HttpRequest) - - // Report back to scheduler that processing of the query has finished. - if err := c.Send(&schedulerpb.QuerierToScheduler{}); err != nil { - level.Error(logger).Log("msg", "error notifying scheduler about finished query", "err", err, "addr", address) - } - }() - } -} - -func (sp *schedulerProcessor) runRequest(ctx context.Context, logger log.Logger, queryID uint64, frontendAddress string, statsEnabled bool, request *httpgrpc.HTTPRequest) { - var stats *querier_stats.Stats - if statsEnabled { - stats, ctx = querier_stats.ContextWithEmptyStats(ctx) - } - - response, err := sp.handler.Handle(ctx, request) - if err != nil { - var ok bool - response, ok = httpgrpc.HTTPResponseFromError(err) - if !ok { - response = &httpgrpc.HTTPResponse{ - Code: http.StatusInternalServerError, - Body: []byte(err.Error()), - } - } - } - - // Ensure responses that are too big are not retried. - if len(response.Body) >= sp.maxMessageSize { - level.Error(logger).Log("msg", "response larger than max message size", "size", len(response.Body), "maxMessageSize", sp.maxMessageSize) - - errMsg := fmt.Sprintf("response larger than the max message size (%d vs %d)", len(response.Body), sp.maxMessageSize) - response = &httpgrpc.HTTPResponse{ - Code: http.StatusRequestEntityTooLarge, - Body: []byte(errMsg), - } - } - - c, err := sp.frontendPool.GetClientFor(frontendAddress) - if err == nil { - // Response is empty and uninteresting. - _, err = c.(frontendv2pb.FrontendForQuerierClient).QueryResult(ctx, &frontendv2pb.QueryResultRequest{ - QueryID: queryID, - HttpResponse: response, - Stats: stats, - }) - } - if err != nil { - level.Error(logger).Log("msg", "error notifying frontend about finished query", "err", err, "frontend", frontendAddress) - } -} - -func (sp *schedulerProcessor) createFrontendClient(addr string) (client.PoolClient, error) { - opts, err := sp.grpcConfig.DialOption([]grpc.UnaryClientInterceptor{ - otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), - middleware.ClientUserHeaderInterceptor, - middleware.UnaryClientInstrumentInterceptor(sp.frontendClientRequestDuration), - }, nil) - - if err != nil { - return nil, err - } - - conn, err := grpc.Dial(addr, opts...) - if err != nil { - return nil, err - } - - return &frontendClient{ - FrontendForQuerierClient: frontendv2pb.NewFrontendForQuerierClient(conn), - HealthClient: grpc_health_v1.NewHealthClient(conn), - conn: conn, - }, nil -} - -type frontendClient struct { - frontendv2pb.FrontendForQuerierClient - grpc_health_v1.HealthClient - conn *grpc.ClientConn -} - -func (fc *frontendClient) Close() error { - return fc.conn.Close() -} diff --git a/modules/querier/worker/worker.go b/modules/querier/worker/worker.go index e1279e9bd61..3ed66a49672 100644 --- a/modules/querier/worker/worker.go +++ b/modules/querier/worker/worker.go @@ -20,9 +20,8 @@ import ( ) type Config struct { - FrontendAddress string `yaml:"frontend_address"` - SchedulerAddress string `yaml:"scheduler_address"` - DNSLookupPeriod time.Duration `yaml:"dns_lookup_duration"` + FrontendAddress string `yaml:"frontend_address"` + DNSLookupPeriod time.Duration `yaml:"dns_lookup_duration"` Parallelism int `yaml:"parallelism"` MatchMaxConcurrency bool `yaml:"match_max_concurrent"` @@ -34,7 +33,6 @@ type Config struct { } func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&cfg.SchedulerAddress, "querier.scheduler-address", "", "Hostname (and port) of scheduler that querier will periodically resolve, connect to and receive queries from. Only one of -querier.frontend-address or -querier.scheduler-address can be set. If neither is set, queries are only received via HTTP endpoint.") f.StringVar(&cfg.FrontendAddress, "querier.frontend-address", "", "Address of query frontend service, in host:port format. If -querier.scheduler-address is set as well, querier will use scheduler instead. Only one of -querier.frontend-address or -querier.scheduler-address can be set. If neither is set, queries are only received via HTTP endpoint.") f.DurationVar(&cfg.DNSLookupPeriod, "querier.dns-lookup-period", 10*time.Second, "How often to query DNS for query-frontend or query-scheduler address.") @@ -47,8 +45,8 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { } func (cfg *Config) Validate(log log.Logger) error { - if cfg.FrontendAddress != "" && cfg.SchedulerAddress != "" { - return errors.New("frontend address and scheduler address are mutually exclusive, please use only one") + if cfg.FrontendAddress != "" { + return errors.New("starting querier worker without frontend address is not supported") } return cfg.GRPCClientConfig.Validate(log) } @@ -102,22 +100,10 @@ func NewQuerierWorker(cfg Config, handler RequestHandler, log log.Logger, reg pr var servs []services.Service var address string - switch { - case cfg.SchedulerAddress != "": - level.Info(log).Log("msg", "Starting querier worker connected to query-scheduler", "scheduler", cfg.SchedulerAddress) + level.Info(log).Log("msg", "Starting querier worker connected to query-frontend", "frontend", cfg.FrontendAddress) - address = cfg.SchedulerAddress - processor, servs = newSchedulerProcessor(cfg, handler, log, reg) - - case cfg.FrontendAddress != "": - level.Info(log).Log("msg", "Starting querier worker connected to query-frontend", "frontend", cfg.FrontendAddress) - - address = cfg.FrontendAddress - processor = newFrontendProcessor(cfg, handler, log) - - default: - return nil, errors.New("no query-scheduler or query-frontend address") - } + address = cfg.FrontendAddress + processor = newFrontendProcessor(cfg, handler, log) return newQuerierWorkerWithProcessor(cfg, log, processor, address, servs) } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go deleted file mode 100644 index ccc7682f28d..00000000000 --- a/pkg/scheduler/scheduler.go +++ /dev/null @@ -1,528 +0,0 @@ -package scheduler - -import ( - "context" - "flag" - "io" - "net/http" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/grafana/dskit/grpcclient" - "github.com/grafana/dskit/services" - "github.com/grafana/dskit/tenant" - otgrpc "github.com/opentracing-contrib/go-grpc" - "github.com/opentracing/opentracing-go" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/weaveworks/common/httpgrpc" - "github.com/weaveworks/common/middleware" - "github.com/weaveworks/common/user" - "google.golang.org/grpc" - - "github.com/grafana/tempo/modules/frontend/v2/frontendv2pb" - "github.com/grafana/tempo/pkg/scheduler/queue" - "github.com/grafana/tempo/pkg/scheduler/schedulerpb" - "github.com/grafana/tempo/pkg/util" - "github.com/grafana/tempo/pkg/util/httpgrpcutil" - "github.com/grafana/tempo/pkg/validation" -) - -var ( - errSchedulerIsNotRunning = errors.New("scheduler is not running") -) - -// Scheduler is responsible for queueing and dispatching queries to Queriers. -type Scheduler struct { - services.Service - - cfg Config - log log.Logger - - limits Limits - - connectedFrontendsMu sync.Mutex - connectedFrontends map[string]*connectedFrontend - - requestQueue *queue.RequestQueue - activeUsers *util.ActiveUsersCleanupService - - pendingRequestsMu sync.Mutex - pendingRequests map[requestKey]*schedulerRequest // Request is kept in this map even after being dispatched to querier. It can still be canceled at that time. - - // Subservices manager. - subservices *services.Manager - subservicesWatcher *services.FailureWatcher - - // Metrics. - queueLength *prometheus.GaugeVec - discardedRequests *prometheus.CounterVec - connectedQuerierClients prometheus.GaugeFunc - connectedFrontendClients prometheus.GaugeFunc - queueDuration prometheus.Histogram -} - -type requestKey struct { - frontendAddr string - queryID uint64 -} - -type connectedFrontend struct { - connections int - - // This context is used for running all queries from the same frontend. - // When last frontend connection is closed, context is canceled. - ctx context.Context - cancel context.CancelFunc -} - -type Config struct { - MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant"` - QuerierForgetDelay time.Duration `yaml:"querier_forget_delay"` - GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=This configures the gRPC client used to report errors back to the query-frontend."` -} - -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.IntVar(&cfg.MaxOutstandingPerTenant, "query-scheduler.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per query-scheduler. In-flight requests above this limit will fail with HTTP response status code 429.") - f.DurationVar(&cfg.QuerierForgetDelay, "query-scheduler.querier-forget-delay", 0, "If a querier disconnects without sending notification about graceful shutdown, the query-scheduler will keep the querier in the tenant's shard until the forget delay has passed. This feature is useful to reduce the blast radius when shuffle-sharding is enabled.") - cfg.GRPCClientConfig.RegisterFlagsWithPrefix("query-scheduler.grpc-client-config", f) -} - -// NewScheduler creates a new Scheduler. -func NewScheduler(cfg Config, limits Limits, log log.Logger, registerer prometheus.Registerer) (*Scheduler, error) { - s := &Scheduler{ - cfg: cfg, - log: log, - limits: limits, - - pendingRequests: map[requestKey]*schedulerRequest{}, - connectedFrontends: map[string]*connectedFrontend{}, - } - - s.queueLength = promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{ - Name: "cortex_query_scheduler_queue_length", - Help: "Number of queries in the queue.", - }, []string{"user"}) - - s.discardedRequests = promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_query_scheduler_discarded_requests_total", - Help: "Total number of query requests discarded.", - }, []string{"user"}) - s.requestQueue = queue.NewRequestQueue(cfg.MaxOutstandingPerTenant, cfg.QuerierForgetDelay, s.queueLength, s.discardedRequests) - - s.queueDuration = promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ - Name: "cortex_query_scheduler_queue_duration_seconds", - Help: "Time spend by requests in queue before getting picked up by a querier.", - Buckets: prometheus.DefBuckets, - }) - s.connectedQuerierClients = promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ - Name: "cortex_query_scheduler_connected_querier_clients", - Help: "Number of querier worker clients currently connected to the query-scheduler.", - }, s.requestQueue.GetConnectedQuerierWorkersMetric) - s.connectedFrontendClients = promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ - Name: "cortex_query_scheduler_connected_frontend_clients", - Help: "Number of query-frontend worker clients currently connected to the query-scheduler.", - }, s.getConnectedFrontendClientsMetric) - - s.activeUsers = util.NewActiveUsersCleanupWithDefaultValues(s.cleanupMetricsForInactiveUser) - - var err error - s.subservices, err = services.NewManager(s.requestQueue, s.activeUsers) - if err != nil { - return nil, err - } - - s.Service = services.NewBasicService(s.starting, s.running, s.stopping) - return s, nil -} - -// Limits needed for the Query Scheduler - interface used for decoupling. -type Limits interface { - // MaxQueriersPerUser returns max queriers to use per tenant, or 0 if shuffle sharding is disabled. - MaxQueriersPerUser(user string) int -} - -type schedulerRequest struct { - frontendAddress string - userID string - queryID uint64 - request *httpgrpc.HTTPRequest - statsEnabled bool - - enqueueTime time.Time - - ctx context.Context - ctxCancel context.CancelFunc - queueSpan opentracing.Span - - // This is only used for testing. - parentSpanContext opentracing.SpanContext -} - -// FrontendLoop handles connection from frontend. -func (s *Scheduler) FrontendLoop(frontend schedulerpb.SchedulerForFrontend_FrontendLoopServer) error { - frontendAddress, frontendCtx, err := s.frontendConnected(frontend) - if err != nil { - return err - } - defer s.frontendDisconnected(frontendAddress) - - // Response to INIT. If scheduler is not running, we skip for-loop, send SHUTTING_DOWN and exit this method. - if s.State() == services.Running { - if err := frontend.Send(&schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}); err != nil { - return err - } - } - - // We stop accepting new queries in Stopping state. By returning quickly, we disconnect frontends, which in turns - // cancels all their queries. - for s.State() == services.Running { - msg, err := frontend.Recv() - if err != nil { - // No need to report this as error, it is expected when query-frontend performs SendClose() (as frontendSchedulerWorker does). - if err == io.EOF { - return nil - } - return err - } - - if s.State() != services.Running { - break // break out of the loop, and send SHUTTING_DOWN message. - } - - var resp *schedulerpb.SchedulerToFrontend - - switch msg.GetType() { - case schedulerpb.ENQUEUE: - err = s.enqueueRequest(frontendCtx, frontendAddress, msg) - switch { - case err == nil: - resp = &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} - case err == queue.ErrTooManyRequests: - resp = &schedulerpb.SchedulerToFrontend{Status: schedulerpb.TOO_MANY_REQUESTS_PER_TENANT} - default: - resp = &schedulerpb.SchedulerToFrontend{Status: schedulerpb.ERROR, Error: err.Error()} - } - - case schedulerpb.CANCEL: - s.cancelRequestAndRemoveFromPending(frontendAddress, msg.QueryID) - resp = &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} - - default: - level.Error(s.log).Log("msg", "unknown request type from frontend", "addr", frontendAddress, "type", msg.GetType()) - return errors.New("unknown request type") - } - - err = frontend.Send(resp) - // Failure to send response results in ending this connection. - if err != nil { - return err - } - } - - // Report shutdown back to frontend, so that it can retry with different scheduler. Also stop the frontend loop. - return frontend.Send(&schedulerpb.SchedulerToFrontend{Status: schedulerpb.SHUTTING_DOWN}) -} - -func (s *Scheduler) frontendConnected(frontend schedulerpb.SchedulerForFrontend_FrontendLoopServer) (string, context.Context, error) { - msg, err := frontend.Recv() - if err != nil { - return "", nil, err - } - if msg.Type != schedulerpb.INIT || msg.FrontendAddress == "" { - return "", nil, errors.New("no frontend address") - } - - s.connectedFrontendsMu.Lock() - defer s.connectedFrontendsMu.Unlock() - - cf := s.connectedFrontends[msg.FrontendAddress] - if cf == nil { - cf = &connectedFrontend{ - connections: 0, - } - cf.ctx, cf.cancel = context.WithCancel(context.Background()) - s.connectedFrontends[msg.FrontendAddress] = cf - } - - cf.connections++ - return msg.FrontendAddress, cf.ctx, nil -} - -func (s *Scheduler) frontendDisconnected(frontendAddress string) { - s.connectedFrontendsMu.Lock() - defer s.connectedFrontendsMu.Unlock() - - cf := s.connectedFrontends[frontendAddress] - cf.connections-- - if cf.connections == 0 { - delete(s.connectedFrontends, frontendAddress) - cf.cancel() - } -} - -func (s *Scheduler) enqueueRequest(frontendContext context.Context, frontendAddr string, msg *schedulerpb.FrontendToScheduler) error { - // Create new context for this request, to support cancellation. - ctx, cancel := context.WithCancel(frontendContext) - shouldCancel := true - defer func() { - if shouldCancel { - cancel() - } - }() - - // Extract tracing information from headers in HTTP request. FrontendContext doesn't have the correct tracing - // information, since that is a long-running request. - tracer := opentracing.GlobalTracer() - parentSpanContext, err := httpgrpcutil.GetParentSpanForRequest(tracer, msg.HttpRequest) - if err != nil { - return err - } - - userID := msg.GetUserID() - - req := &schedulerRequest{ - frontendAddress: frontendAddr, - userID: msg.UserID, - queryID: msg.QueryID, - request: msg.HttpRequest, - statsEnabled: msg.StatsEnabled, - } - - now := time.Now() - - req.parentSpanContext = parentSpanContext - req.queueSpan, req.ctx = opentracing.StartSpanFromContextWithTracer(ctx, tracer, "queued", opentracing.ChildOf(parentSpanContext)) - req.enqueueTime = now - req.ctxCancel = cancel - - // aggregate the max queriers limit in the case of a multi tenant query - tenantIDs, err := tenant.TenantIDsFromOrgID(userID) - if err != nil { - return err - } - maxQueriers := validation.SmallestPositiveNonZeroIntPerTenant(tenantIDs, s.limits.MaxQueriersPerUser) - - s.activeUsers.UpdateUserTimestamp(userID, now) - return s.requestQueue.EnqueueRequest(userID, req, maxQueriers, func() { - shouldCancel = false - - s.pendingRequestsMu.Lock() - defer s.pendingRequestsMu.Unlock() - s.pendingRequests[requestKey{frontendAddr: frontendAddr, queryID: msg.QueryID}] = req - }) -} - -// This method doesn't do removal from the queue. -func (s *Scheduler) cancelRequestAndRemoveFromPending(frontendAddr string, queryID uint64) { - s.pendingRequestsMu.Lock() - defer s.pendingRequestsMu.Unlock() - - key := requestKey{frontendAddr: frontendAddr, queryID: queryID} - req := s.pendingRequests[key] - if req != nil { - req.ctxCancel() - } - delete(s.pendingRequests, key) -} - -// QuerierLoop is started by querier to receive queries from scheduler. -func (s *Scheduler) QuerierLoop(querier schedulerpb.SchedulerForQuerier_QuerierLoopServer) error { - resp, err := querier.Recv() - if err != nil { - return err - } - - querierID := resp.GetQuerierID() - - s.requestQueue.RegisterQuerierConnection(querierID) - defer s.requestQueue.UnregisterQuerierConnection(querierID) - - // If the downstream connection to querier is cancelled, - // we need to ping the condition variable to unblock getNextRequestForQuerier. - // Ideally we'd have ctx aware condition variables... - go func() { - <-querier.Context().Done() - s.requestQueue.QuerierDisconnecting() - }() - - lastUserIndex := queue.FirstUser() - - // In stopping state scheduler is not accepting new queries, but still dispatching queries in the queues. - for s.isRunningOrStopping() { - req, idx, err := s.requestQueue.GetNextRequestForQuerier(querier.Context(), lastUserIndex, querierID) - if err != nil { - return err - } - lastUserIndex = idx - - r := req.(*schedulerRequest) - - s.queueDuration.Observe(time.Since(r.enqueueTime).Seconds()) - r.queueSpan.Finish() - - /* - We want to dequeue the next unexpired request from the chosen tenant queue. - The chance of choosing a particular tenant for dequeueing is (1/active_tenants). - This is problematic under load, especially with other middleware enabled such as - querier.split-by-interval, where one request may fan out into many. - If expired requests aren't exhausted before checking another tenant, it would take - n_active_tenants * n_expired_requests_at_front_of_queue requests being processed - before an active request was handled for the tenant in question. - If this tenant meanwhile continued to queue requests, - it's possible that it's own queue would perpetually contain only expired requests. - */ - - if r.ctx.Err() != nil { - // Remove from pending requests. - s.cancelRequestAndRemoveFromPending(r.frontendAddress, r.queryID) - - lastUserIndex = lastUserIndex.ReuseLastUser() - continue - } - - if err := s.forwardRequestToQuerier(querier, r); err != nil { - return err - } - } - - return errSchedulerIsNotRunning -} - -func (s *Scheduler) NotifyQuerierShutdown(_ context.Context, req *schedulerpb.NotifyQuerierShutdownRequest) (*schedulerpb.NotifyQuerierShutdownResponse, error) { - level.Info(s.log).Log("msg", "received shutdown notification from querier", "querier", req.GetQuerierID()) - s.requestQueue.NotifyQuerierShutdown(req.GetQuerierID()) - - return &schedulerpb.NotifyQuerierShutdownResponse{}, nil -} - -func (s *Scheduler) forwardRequestToQuerier(querier schedulerpb.SchedulerForQuerier_QuerierLoopServer, req *schedulerRequest) error { - // Make sure to cancel request at the end to cleanup resources. - defer s.cancelRequestAndRemoveFromPending(req.frontendAddress, req.queryID) - - // Handle the stream sending & receiving on a goroutine so we can - // monitoring the contexts in a select and cancel things appropriately. - errCh := make(chan error, 1) - go func() { - err := querier.Send(&schedulerpb.SchedulerToQuerier{ - UserID: req.userID, - QueryID: req.queryID, - FrontendAddress: req.frontendAddress, - HttpRequest: req.request, - StatsEnabled: req.statsEnabled, - }) - if err != nil { - errCh <- err - return - } - - _, err = querier.Recv() - errCh <- err - }() - - select { - case <-req.ctx.Done(): - // If the upstream request is cancelled (eg. frontend issued CANCEL or closed connection), - // we need to cancel the downstream req. Only way we can do that is to close the stream (by returning error here). - // Querier is expecting this semantics. - return req.ctx.Err() - - case err := <-errCh: - // Is there was an error handling this request due to network IO, - // then error out this upstream request _and_ stream. - - if err != nil { - s.forwardErrorToFrontend(req.ctx, req, err) - } - return err - } -} - -func (s *Scheduler) forwardErrorToFrontend(ctx context.Context, req *schedulerRequest, requestErr error) { - opts, err := s.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{ - otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), - middleware.ClientUserHeaderInterceptor}, - nil) - if err != nil { - level.Warn(s.log).Log("msg", "failed to create gRPC options for the connection to frontend to report error", "frontend", req.frontendAddress, "err", err, "requestErr", requestErr) - return - } - - conn, err := grpc.DialContext(ctx, req.frontendAddress, opts...) - if err != nil { - level.Warn(s.log).Log("msg", "failed to create gRPC connection to frontend to report error", "frontend", req.frontendAddress, "err", err, "requestErr", requestErr) - return - } - - defer func() { - _ = conn.Close() - }() - - client := frontendv2pb.NewFrontendForQuerierClient(conn) - - userCtx := user.InjectOrgID(ctx, req.userID) - _, err = client.QueryResult(userCtx, &frontendv2pb.QueryResultRequest{ - QueryID: req.queryID, - HttpResponse: &httpgrpc.HTTPResponse{ - Code: http.StatusInternalServerError, - Body: []byte(requestErr.Error()), - }, - }) - - if err != nil { - level.Warn(s.log).Log("msg", "failed to forward error to frontend", "frontend", req.frontendAddress, "err", err, "requestErr", requestErr) - return - } -} - -func (s *Scheduler) isRunningOrStopping() bool { - st := s.State() - return st == services.Running || st == services.Stopping -} - -func (s *Scheduler) starting(ctx context.Context) error { - s.subservicesWatcher.WatchManager(s.subservices) - - if err := services.StartManagerAndAwaitHealthy(ctx, s.subservices); err != nil { - return errors.Wrap(err, "unable to start scheduler subservices") - } - - return nil -} - -func (s *Scheduler) running(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - return nil - case err := <-s.subservicesWatcher.Chan(): - return errors.Wrap(err, "scheduler subservice failed") - } - } -} - -// Close the Scheduler. -func (s *Scheduler) stopping(_ error) error { - // This will also stop the requests queue, which stop accepting new requests and errors out any pending requests. - return services.StopManagerAndAwaitStopped(context.Background(), s.subservices) -} - -func (s *Scheduler) cleanupMetricsForInactiveUser(user string) { - s.queueLength.DeleteLabelValues(user) - s.discardedRequests.DeleteLabelValues(user) -} - -func (s *Scheduler) getConnectedFrontendClientsMetric() float64 { - s.connectedFrontendsMu.Lock() - defer s.connectedFrontendsMu.Unlock() - - count := 0 - for _, workers := range s.connectedFrontends { - count += workers.connections - } - - return float64(count) -} diff --git a/pkg/scheduler/schedulerpb/scheduler.pb.go b/pkg/scheduler/schedulerpb/scheduler.pb.go deleted file mode 100644 index 1b6fda6e692..00000000000 --- a/pkg/scheduler/schedulerpb/scheduler.pb.go +++ /dev/null @@ -1,2323 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: scheduler.proto - -package schedulerpb - -import ( - context "context" - fmt "fmt" - _ "github.com/gogo/protobuf/gogoproto" - proto "github.com/gogo/protobuf/proto" - httpgrpc "github.com/weaveworks/common/httpgrpc" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - io "io" - math "math" - math_bits "math/bits" - reflect "reflect" - strconv "strconv" - strings "strings" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -type FrontendToSchedulerType int32 - -const ( - INIT FrontendToSchedulerType = 0 - ENQUEUE FrontendToSchedulerType = 1 - CANCEL FrontendToSchedulerType = 2 -) - -var FrontendToSchedulerType_name = map[int32]string{ - 0: "INIT", - 1: "ENQUEUE", - 2: "CANCEL", -} - -var FrontendToSchedulerType_value = map[string]int32{ - "INIT": 0, - "ENQUEUE": 1, - "CANCEL": 2, -} - -func (FrontendToSchedulerType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_2b3fc28395a6d9c5, []int{0} -} - -type SchedulerToFrontendStatus int32 - -const ( - OK SchedulerToFrontendStatus = 0 - TOO_MANY_REQUESTS_PER_TENANT SchedulerToFrontendStatus = 1 - ERROR SchedulerToFrontendStatus = 2 - SHUTTING_DOWN SchedulerToFrontendStatus = 3 -) - -var SchedulerToFrontendStatus_name = map[int32]string{ - 0: "OK", - 1: "TOO_MANY_REQUESTS_PER_TENANT", - 2: "ERROR", - 3: "SHUTTING_DOWN", -} - -var SchedulerToFrontendStatus_value = map[string]int32{ - "OK": 0, - "TOO_MANY_REQUESTS_PER_TENANT": 1, - "ERROR": 2, - "SHUTTING_DOWN": 3, -} - -func (SchedulerToFrontendStatus) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_2b3fc28395a6d9c5, []int{1} -} - -// Querier reports its own clientID when it connects, so that scheduler knows how many *different* queriers are connected. -// To signal that querier is ready to accept another request, querier sends empty message. -type QuerierToScheduler struct { - QuerierID string `protobuf:"bytes,1,opt,name=querierID,proto3" json:"querierID,omitempty"` -} - -func (m *QuerierToScheduler) Reset() { *m = QuerierToScheduler{} } -func (*QuerierToScheduler) ProtoMessage() {} -func (*QuerierToScheduler) Descriptor() ([]byte, []int) { - return fileDescriptor_2b3fc28395a6d9c5, []int{0} -} -func (m *QuerierToScheduler) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *QuerierToScheduler) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_QuerierToScheduler.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *QuerierToScheduler) XXX_Merge(src proto.Message) { - xxx_messageInfo_QuerierToScheduler.Merge(m, src) -} -func (m *QuerierToScheduler) XXX_Size() int { - return m.Size() -} -func (m *QuerierToScheduler) XXX_DiscardUnknown() { - xxx_messageInfo_QuerierToScheduler.DiscardUnknown(m) -} - -var xxx_messageInfo_QuerierToScheduler proto.InternalMessageInfo - -func (m *QuerierToScheduler) GetQuerierID() string { - if m != nil { - return m.QuerierID - } - return "" -} - -type SchedulerToQuerier struct { - // Query ID as reported by frontend. When querier sends the response back to frontend (using frontendAddress), - // it identifies the query by using this ID. - QueryID uint64 `protobuf:"varint,1,opt,name=queryID,proto3" json:"queryID,omitempty"` - HttpRequest *httpgrpc.HTTPRequest `protobuf:"bytes,2,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` - // Where should querier send HTTP Response to (using FrontendForQuerier interface). - FrontendAddress string `protobuf:"bytes,3,opt,name=frontendAddress,proto3" json:"frontendAddress,omitempty"` - // User who initiated the request. Needed to send reply back to frontend. - UserID string `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"` - // Whether query statistics tracking should be enabled. The response will include - // statistics only when this option is enabled. - StatsEnabled bool `protobuf:"varint,5,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"` -} - -func (m *SchedulerToQuerier) Reset() { *m = SchedulerToQuerier{} } -func (*SchedulerToQuerier) ProtoMessage() {} -func (*SchedulerToQuerier) Descriptor() ([]byte, []int) { - return fileDescriptor_2b3fc28395a6d9c5, []int{1} -} -func (m *SchedulerToQuerier) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *SchedulerToQuerier) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_SchedulerToQuerier.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *SchedulerToQuerier) XXX_Merge(src proto.Message) { - xxx_messageInfo_SchedulerToQuerier.Merge(m, src) -} -func (m *SchedulerToQuerier) XXX_Size() int { - return m.Size() -} -func (m *SchedulerToQuerier) XXX_DiscardUnknown() { - xxx_messageInfo_SchedulerToQuerier.DiscardUnknown(m) -} - -var xxx_messageInfo_SchedulerToQuerier proto.InternalMessageInfo - -func (m *SchedulerToQuerier) GetQueryID() uint64 { - if m != nil { - return m.QueryID - } - return 0 -} - -func (m *SchedulerToQuerier) GetHttpRequest() *httpgrpc.HTTPRequest { - if m != nil { - return m.HttpRequest - } - return nil -} - -func (m *SchedulerToQuerier) GetFrontendAddress() string { - if m != nil { - return m.FrontendAddress - } - return "" -} - -func (m *SchedulerToQuerier) GetUserID() string { - if m != nil { - return m.UserID - } - return "" -} - -func (m *SchedulerToQuerier) GetStatsEnabled() bool { - if m != nil { - return m.StatsEnabled - } - return false -} - -type FrontendToScheduler struct { - Type FrontendToSchedulerType `protobuf:"varint,1,opt,name=type,proto3,enum=schedulerpb.FrontendToSchedulerType" json:"type,omitempty"` - // Used by INIT message. Will be put into all requests passed to querier. - FrontendAddress string `protobuf:"bytes,2,opt,name=frontendAddress,proto3" json:"frontendAddress,omitempty"` - // Used by ENQUEUE and CANCEL. - // Each frontend manages its own queryIDs. Different frontends may use same set of query IDs. - QueryID uint64 `protobuf:"varint,3,opt,name=queryID,proto3" json:"queryID,omitempty"` - // Following are used by ENQUEUE only. - UserID string `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"` - HttpRequest *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` - StatsEnabled bool `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"` -} - -func (m *FrontendToScheduler) Reset() { *m = FrontendToScheduler{} } -func (*FrontendToScheduler) ProtoMessage() {} -func (*FrontendToScheduler) Descriptor() ([]byte, []int) { - return fileDescriptor_2b3fc28395a6d9c5, []int{2} -} -func (m *FrontendToScheduler) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *FrontendToScheduler) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_FrontendToScheduler.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *FrontendToScheduler) XXX_Merge(src proto.Message) { - xxx_messageInfo_FrontendToScheduler.Merge(m, src) -} -func (m *FrontendToScheduler) XXX_Size() int { - return m.Size() -} -func (m *FrontendToScheduler) XXX_DiscardUnknown() { - xxx_messageInfo_FrontendToScheduler.DiscardUnknown(m) -} - -var xxx_messageInfo_FrontendToScheduler proto.InternalMessageInfo - -func (m *FrontendToScheduler) GetType() FrontendToSchedulerType { - if m != nil { - return m.Type - } - return INIT -} - -func (m *FrontendToScheduler) GetFrontendAddress() string { - if m != nil { - return m.FrontendAddress - } - return "" -} - -func (m *FrontendToScheduler) GetQueryID() uint64 { - if m != nil { - return m.QueryID - } - return 0 -} - -func (m *FrontendToScheduler) GetUserID() string { - if m != nil { - return m.UserID - } - return "" -} - -func (m *FrontendToScheduler) GetHttpRequest() *httpgrpc.HTTPRequest { - if m != nil { - return m.HttpRequest - } - return nil -} - -func (m *FrontendToScheduler) GetStatsEnabled() bool { - if m != nil { - return m.StatsEnabled - } - return false -} - -type SchedulerToFrontend struct { - Status SchedulerToFrontendStatus `protobuf:"varint,1,opt,name=status,proto3,enum=schedulerpb.SchedulerToFrontendStatus" json:"status,omitempty"` - Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` -} - -func (m *SchedulerToFrontend) Reset() { *m = SchedulerToFrontend{} } -func (*SchedulerToFrontend) ProtoMessage() {} -func (*SchedulerToFrontend) Descriptor() ([]byte, []int) { - return fileDescriptor_2b3fc28395a6d9c5, []int{3} -} -func (m *SchedulerToFrontend) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *SchedulerToFrontend) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_SchedulerToFrontend.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *SchedulerToFrontend) XXX_Merge(src proto.Message) { - xxx_messageInfo_SchedulerToFrontend.Merge(m, src) -} -func (m *SchedulerToFrontend) XXX_Size() int { - return m.Size() -} -func (m *SchedulerToFrontend) XXX_DiscardUnknown() { - xxx_messageInfo_SchedulerToFrontend.DiscardUnknown(m) -} - -var xxx_messageInfo_SchedulerToFrontend proto.InternalMessageInfo - -func (m *SchedulerToFrontend) GetStatus() SchedulerToFrontendStatus { - if m != nil { - return m.Status - } - return OK -} - -func (m *SchedulerToFrontend) GetError() string { - if m != nil { - return m.Error - } - return "" -} - -type NotifyQuerierShutdownRequest struct { - QuerierID string `protobuf:"bytes,1,opt,name=querierID,proto3" json:"querierID,omitempty"` -} - -func (m *NotifyQuerierShutdownRequest) Reset() { *m = NotifyQuerierShutdownRequest{} } -func (*NotifyQuerierShutdownRequest) ProtoMessage() {} -func (*NotifyQuerierShutdownRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_2b3fc28395a6d9c5, []int{4} -} -func (m *NotifyQuerierShutdownRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NotifyQuerierShutdownRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NotifyQuerierShutdownRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *NotifyQuerierShutdownRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NotifyQuerierShutdownRequest.Merge(m, src) -} -func (m *NotifyQuerierShutdownRequest) XXX_Size() int { - return m.Size() -} -func (m *NotifyQuerierShutdownRequest) XXX_DiscardUnknown() { - xxx_messageInfo_NotifyQuerierShutdownRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_NotifyQuerierShutdownRequest proto.InternalMessageInfo - -func (m *NotifyQuerierShutdownRequest) GetQuerierID() string { - if m != nil { - return m.QuerierID - } - return "" -} - -type NotifyQuerierShutdownResponse struct { -} - -func (m *NotifyQuerierShutdownResponse) Reset() { *m = NotifyQuerierShutdownResponse{} } -func (*NotifyQuerierShutdownResponse) ProtoMessage() {} -func (*NotifyQuerierShutdownResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_2b3fc28395a6d9c5, []int{5} -} -func (m *NotifyQuerierShutdownResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NotifyQuerierShutdownResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NotifyQuerierShutdownResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *NotifyQuerierShutdownResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NotifyQuerierShutdownResponse.Merge(m, src) -} -func (m *NotifyQuerierShutdownResponse) XXX_Size() int { - return m.Size() -} -func (m *NotifyQuerierShutdownResponse) XXX_DiscardUnknown() { - xxx_messageInfo_NotifyQuerierShutdownResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_NotifyQuerierShutdownResponse proto.InternalMessageInfo - -func init() { - proto.RegisterEnum("schedulerpb.FrontendToSchedulerType", FrontendToSchedulerType_name, FrontendToSchedulerType_value) - proto.RegisterEnum("schedulerpb.SchedulerToFrontendStatus", SchedulerToFrontendStatus_name, SchedulerToFrontendStatus_value) - proto.RegisterType((*QuerierToScheduler)(nil), "schedulerpb.QuerierToScheduler") - proto.RegisterType((*SchedulerToQuerier)(nil), "schedulerpb.SchedulerToQuerier") - proto.RegisterType((*FrontendToScheduler)(nil), "schedulerpb.FrontendToScheduler") - proto.RegisterType((*SchedulerToFrontend)(nil), "schedulerpb.SchedulerToFrontend") - proto.RegisterType((*NotifyQuerierShutdownRequest)(nil), "schedulerpb.NotifyQuerierShutdownRequest") - proto.RegisterType((*NotifyQuerierShutdownResponse)(nil), "schedulerpb.NotifyQuerierShutdownResponse") -} - -func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) } - -var fileDescriptor_2b3fc28395a6d9c5 = []byte{ - // 650 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcf, 0x4f, 0x13, 0x41, - 0x14, 0xde, 0x29, 0x6d, 0x81, 0x57, 0x94, 0x75, 0x00, 0xad, 0x0d, 0x0e, 0x4d, 0x63, 0x4c, 0x25, - 0xb1, 0x35, 0xd5, 0x44, 0x0f, 0xc4, 0xa4, 0xc2, 0x22, 0x8d, 0xb8, 0x85, 0xe9, 0x34, 0xfe, 0xb8, - 0x34, 0xb4, 0x1d, 0x5a, 0x02, 0xec, 0x2c, 0xb3, 0xbb, 0x92, 0xde, 0x3c, 0x7a, 0xf4, 0xcf, 0xf0, - 0x4f, 0xf1, 0x62, 0xc2, 0x91, 0x83, 0x07, 0x59, 0x2e, 0x1e, 0xf9, 0x13, 0x0c, 0xd3, 0x6d, 0xdd, - 0xd6, 0x16, 0xb8, 0xbd, 0xf7, 0xf6, 0xfb, 0x76, 0xde, 0xf7, 0xbd, 0x37, 0x03, 0xb3, 0x4e, 0xa3, - 0xcd, 0x9b, 0xde, 0x01, 0x97, 0x39, 0x5b, 0x0a, 0x57, 0xe0, 0x44, 0xbf, 0x60, 0xd7, 0x53, 0x4f, - 0x5a, 0x7b, 0x6e, 0xdb, 0xab, 0xe7, 0x1a, 0xe2, 0x30, 0xdf, 0x12, 0x2d, 0x91, 0x57, 0x98, 0xba, - 0xb7, 0xab, 0x32, 0x95, 0xa8, 0xa8, 0xcb, 0x4d, 0x3d, 0x0f, 0xc1, 0x8f, 0xf9, 0xce, 0x67, 0x7e, - 0x2c, 0xe4, 0xbe, 0x93, 0x6f, 0x88, 0xc3, 0x43, 0x61, 0xe5, 0xdb, 0xae, 0x6b, 0xb7, 0xa4, 0xdd, - 0xe8, 0x07, 0x5d, 0x56, 0xa6, 0x00, 0x78, 0xdb, 0xe3, 0x72, 0x8f, 0x4b, 0x26, 0x2a, 0xbd, 0xc3, - 0xf1, 0x22, 0x4c, 0x1f, 0x75, 0xab, 0xa5, 0xb5, 0x24, 0x4a, 0xa3, 0xec, 0x34, 0xfd, 0x57, 0xc8, - 0xfc, 0x44, 0x80, 0xfb, 0x58, 0x26, 0x02, 0x3e, 0x4e, 0xc2, 0xe4, 0x25, 0xa6, 0x13, 0x50, 0xa2, - 0xb4, 0x97, 0xe2, 0x17, 0x90, 0xb8, 0x3c, 0x96, 0xf2, 0x23, 0x8f, 0x3b, 0x6e, 0x32, 0x92, 0x46, - 0xd9, 0x44, 0x61, 0x21, 0xd7, 0x6f, 0x65, 0x83, 0xb1, 0xad, 0xe0, 0x23, 0x0d, 0x23, 0x71, 0x16, - 0x66, 0x77, 0xa5, 0xb0, 0x5c, 0x6e, 0x35, 0x8b, 0xcd, 0xa6, 0xe4, 0x8e, 0x93, 0x9c, 0x50, 0xdd, - 0x0c, 0x97, 0xf1, 0x5d, 0x88, 0x7b, 0x8e, 0x6a, 0x37, 0xaa, 0x00, 0x41, 0x86, 0x33, 0x30, 0xe3, - 0xb8, 0x3b, 0xae, 0x63, 0x58, 0x3b, 0xf5, 0x03, 0xde, 0x4c, 0xc6, 0xd2, 0x28, 0x3b, 0x45, 0x07, - 0x6a, 0x99, 0xaf, 0x11, 0x98, 0x5b, 0x0f, 0xfe, 0x17, 0x76, 0xe1, 0x25, 0x44, 0xdd, 0x8e, 0xcd, - 0x95, 0x9a, 0xdb, 0x85, 0x87, 0xb9, 0xd0, 0x70, 0x72, 0x23, 0xf0, 0xac, 0x63, 0x73, 0xaa, 0x18, - 0xa3, 0xfa, 0x8e, 0x8c, 0xee, 0x3b, 0x64, 0xda, 0xc4, 0xa0, 0x69, 0xe3, 0x14, 0x0d, 0x99, 0x19, - 0xbb, 0xb1, 0x99, 0xc3, 0x56, 0xc4, 0x47, 0x58, 0xb1, 0x0f, 0x73, 0xa1, 0xc9, 0xf6, 0x44, 0xe2, - 0x57, 0x10, 0xbf, 0x84, 0x79, 0x4e, 0xe0, 0xc5, 0xa3, 0x01, 0x2f, 0x46, 0x30, 0x2a, 0x0a, 0x4d, - 0x03, 0x16, 0x9e, 0x87, 0x18, 0x97, 0x52, 0xc8, 0xc0, 0x85, 0x6e, 0x92, 0x59, 0x81, 0x45, 0x53, - 0xb8, 0x7b, 0xbb, 0x9d, 0x60, 0x83, 0x2a, 0x6d, 0xcf, 0x6d, 0x8a, 0x63, 0xab, 0xd7, 0xf0, 0xd5, - 0x5b, 0xb8, 0x04, 0x0f, 0xc6, 0xb0, 0x1d, 0x5b, 0x58, 0x0e, 0x5f, 0x5e, 0x81, 0x7b, 0x63, 0xa6, - 0x84, 0xa7, 0x20, 0x5a, 0x32, 0x4b, 0x4c, 0xd7, 0x70, 0x02, 0x26, 0x0d, 0x73, 0xbb, 0x6a, 0x54, - 0x0d, 0x1d, 0x61, 0x80, 0xf8, 0x6a, 0xd1, 0x5c, 0x35, 0x36, 0xf5, 0xc8, 0x72, 0x03, 0xee, 0x8f, - 0xd5, 0x85, 0xe3, 0x10, 0x29, 0xbf, 0xd5, 0x35, 0x9c, 0x86, 0x45, 0x56, 0x2e, 0xd7, 0xde, 0x15, - 0xcd, 0x8f, 0x35, 0x6a, 0x6c, 0x57, 0x8d, 0x0a, 0xab, 0xd4, 0xb6, 0x0c, 0x5a, 0x63, 0x86, 0x59, - 0x34, 0x99, 0x8e, 0xf0, 0x34, 0xc4, 0x0c, 0x4a, 0xcb, 0x54, 0x8f, 0xe0, 0x3b, 0x70, 0xab, 0xb2, - 0x51, 0x65, 0xac, 0x64, 0xbe, 0xa9, 0xad, 0x95, 0xdf, 0x9b, 0xfa, 0x44, 0xe1, 0x17, 0x0a, 0xf9, - 0xbd, 0x2e, 0x64, 0xef, 0x2a, 0x55, 0x21, 0x11, 0x84, 0x9b, 0x42, 0xd8, 0x78, 0x69, 0xc0, 0xee, - 0xff, 0xef, 0x6b, 0x6a, 0x69, 0xdc, 0x3c, 0x02, 0x6c, 0x46, 0xcb, 0xa2, 0xa7, 0x08, 0x5b, 0xb0, - 0x30, 0xd2, 0x32, 0xfc, 0x78, 0x80, 0x7f, 0xd5, 0x50, 0x52, 0xcb, 0x37, 0x81, 0x76, 0x27, 0x50, - 0xb0, 0x61, 0x3e, 0xac, 0xae, 0xbf, 0x4e, 0x1f, 0x60, 0xa6, 0x17, 0x2b, 0x7d, 0xe9, 0xeb, 0xae, - 0x56, 0x2a, 0x7d, 0xdd, 0xc2, 0x75, 0x15, 0xbe, 0x2e, 0x9e, 0x9c, 0x11, 0xed, 0xf4, 0x8c, 0x68, - 0x17, 0x67, 0x04, 0x7d, 0xf1, 0x09, 0xfa, 0xee, 0x13, 0xf4, 0xc3, 0x27, 0xe8, 0xc4, 0x27, 0xe8, - 0xb7, 0x4f, 0xd0, 0x1f, 0x9f, 0x68, 0x17, 0x3e, 0x41, 0xdf, 0xce, 0x89, 0x76, 0x72, 0x4e, 0xb4, - 0xd3, 0x73, 0xa2, 0x7d, 0x0a, 0x3f, 0xbb, 0xf5, 0xb8, 0x7a, 0x18, 0x9f, 0xfd, 0x0d, 0x00, 0x00, - 0xff, 0xff, 0x49, 0x11, 0xf9, 0x34, 0x9d, 0x05, 0x00, 0x00, -} - -func (x FrontendToSchedulerType) String() string { - s, ok := FrontendToSchedulerType_name[int32(x)] - if ok { - return s - } - return strconv.Itoa(int(x)) -} -func (x SchedulerToFrontendStatus) String() string { - s, ok := SchedulerToFrontendStatus_name[int32(x)] - if ok { - return s - } - return strconv.Itoa(int(x)) -} -func (this *QuerierToScheduler) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*QuerierToScheduler) - if !ok { - that2, ok := that.(QuerierToScheduler) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.QuerierID != that1.QuerierID { - return false - } - return true -} -func (this *SchedulerToQuerier) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*SchedulerToQuerier) - if !ok { - that2, ok := that.(SchedulerToQuerier) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.QueryID != that1.QueryID { - return false - } - if !this.HttpRequest.Equal(that1.HttpRequest) { - return false - } - if this.FrontendAddress != that1.FrontendAddress { - return false - } - if this.UserID != that1.UserID { - return false - } - if this.StatsEnabled != that1.StatsEnabled { - return false - } - return true -} -func (this *FrontendToScheduler) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*FrontendToScheduler) - if !ok { - that2, ok := that.(FrontendToScheduler) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.Type != that1.Type { - return false - } - if this.FrontendAddress != that1.FrontendAddress { - return false - } - if this.QueryID != that1.QueryID { - return false - } - if this.UserID != that1.UserID { - return false - } - if !this.HttpRequest.Equal(that1.HttpRequest) { - return false - } - if this.StatsEnabled != that1.StatsEnabled { - return false - } - return true -} -func (this *SchedulerToFrontend) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*SchedulerToFrontend) - if !ok { - that2, ok := that.(SchedulerToFrontend) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.Status != that1.Status { - return false - } - if this.Error != that1.Error { - return false - } - return true -} -func (this *NotifyQuerierShutdownRequest) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*NotifyQuerierShutdownRequest) - if !ok { - that2, ok := that.(NotifyQuerierShutdownRequest) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.QuerierID != that1.QuerierID { - return false - } - return true -} -func (this *NotifyQuerierShutdownResponse) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*NotifyQuerierShutdownResponse) - if !ok { - that2, ok := that.(NotifyQuerierShutdownResponse) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - return true -} -func (this *QuerierToScheduler) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 5) - s = append(s, "&schedulerpb.QuerierToScheduler{") - s = append(s, "QuerierID: "+fmt.Sprintf("%#v", this.QuerierID)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func (this *SchedulerToQuerier) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 9) - s = append(s, "&schedulerpb.SchedulerToQuerier{") - s = append(s, "QueryID: "+fmt.Sprintf("%#v", this.QueryID)+",\n") - if this.HttpRequest != nil { - s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n") - } - s = append(s, "FrontendAddress: "+fmt.Sprintf("%#v", this.FrontendAddress)+",\n") - s = append(s, "UserID: "+fmt.Sprintf("%#v", this.UserID)+",\n") - s = append(s, "StatsEnabled: "+fmt.Sprintf("%#v", this.StatsEnabled)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func (this *FrontendToScheduler) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 10) - s = append(s, "&schedulerpb.FrontendToScheduler{") - s = append(s, "Type: "+fmt.Sprintf("%#v", this.Type)+",\n") - s = append(s, "FrontendAddress: "+fmt.Sprintf("%#v", this.FrontendAddress)+",\n") - s = append(s, "QueryID: "+fmt.Sprintf("%#v", this.QueryID)+",\n") - s = append(s, "UserID: "+fmt.Sprintf("%#v", this.UserID)+",\n") - if this.HttpRequest != nil { - s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n") - } - s = append(s, "StatsEnabled: "+fmt.Sprintf("%#v", this.StatsEnabled)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func (this *SchedulerToFrontend) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&schedulerpb.SchedulerToFrontend{") - s = append(s, "Status: "+fmt.Sprintf("%#v", this.Status)+",\n") - s = append(s, "Error: "+fmt.Sprintf("%#v", this.Error)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func (this *NotifyQuerierShutdownRequest) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 5) - s = append(s, "&schedulerpb.NotifyQuerierShutdownRequest{") - s = append(s, "QuerierID: "+fmt.Sprintf("%#v", this.QuerierID)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func (this *NotifyQuerierShutdownResponse) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 4) - s = append(s, "&schedulerpb.NotifyQuerierShutdownResponse{") - s = append(s, "}") - return strings.Join(s, "") -} -func valueToGoStringScheduler(v interface{}, typ string) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// SchedulerForQuerierClient is the client API for SchedulerForQuerier service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type SchedulerForQuerierClient interface { - // After calling this method, both Querier and Scheduler enter a loop, in which querier waits for - // "SchedulerToQuerier" messages containing HTTP requests and processes them. After processing the request, - // querier signals that it is ready to accept another one by sending empty QuerierToScheduler message. - // - // Long-running loop is used to detect broken connection between scheduler and querier. This is important - // for scheduler to keep a list of connected queriers up-to-date. - QuerierLoop(ctx context.Context, opts ...grpc.CallOption) (SchedulerForQuerier_QuerierLoopClient, error) - // The querier notifies the query-scheduler that it started a graceful shutdown. - NotifyQuerierShutdown(ctx context.Context, in *NotifyQuerierShutdownRequest, opts ...grpc.CallOption) (*NotifyQuerierShutdownResponse, error) -} - -type schedulerForQuerierClient struct { - cc *grpc.ClientConn -} - -func NewSchedulerForQuerierClient(cc *grpc.ClientConn) SchedulerForQuerierClient { - return &schedulerForQuerierClient{cc} -} - -func (c *schedulerForQuerierClient) QuerierLoop(ctx context.Context, opts ...grpc.CallOption) (SchedulerForQuerier_QuerierLoopClient, error) { - stream, err := c.cc.NewStream(ctx, &_SchedulerForQuerier_serviceDesc.Streams[0], "/schedulerpb.SchedulerForQuerier/QuerierLoop", opts...) - if err != nil { - return nil, err - } - x := &schedulerForQuerierQuerierLoopClient{stream} - return x, nil -} - -type SchedulerForQuerier_QuerierLoopClient interface { - Send(*QuerierToScheduler) error - Recv() (*SchedulerToQuerier, error) - grpc.ClientStream -} - -type schedulerForQuerierQuerierLoopClient struct { - grpc.ClientStream -} - -func (x *schedulerForQuerierQuerierLoopClient) Send(m *QuerierToScheduler) error { - return x.ClientStream.SendMsg(m) -} - -func (x *schedulerForQuerierQuerierLoopClient) Recv() (*SchedulerToQuerier, error) { - m := new(SchedulerToQuerier) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *schedulerForQuerierClient) NotifyQuerierShutdown(ctx context.Context, in *NotifyQuerierShutdownRequest, opts ...grpc.CallOption) (*NotifyQuerierShutdownResponse, error) { - out := new(NotifyQuerierShutdownResponse) - err := c.cc.Invoke(ctx, "/schedulerpb.SchedulerForQuerier/NotifyQuerierShutdown", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// SchedulerForQuerierServer is the server API for SchedulerForQuerier service. -type SchedulerForQuerierServer interface { - // After calling this method, both Querier and Scheduler enter a loop, in which querier waits for - // "SchedulerToQuerier" messages containing HTTP requests and processes them. After processing the request, - // querier signals that it is ready to accept another one by sending empty QuerierToScheduler message. - // - // Long-running loop is used to detect broken connection between scheduler and querier. This is important - // for scheduler to keep a list of connected queriers up-to-date. - QuerierLoop(SchedulerForQuerier_QuerierLoopServer) error - // The querier notifies the query-scheduler that it started a graceful shutdown. - NotifyQuerierShutdown(context.Context, *NotifyQuerierShutdownRequest) (*NotifyQuerierShutdownResponse, error) -} - -// UnimplementedSchedulerForQuerierServer can be embedded to have forward compatible implementations. -type UnimplementedSchedulerForQuerierServer struct { -} - -func (*UnimplementedSchedulerForQuerierServer) QuerierLoop(srv SchedulerForQuerier_QuerierLoopServer) error { - return status.Errorf(codes.Unimplemented, "method QuerierLoop not implemented") -} -func (*UnimplementedSchedulerForQuerierServer) NotifyQuerierShutdown(ctx context.Context, req *NotifyQuerierShutdownRequest) (*NotifyQuerierShutdownResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method NotifyQuerierShutdown not implemented") -} - -func RegisterSchedulerForQuerierServer(s *grpc.Server, srv SchedulerForQuerierServer) { - s.RegisterService(&_SchedulerForQuerier_serviceDesc, srv) -} - -func _SchedulerForQuerier_QuerierLoop_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(SchedulerForQuerierServer).QuerierLoop(&schedulerForQuerierQuerierLoopServer{stream}) -} - -type SchedulerForQuerier_QuerierLoopServer interface { - Send(*SchedulerToQuerier) error - Recv() (*QuerierToScheduler, error) - grpc.ServerStream -} - -type schedulerForQuerierQuerierLoopServer struct { - grpc.ServerStream -} - -func (x *schedulerForQuerierQuerierLoopServer) Send(m *SchedulerToQuerier) error { - return x.ServerStream.SendMsg(m) -} - -func (x *schedulerForQuerierQuerierLoopServer) Recv() (*QuerierToScheduler, error) { - m := new(QuerierToScheduler) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _SchedulerForQuerier_NotifyQuerierShutdown_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(NotifyQuerierShutdownRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SchedulerForQuerierServer).NotifyQuerierShutdown(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/schedulerpb.SchedulerForQuerier/NotifyQuerierShutdown", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SchedulerForQuerierServer).NotifyQuerierShutdown(ctx, req.(*NotifyQuerierShutdownRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _SchedulerForQuerier_serviceDesc = grpc.ServiceDesc{ - ServiceName: "schedulerpb.SchedulerForQuerier", - HandlerType: (*SchedulerForQuerierServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "NotifyQuerierShutdown", - Handler: _SchedulerForQuerier_NotifyQuerierShutdown_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "QuerierLoop", - Handler: _SchedulerForQuerier_QuerierLoop_Handler, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "scheduler.proto", -} - -// SchedulerForFrontendClient is the client API for SchedulerForFrontend service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type SchedulerForFrontendClient interface { - // After calling this method, both Frontend and Scheduler enter a loop. Frontend will keep sending ENQUEUE and - // CANCEL requests, and scheduler is expected to process them. Scheduler returns one response for each request. - // - // Long-running loop is used to detect broken connection between frontend and scheduler. This is important for both - // parties... if connection breaks, frontend can cancel (and possibly retry on different scheduler) all pending - // requests sent to this scheduler, while scheduler can cancel queued requests from given frontend. - FrontendLoop(ctx context.Context, opts ...grpc.CallOption) (SchedulerForFrontend_FrontendLoopClient, error) -} - -type schedulerForFrontendClient struct { - cc *grpc.ClientConn -} - -func NewSchedulerForFrontendClient(cc *grpc.ClientConn) SchedulerForFrontendClient { - return &schedulerForFrontendClient{cc} -} - -func (c *schedulerForFrontendClient) FrontendLoop(ctx context.Context, opts ...grpc.CallOption) (SchedulerForFrontend_FrontendLoopClient, error) { - stream, err := c.cc.NewStream(ctx, &_SchedulerForFrontend_serviceDesc.Streams[0], "/schedulerpb.SchedulerForFrontend/FrontendLoop", opts...) - if err != nil { - return nil, err - } - x := &schedulerForFrontendFrontendLoopClient{stream} - return x, nil -} - -type SchedulerForFrontend_FrontendLoopClient interface { - Send(*FrontendToScheduler) error - Recv() (*SchedulerToFrontend, error) - grpc.ClientStream -} - -type schedulerForFrontendFrontendLoopClient struct { - grpc.ClientStream -} - -func (x *schedulerForFrontendFrontendLoopClient) Send(m *FrontendToScheduler) error { - return x.ClientStream.SendMsg(m) -} - -func (x *schedulerForFrontendFrontendLoopClient) Recv() (*SchedulerToFrontend, error) { - m := new(SchedulerToFrontend) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// SchedulerForFrontendServer is the server API for SchedulerForFrontend service. -type SchedulerForFrontendServer interface { - // After calling this method, both Frontend and Scheduler enter a loop. Frontend will keep sending ENQUEUE and - // CANCEL requests, and scheduler is expected to process them. Scheduler returns one response for each request. - // - // Long-running loop is used to detect broken connection between frontend and scheduler. This is important for both - // parties... if connection breaks, frontend can cancel (and possibly retry on different scheduler) all pending - // requests sent to this scheduler, while scheduler can cancel queued requests from given frontend. - FrontendLoop(SchedulerForFrontend_FrontendLoopServer) error -} - -// UnimplementedSchedulerForFrontendServer can be embedded to have forward compatible implementations. -type UnimplementedSchedulerForFrontendServer struct { -} - -func (*UnimplementedSchedulerForFrontendServer) FrontendLoop(srv SchedulerForFrontend_FrontendLoopServer) error { - return status.Errorf(codes.Unimplemented, "method FrontendLoop not implemented") -} - -func RegisterSchedulerForFrontendServer(s *grpc.Server, srv SchedulerForFrontendServer) { - s.RegisterService(&_SchedulerForFrontend_serviceDesc, srv) -} - -func _SchedulerForFrontend_FrontendLoop_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(SchedulerForFrontendServer).FrontendLoop(&schedulerForFrontendFrontendLoopServer{stream}) -} - -type SchedulerForFrontend_FrontendLoopServer interface { - Send(*SchedulerToFrontend) error - Recv() (*FrontendToScheduler, error) - grpc.ServerStream -} - -type schedulerForFrontendFrontendLoopServer struct { - grpc.ServerStream -} - -func (x *schedulerForFrontendFrontendLoopServer) Send(m *SchedulerToFrontend) error { - return x.ServerStream.SendMsg(m) -} - -func (x *schedulerForFrontendFrontendLoopServer) Recv() (*FrontendToScheduler, error) { - m := new(FrontendToScheduler) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -var _SchedulerForFrontend_serviceDesc = grpc.ServiceDesc{ - ServiceName: "schedulerpb.SchedulerForFrontend", - HandlerType: (*SchedulerForFrontendServer)(nil), - Methods: []grpc.MethodDesc{}, - Streams: []grpc.StreamDesc{ - { - StreamName: "FrontendLoop", - Handler: _SchedulerForFrontend_FrontendLoop_Handler, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "scheduler.proto", -} - -func (m *QuerierToScheduler) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *QuerierToScheduler) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *QuerierToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.QuerierID) > 0 { - i -= len(m.QuerierID) - copy(dAtA[i:], m.QuerierID) - i = encodeVarintScheduler(dAtA, i, uint64(len(m.QuerierID))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *SchedulerToQuerier) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *SchedulerToQuerier) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *SchedulerToQuerier) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.StatsEnabled { - i-- - if m.StatsEnabled { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x28 - } - if len(m.UserID) > 0 { - i -= len(m.UserID) - copy(dAtA[i:], m.UserID) - i = encodeVarintScheduler(dAtA, i, uint64(len(m.UserID))) - i-- - dAtA[i] = 0x22 - } - if len(m.FrontendAddress) > 0 { - i -= len(m.FrontendAddress) - copy(dAtA[i:], m.FrontendAddress) - i = encodeVarintScheduler(dAtA, i, uint64(len(m.FrontendAddress))) - i-- - dAtA[i] = 0x1a - } - if m.HttpRequest != nil { - { - size, err := m.HttpRequest.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintScheduler(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - if m.QueryID != 0 { - i = encodeVarintScheduler(dAtA, i, uint64(m.QueryID)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *FrontendToScheduler) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *FrontendToScheduler) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *FrontendToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.StatsEnabled { - i-- - if m.StatsEnabled { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x30 - } - if m.HttpRequest != nil { - { - size, err := m.HttpRequest.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintScheduler(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x2a - } - if len(m.UserID) > 0 { - i -= len(m.UserID) - copy(dAtA[i:], m.UserID) - i = encodeVarintScheduler(dAtA, i, uint64(len(m.UserID))) - i-- - dAtA[i] = 0x22 - } - if m.QueryID != 0 { - i = encodeVarintScheduler(dAtA, i, uint64(m.QueryID)) - i-- - dAtA[i] = 0x18 - } - if len(m.FrontendAddress) > 0 { - i -= len(m.FrontendAddress) - copy(dAtA[i:], m.FrontendAddress) - i = encodeVarintScheduler(dAtA, i, uint64(len(m.FrontendAddress))) - i-- - dAtA[i] = 0x12 - } - if m.Type != 0 { - i = encodeVarintScheduler(dAtA, i, uint64(m.Type)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *SchedulerToFrontend) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *SchedulerToFrontend) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *SchedulerToFrontend) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Error) > 0 { - i -= len(m.Error) - copy(dAtA[i:], m.Error) - i = encodeVarintScheduler(dAtA, i, uint64(len(m.Error))) - i-- - dAtA[i] = 0x12 - } - if m.Status != 0 { - i = encodeVarintScheduler(dAtA, i, uint64(m.Status)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *NotifyQuerierShutdownRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *NotifyQuerierShutdownRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *NotifyQuerierShutdownRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.QuerierID) > 0 { - i -= len(m.QuerierID) - copy(dAtA[i:], m.QuerierID) - i = encodeVarintScheduler(dAtA, i, uint64(len(m.QuerierID))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *NotifyQuerierShutdownResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *NotifyQuerierShutdownResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *NotifyQuerierShutdownResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - return len(dAtA) - i, nil -} - -func encodeVarintScheduler(dAtA []byte, offset int, v uint64) int { - offset -= sovScheduler(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *QuerierToScheduler) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.QuerierID) - if l > 0 { - n += 1 + l + sovScheduler(uint64(l)) - } - return n -} - -func (m *SchedulerToQuerier) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.QueryID != 0 { - n += 1 + sovScheduler(uint64(m.QueryID)) - } - if m.HttpRequest != nil { - l = m.HttpRequest.Size() - n += 1 + l + sovScheduler(uint64(l)) - } - l = len(m.FrontendAddress) - if l > 0 { - n += 1 + l + sovScheduler(uint64(l)) - } - l = len(m.UserID) - if l > 0 { - n += 1 + l + sovScheduler(uint64(l)) - } - if m.StatsEnabled { - n += 2 - } - return n -} - -func (m *FrontendToScheduler) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Type != 0 { - n += 1 + sovScheduler(uint64(m.Type)) - } - l = len(m.FrontendAddress) - if l > 0 { - n += 1 + l + sovScheduler(uint64(l)) - } - if m.QueryID != 0 { - n += 1 + sovScheduler(uint64(m.QueryID)) - } - l = len(m.UserID) - if l > 0 { - n += 1 + l + sovScheduler(uint64(l)) - } - if m.HttpRequest != nil { - l = m.HttpRequest.Size() - n += 1 + l + sovScheduler(uint64(l)) - } - if m.StatsEnabled { - n += 2 - } - return n -} - -func (m *SchedulerToFrontend) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Status != 0 { - n += 1 + sovScheduler(uint64(m.Status)) - } - l = len(m.Error) - if l > 0 { - n += 1 + l + sovScheduler(uint64(l)) - } - return n -} - -func (m *NotifyQuerierShutdownRequest) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.QuerierID) - if l > 0 { - n += 1 + l + sovScheduler(uint64(l)) - } - return n -} - -func (m *NotifyQuerierShutdownResponse) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - return n -} - -func sovScheduler(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozScheduler(x uint64) (n int) { - return sovScheduler(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (this *QuerierToScheduler) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&QuerierToScheduler{`, - `QuerierID:` + fmt.Sprintf("%v", this.QuerierID) + `,`, - `}`, - }, "") - return s -} -func (this *SchedulerToQuerier) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&SchedulerToQuerier{`, - `QueryID:` + fmt.Sprintf("%v", this.QueryID) + `,`, - `HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`, - `FrontendAddress:` + fmt.Sprintf("%v", this.FrontendAddress) + `,`, - `UserID:` + fmt.Sprintf("%v", this.UserID) + `,`, - `StatsEnabled:` + fmt.Sprintf("%v", this.StatsEnabled) + `,`, - `}`, - }, "") - return s -} -func (this *FrontendToScheduler) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&FrontendToScheduler{`, - `Type:` + fmt.Sprintf("%v", this.Type) + `,`, - `FrontendAddress:` + fmt.Sprintf("%v", this.FrontendAddress) + `,`, - `QueryID:` + fmt.Sprintf("%v", this.QueryID) + `,`, - `UserID:` + fmt.Sprintf("%v", this.UserID) + `,`, - `HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`, - `StatsEnabled:` + fmt.Sprintf("%v", this.StatsEnabled) + `,`, - `}`, - }, "") - return s -} -func (this *SchedulerToFrontend) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&SchedulerToFrontend{`, - `Status:` + fmt.Sprintf("%v", this.Status) + `,`, - `Error:` + fmt.Sprintf("%v", this.Error) + `,`, - `}`, - }, "") - return s -} -func (this *NotifyQuerierShutdownRequest) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&NotifyQuerierShutdownRequest{`, - `QuerierID:` + fmt.Sprintf("%v", this.QuerierID) + `,`, - `}`, - }, "") - return s -} -func (this *NotifyQuerierShutdownResponse) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&NotifyQuerierShutdownResponse{`, - `}`, - }, "") - return s -} -func valueToStringScheduler(v interface{}) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("*%v", pv) -} -func (m *QuerierToScheduler) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: QuerierToScheduler: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: QuerierToScheduler: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field QuerierID", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.QuerierID = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipScheduler(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *SchedulerToQuerier) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: SchedulerToQuerier: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: SchedulerToQuerier: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field QueryID", wireType) - } - m.QueryID = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.QueryID |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field HttpRequest", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.HttpRequest == nil { - m.HttpRequest = &httpgrpc.HTTPRequest{} - } - if err := m.HttpRequest.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field FrontendAddress", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.FrontendAddress = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field UserID", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.UserID = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field StatsEnabled", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.StatsEnabled = bool(v != 0) - default: - iNdEx = preIndex - skippy, err := skipScheduler(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: FrontendToScheduler: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: FrontendToScheduler: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) - } - m.Type = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Type |= FrontendToSchedulerType(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field FrontendAddress", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.FrontendAddress = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field QueryID", wireType) - } - m.QueryID = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.QueryID |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field UserID", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.UserID = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field HttpRequest", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.HttpRequest == nil { - m.HttpRequest = &httpgrpc.HTTPRequest{} - } - if err := m.HttpRequest.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 6: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field StatsEnabled", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.StatsEnabled = bool(v != 0) - default: - iNdEx = preIndex - skippy, err := skipScheduler(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *SchedulerToFrontend) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: SchedulerToFrontend: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: SchedulerToFrontend: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) - } - m.Status = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Status |= SchedulerToFrontendStatus(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Error = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipScheduler(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *NotifyQuerierShutdownRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: NotifyQuerierShutdownRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: NotifyQuerierShutdownRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field QuerierID", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.QuerierID = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipScheduler(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *NotifyQuerierShutdownResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: NotifyQuerierShutdownResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: NotifyQuerierShutdownResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - default: - iNdEx = preIndex - skippy, err := skipScheduler(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipScheduler(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowScheduler - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowScheduler - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowScheduler - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthScheduler - } - iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthScheduler - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowScheduler - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipScheduler(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthScheduler - } - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} - -var ( - ErrInvalidLengthScheduler = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowScheduler = fmt.Errorf("proto: integer overflow") -) diff --git a/pkg/scheduler/schedulerpb/scheduler.proto b/pkg/scheduler/schedulerpb/scheduler.proto deleted file mode 100644 index 3ae6437567a..00000000000 --- a/pkg/scheduler/schedulerpb/scheduler.proto +++ /dev/null @@ -1,99 +0,0 @@ -syntax = "proto3"; - -package schedulerpb; - -option go_package = "schedulerpb"; - -import "github.com/gogo/protobuf/gogoproto/gogo.proto"; -import "github.com/weaveworks/common/httpgrpc/httpgrpc.proto"; - -option (gogoproto.marshaler_all) = true; -option (gogoproto.unmarshaler_all) = true; - -// Scheduler interface exposed to Queriers. -service SchedulerForQuerier { - // After calling this method, both Querier and Scheduler enter a loop, in which querier waits for - // "SchedulerToQuerier" messages containing HTTP requests and processes them. After processing the request, - // querier signals that it is ready to accept another one by sending empty QuerierToScheduler message. - // - // Long-running loop is used to detect broken connection between scheduler and querier. This is important - // for scheduler to keep a list of connected queriers up-to-date. - rpc QuerierLoop(stream QuerierToScheduler) returns (stream SchedulerToQuerier) { }; - - // The querier notifies the query-scheduler that it started a graceful shutdown. - rpc NotifyQuerierShutdown(NotifyQuerierShutdownRequest) returns (NotifyQuerierShutdownResponse); -} - -// Querier reports its own clientID when it connects, so that scheduler knows how many *different* queriers are connected. -// To signal that querier is ready to accept another request, querier sends empty message. -message QuerierToScheduler { - string querierID = 1; -} - -message SchedulerToQuerier { - // Query ID as reported by frontend. When querier sends the response back to frontend (using frontendAddress), - // it identifies the query by using this ID. - uint64 queryID = 1; - httpgrpc.HTTPRequest httpRequest = 2; - - // Where should querier send HTTP Response to (using FrontendForQuerier interface). - string frontendAddress = 3; - - // User who initiated the request. Needed to send reply back to frontend. - string userID = 4; - - // Whether query statistics tracking should be enabled. The response will include - // statistics only when this option is enabled. - bool statsEnabled = 5; -} - -// Scheduler interface exposed to Frontend. Frontend can enqueue and cancel requests. -service SchedulerForFrontend { - // After calling this method, both Frontend and Scheduler enter a loop. Frontend will keep sending ENQUEUE and - // CANCEL requests, and scheduler is expected to process them. Scheduler returns one response for each request. - // - // Long-running loop is used to detect broken connection between frontend and scheduler. This is important for both - // parties... if connection breaks, frontend can cancel (and possibly retry on different scheduler) all pending - // requests sent to this scheduler, while scheduler can cancel queued requests from given frontend. - rpc FrontendLoop(stream FrontendToScheduler) returns (stream SchedulerToFrontend) { }; -} - -enum FrontendToSchedulerType { - INIT = 0; - ENQUEUE = 1; - CANCEL = 2; -} - -message FrontendToScheduler { - FrontendToSchedulerType type = 1; - - // Used by INIT message. Will be put into all requests passed to querier. - string frontendAddress = 2; - - // Used by ENQUEUE and CANCEL. - // Each frontend manages its own queryIDs. Different frontends may use same set of query IDs. - uint64 queryID = 3; - - // Following are used by ENQUEUE only. - string userID = 4; - httpgrpc.HTTPRequest httpRequest = 5; - bool statsEnabled = 6; -} - -enum SchedulerToFrontendStatus { - OK = 0; - TOO_MANY_REQUESTS_PER_TENANT = 1; - ERROR = 2; - SHUTTING_DOWN = 3; -} - -message SchedulerToFrontend { - SchedulerToFrontendStatus status = 1; - string error = 2; -} - -message NotifyQuerierShutdownRequest { - string querierID = 1; -} - -message NotifyQuerierShutdownResponse {} diff --git a/pkg/util/client.go b/pkg/util/client.go index 42e35fecc77..6d8d7d68907 100644 --- a/pkg/util/client.go +++ b/pkg/util/client.go @@ -8,7 +8,7 @@ import ( "strconv" "github.com/golang/protobuf/jsonpb" //nolint:all - "github.com/golang/protobuf/proto" //nolint:all + "github.com/golang/protobuf/proto" //nolint:all "github.com/grafana/tempo/pkg/tempopb" "github.com/klauspost/compress/gzhttp" diff --git a/tempodb/encoding/vparquet/block_iterator.go b/tempodb/encoding/vparquet/block_iterator.go index 699e0dd7478..9b9aef11566 100644 --- a/tempodb/encoding/vparquet/block_iterator.go +++ b/tempodb/encoding/vparquet/block_iterator.go @@ -13,7 +13,7 @@ import ( "github.com/grafana/tempo/tempodb/encoding/common" ) -func (b *backendBlock) open(ctx context.Context) (*parquet.File, *parquet.Reader, error) { //nolint:all //deprecated +func (b *backendBlock) open(ctx context.Context) (*parquet.File, *parquet.Reader, error) { //nolint:all //deprecated rr := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta.BlockID, b.meta.TenantID) // 128 MB memory buffering @@ -53,7 +53,7 @@ func (b *backendBlock) RawIterator(ctx context.Context, pool *rowPool) (*rawIter type blockIterator struct { blockID string - r *parquet.Reader //nolint:all //deprecated + r *parquet.Reader //nolint:all //deprecated } func (i *blockIterator) Next(context.Context) (*Trace, error) { @@ -74,7 +74,7 @@ func (i *blockIterator) Close() { type rawIterator struct { blockID string - r *parquet.Reader //nolint:all //deprecated + r *parquet.Reader //nolint:all //deprecated traceIDIndex int pool *rowPool }