From 29575829557ed16489adc762c5b16c1a57e89da0 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 11 Oct 2024 16:29:52 -0700 Subject: [PATCH 01/18] update --- cmd/thanos/receive.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 569ec1e57e..1ce60291f5 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -5,6 +5,7 @@ package main import ( "context" + "net/http" "os" "path" "strings" @@ -330,6 +331,12 @@ func runReceive( }) } + level.Debug(logger).Log("msg", "setting up downscale HTTP server") + { + http.HandleFunc("/downscale", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) + http.ListenAndServe(":8080", nil) + } + level.Debug(logger).Log("msg", "setting up gRPC server") { tlsCfg, err := tls.NewServerConfig(log.With(logger, "protocol", "gRPC"), conf.grpcConfig.tlsSrvCert, conf.grpcConfig.tlsSrvKey, conf.grpcConfig.tlsSrvClientCA) From 26c785ba47534e41628f04c73149b5ea94b27b4c Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 11 Oct 2024 21:49:16 -0700 Subject: [PATCH 02/18] update --- cmd/thanos/receive.go | 8 +------- pkg/receive/multitsdb.go | 4 ++++ pkg/server/http/http.go | 8 ++++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 1ce60291f5..7ab8cb2e13 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -5,7 +5,6 @@ package main import ( "context" - "net/http" "os" "path" "strings" @@ -320,6 +319,7 @@ func runReceive( httpserver.WithGracePeriod(time.Duration(*conf.httpGracePeriod)), httpserver.WithTLSConfig(*conf.httpTLSConfig), ) + httpserver.RegisterDownscale(srv, dbs.GetTenants()) g.Add(func() error { statusProber.Healthy() return srv.ListenAndServe() @@ -331,12 +331,6 @@ func runReceive( }) } - level.Debug(logger).Log("msg", "setting up downscale HTTP server") - { - http.HandleFunc("/downscale", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) - http.ListenAndServe(":8080", nil) - } - level.Debug(logger).Log("msg", "setting up gRPC server") { tlsCfg, err := tls.NewServerConfig(log.With(logger, "protocol", "gRPC"), conf.grpcConfig.tlsSrvCert, conf.grpcConfig.tlsSrvKey, conf.grpcConfig.tlsSrvClientCA) diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index ccecd39523..71d2826c0f 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -122,6 +122,10 @@ func NewMultiTSDB( return mt } +func (t *MultiTSDB) GetTenants() map[string]*tenant { + return t.tenants +} + type localClient struct { store *store.TSDBStore diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index fc92100e3f..e65bcc45de 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -5,6 +5,7 @@ package http import ( "context" + "fmt" "net/http" "net/http/pprof" @@ -68,6 +69,13 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, } } +func RegisterDownscale[K comparable, V any](s *Server, m map[K]V) { + s.mux.Handle("/-/downscale", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, len(m)) + })) +} + // ListenAndServe listens on the TCP network address and handles requests on incoming connections. func (s *Server) ListenAndServe() error { level.Info(s.logger).Log("msg", "listening for requests and metrics", "address", s.opts.listen) From 12e581d829e3a0551d46a762fcab02b7419652c0 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 11 Oct 2024 22:22:36 -0700 Subject: [PATCH 03/18] update --- pkg/server/http/http.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index e65bcc45de..f0a142f626 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -5,9 +5,10 @@ package http import ( "context" - "fmt" + "encoding/json" "net/http" "net/http/pprof" + "time" "github.com/felixge/fgprof" "github.com/go-kit/log" @@ -71,8 +72,19 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, func RegisterDownscale[K comparable, V any](s *Server, m map[K]V) { s.mux.Handle("/-/downscale", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - fmt.Fprintln(w, len(m)) + if r.Method == http.MethodPost { + if len(m) == 0 { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(struct { + Timestamp int64 `json:"timestamp"` + }{Timestamp: time.Now().Unix()}) + } else { + w.WriteHeader(http.StatusForbidden) + } + } else { + w.WriteHeader(http.StatusOK) + } })) } From c24fef43e49e77f8ada1c4a075ecb31dde72e409 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 11 Oct 2024 22:23:16 -0700 Subject: [PATCH 04/18] update --- pkg/server/http/http.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index f0a142f626..4eb818ab19 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -8,6 +8,7 @@ import ( "encoding/json" "net/http" "net/http/pprof" + "strconv" "time" "github.com/felixge/fgprof" @@ -72,6 +73,7 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, func RegisterDownscale[K comparable, V any](s *Server, m map[K]V) { s.mux.Handle("/-/downscale", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Tenant-Count", strconv.Itoa(len(m))) if r.Method == http.MethodPost { if len(m) == 0 { w.WriteHeader(http.StatusOK) From 5f18925c5697f67e724456f32b7e525ab128dc7c Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 11 Oct 2024 22:44:01 -0700 Subject: [PATCH 05/18] update --- cmd/thanos/receive.go | 1 + pkg/server/http/http.go | 35 +++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 7ab8cb2e13..ecd9692c06 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -319,6 +319,7 @@ func runReceive( httpserver.WithGracePeriod(time.Duration(*conf.httpGracePeriod)), httpserver.WithTLSConfig(*conf.httpTLSConfig), ) + var lastDownscalePrepareTimestamp *int64 = nil httpserver.RegisterDownscale(srv, dbs.GetTenants()) g.Add(func() error { statusProber.Healthy() diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index 4eb818ab19..e36c078b81 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -6,11 +6,6 @@ package http import ( "context" "encoding/json" - "net/http" - "net/http/pprof" - "strconv" - "time" - "github.com/felixge/fgprof" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -20,6 +15,10 @@ import ( toolkit_web "github.com/prometheus/exporter-toolkit/web" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" + "net/http" + "net/http/pprof" + "strconv" + "time" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/prober" @@ -71,21 +70,25 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, } } -func RegisterDownscale[K comparable, V any](s *Server, m map[K]V) { +func RegisterDownscale[K comparable, V any](s *Server, m map[K]V, t *int64) { s.mux.Handle("/-/downscale", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Tenant-Count", strconv.Itoa(len(m))) - if r.Method == http.MethodPost { - if len(m) == 0 { - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(struct { - Timestamp int64 `json:"timestamp"` - }{Timestamp: time.Now().Unix()}) - } else { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusOK) + if r.Method == http.MethodDelete { + return + } + w.Header().Set("Content-Type", "application/json") + if len(m) == 0 { + level.Info(s.logger).Log("msg", "no tenants, good to downscale") + if t == nil { + *t = time.Now().Unix() } + json.NewEncoder(w).Encode(struct { + Timestamp int64 `json:"timestamp"` + }{Timestamp: *t}) } else { - w.WriteHeader(http.StatusOK) + *t = time.Now().Unix() + w.WriteHeader(http.StatusForbidden) } })) } From 681fd2a4ac0bff8944b68727b65785621aade468 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 11 Oct 2024 22:46:59 -0700 Subject: [PATCH 06/18] update --- cmd/thanos/receive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index ecd9692c06..99c289a8fe 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -320,7 +320,7 @@ func runReceive( httpserver.WithTLSConfig(*conf.httpTLSConfig), ) var lastDownscalePrepareTimestamp *int64 = nil - httpserver.RegisterDownscale(srv, dbs.GetTenants()) + httpserver.RegisterDownscale(srv, dbs.GetTenants(), lastDownscalePrepareTimestamp) g.Add(func() error { statusProber.Healthy() return srv.ListenAndServe() From 481dafdfa8ffcd898b1b6d8f3e649d50eafe73b7 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 11 Oct 2024 22:59:54 -0700 Subject: [PATCH 07/18] update --- pkg/server/http/http.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index e36c078b81..d6723ae535 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -77,12 +77,13 @@ func RegisterDownscale[K comparable, V any](s *Server, m map[K]V, t *int64) { if r.Method == http.MethodDelete { return } + if t == nil { + now := time.Now().Unix() + t = &now + } w.Header().Set("Content-Type", "application/json") if len(m) == 0 { level.Info(s.logger).Log("msg", "no tenants, good to downscale") - if t == nil { - *t = time.Now().Unix() - } json.NewEncoder(w).Encode(struct { Timestamp int64 `json:"timestamp"` }{Timestamp: *t}) From ebb4a0cdeced3b051ac39532e3dd38cd1efa4753 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 11 Oct 2024 23:00:22 -0700 Subject: [PATCH 08/18] update --- pkg/server/http/http.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index d6723ae535..f3a7d9623f 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -89,7 +89,6 @@ func RegisterDownscale[K comparable, V any](s *Server, m map[K]V, t *int64) { }{Timestamp: *t}) } else { *t = time.Now().Unix() - w.WriteHeader(http.StatusForbidden) } })) } From d77381436d45d974e2b52855b3ecdccbe137caed Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 11 Oct 2024 23:14:10 -0700 Subject: [PATCH 09/18] update --- pkg/server/http/http.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index f3a7d9623f..7288db747d 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -77,19 +77,14 @@ func RegisterDownscale[K comparable, V any](s *Server, m map[K]V, t *int64) { if r.Method == http.MethodDelete { return } - if t == nil { + w.Header().Set("Content-Type", "application/json") + if t == nil || len(m) > 0 { now := time.Now().Unix() t = &now } - w.Header().Set("Content-Type", "application/json") - if len(m) == 0 { - level.Info(s.logger).Log("msg", "no tenants, good to downscale") - json.NewEncoder(w).Encode(struct { - Timestamp int64 `json:"timestamp"` - }{Timestamp: *t}) - } else { - *t = time.Now().Unix() - } + json.NewEncoder(w).Encode(struct { + Timestamp int64 `json:"timestamp"` + }{Timestamp: *t}) })) } From 27069226b81c3f1016ffb6d81b8b73bac7fcb0dd Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 11 Oct 2024 23:32:04 -0700 Subject: [PATCH 10/18] update --- cmd/thanos/receive.go | 2 +- pkg/receive/multitsdb.go | 4 ++++ pkg/server/http/http.go | 7 ++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 99c289a8fe..1e49362ae5 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -320,7 +320,7 @@ func runReceive( httpserver.WithTLSConfig(*conf.httpTLSConfig), ) var lastDownscalePrepareTimestamp *int64 = nil - httpserver.RegisterDownscale(srv, dbs.GetTenants(), lastDownscalePrepareTimestamp) + httpserver.RegisterDownscale(srv, dbs.GetTenants(), dbs.GetMutex(), lastDownscalePrepareTimestamp) g.Add(func() error { statusProber.Healthy() return srv.ListenAndServe() diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index 71d2826c0f..d6a982d291 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -126,6 +126,10 @@ func (t *MultiTSDB) GetTenants() map[string]*tenant { return t.tenants } +func (t *MultiTSDB) GetMutex() *sync.RWMutex { + return t.mtx +} + type localClient struct { store *store.TSDBStore diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index 7288db747d..3abe1433c4 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -18,6 +18,7 @@ import ( "net/http" "net/http/pprof" "strconv" + "sync" "time" "github.com/thanos-io/thanos/pkg/component" @@ -70,13 +71,17 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, } } -func RegisterDownscale[K comparable, V any](s *Server, m map[K]V, t *int64) { +// RegisterDownscale registers HTTP handler compatible with Grafana rollout operator. +// See https://github.com/databricks/rollout-operator?tab=readme-ov-file#delayed-scaledown. +func RegisterDownscale[K comparable, V any](s *Server, m map[K]V, mtx *sync.RWMutex, t *int64) { s.mux.Handle("/-/downscale", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Tenant-Count", strconv.Itoa(len(m))) w.WriteHeader(http.StatusOK) if r.Method == http.MethodDelete { return } + mtx.RLock() + defer mtx.RUnlock() w.Header().Set("Content-Type", "application/json") if t == nil || len(m) > 0 { now := time.Now().Unix() From 7d45490a18508ae77177809c847f3df7f5aab1f8 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Tue, 15 Oct 2024 08:42:18 -0700 Subject: [PATCH 11/18] update --- cmd/thanos/receive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 1e49362ae5..cbb6353820 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -464,7 +464,7 @@ func runReceive( ctx, cancel := context.WithCancel(context.Background()) g.Add(func() error { return runutil.Repeat(conf.topMetricsUpdateInterval, ctx.Done(), func() error { - level.Info(logger).Log("msg", "getting top metrics") + level.Debug(logger).Log("msg", "getting top metrics") for _, ts := range dbs.TenantStats(conf.numTopMetricsPerTenant, labels.MetricName) { for _, ms := range ts.Stats.IndexPostingStats.CardinalityMetricsStats { if ms.Count >= conf.topMetricsMinimumCardinality { From 735304f60c95a529fe28b55f609d02cc0d6d1206 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Tue, 15 Oct 2024 08:46:53 -0700 Subject: [PATCH 12/18] update --- pkg/server/http/http.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index 3abe1433c4..bac68d5ff5 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -75,15 +75,16 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, // See https://github.com/databricks/rollout-operator?tab=readme-ov-file#delayed-scaledown. func RegisterDownscale[K comparable, V any](s *Server, m map[K]V, mtx *sync.RWMutex, t *int64) { s.mux.Handle("/-/downscale", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Tenant-Count", strconv.Itoa(len(m))) + mtx.RLock() + n := len(m) + mtx.RUnlock() + w.Header().Set("Tenant-Count", strconv.Itoa(n)) w.WriteHeader(http.StatusOK) if r.Method == http.MethodDelete { return } - mtx.RLock() - defer mtx.RUnlock() w.Header().Set("Content-Type", "application/json") - if t == nil || len(m) > 0 { + if t == nil || n > 0 { now := time.Now().Unix() t = &now } From ffc7bc33d8888a5ae4b95048e8937edfb12b8047 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Tue, 15 Oct 2024 11:18:32 -0700 Subject: [PATCH 13/18] update --- cmd/thanos/receive.go | 21 +++++++++++++++++++-- pkg/receive/multitsdb.go | 10 ++++------ pkg/server/http/http.go | 30 ------------------------------ 3 files changed, 23 insertions(+), 38 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index cbb6353820..3ebf1e53b4 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -5,8 +5,11 @@ package main import ( "context" + "encoding/json" + "net/http" "os" "path" + "strconv" "strings" "time" @@ -319,8 +322,22 @@ func runReceive( httpserver.WithGracePeriod(time.Duration(*conf.httpGracePeriod)), httpserver.WithTLSConfig(*conf.httpTLSConfig), ) - var lastDownscalePrepareTimestamp *int64 = nil - httpserver.RegisterDownscale(srv, dbs.GetTenants(), dbs.GetMutex(), lastDownscalePrepareTimestamp) + var lastDownscalePrepareTimestamp int64 = 0 + srv.Handle("/-/downscale", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := dbs.GetTenantsLen() + w.Header().Set("Tenant-Count", strconv.Itoa(n)) + w.WriteHeader(http.StatusOK) + if r.Method == http.MethodDelete { + return + } + w.Header().Set("Content-Type", "application/json") + if lastDownscalePrepareTimestamp == 0 || n > 0 { + lastDownscalePrepareTimestamp = time.Now().Unix() + } + json.NewEncoder(w).Encode(struct { + Timestamp int64 `json:"timestamp"` + }{Timestamp: lastDownscalePrepareTimestamp}) + })) g.Add(func() error { statusProber.Healthy() return srv.ListenAndServe() diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index d6a982d291..5646aab877 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -122,12 +122,10 @@ func NewMultiTSDB( return mt } -func (t *MultiTSDB) GetTenants() map[string]*tenant { - return t.tenants -} - -func (t *MultiTSDB) GetMutex() *sync.RWMutex { - return t.mtx +func (t *MultiTSDB) GetTenantsLen() int { + t.mtx.RLock() + defer t.mtx.RUnlock() + return len(t.tenants) } type localClient struct { diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index bac68d5ff5..99e6bcf7f4 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -5,7 +5,6 @@ package http import ( "context" - "encoding/json" "github.com/felixge/fgprof" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -17,12 +16,6 @@ import ( "golang.org/x/net/http2/h2c" "net/http" "net/http/pprof" - "strconv" - "sync" - "time" - - "github.com/thanos-io/thanos/pkg/component" - "github.com/thanos-io/thanos/pkg/prober" ) // A Server defines parameters for serve HTTP requests, a wrapper around http.Server. @@ -71,29 +64,6 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component, } } -// RegisterDownscale registers HTTP handler compatible with Grafana rollout operator. -// See https://github.com/databricks/rollout-operator?tab=readme-ov-file#delayed-scaledown. -func RegisterDownscale[K comparable, V any](s *Server, m map[K]V, mtx *sync.RWMutex, t *int64) { - s.mux.Handle("/-/downscale", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - mtx.RLock() - n := len(m) - mtx.RUnlock() - w.Header().Set("Tenant-Count", strconv.Itoa(n)) - w.WriteHeader(http.StatusOK) - if r.Method == http.MethodDelete { - return - } - w.Header().Set("Content-Type", "application/json") - if t == nil || n > 0 { - now := time.Now().Unix() - t = &now - } - json.NewEncoder(w).Encode(struct { - Timestamp int64 `json:"timestamp"` - }{Timestamp: *t}) - })) -} - // ListenAndServe listens on the TCP network address and handles requests on incoming connections. func (s *Server) ListenAndServe() error { level.Info(s.logger).Log("msg", "listening for requests and metrics", "address", s.opts.listen) From 6c7e2c1e466303a8d2f68ab2039f2c5d293676c0 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Tue, 15 Oct 2024 11:18:52 -0700 Subject: [PATCH 14/18] update --- pkg/server/http/http.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index 99e6bcf7f4..f558dec190 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -12,6 +12,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" toolkit_web "github.com/prometheus/exporter-toolkit/web" + "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/prober" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" "net/http" From 2dac034fe12370f44a86b9e9e9a12c2e175ea585 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Tue, 15 Oct 2024 11:20:15 -0700 Subject: [PATCH 15/18] update --- pkg/server/http/http.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/server/http/http.go b/pkg/server/http/http.go index f558dec190..fc92100e3f 100644 --- a/pkg/server/http/http.go +++ b/pkg/server/http/http.go @@ -5,6 +5,9 @@ package http import ( "context" + "net/http" + "net/http/pprof" + "github.com/felixge/fgprof" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -12,12 +15,11 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" toolkit_web "github.com/prometheus/exporter-toolkit/web" - "github.com/thanos-io/thanos/pkg/component" - "github.com/thanos-io/thanos/pkg/prober" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" - "net/http" - "net/http/pprof" + + "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/prober" ) // A Server defines parameters for serve HTTP requests, a wrapper around http.Server. From 8da5b4ab0a1ad71496732738f89dd8de355c43dc Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Tue, 15 Oct 2024 16:47:20 -0700 Subject: [PATCH 16/18] add error handling --- cmd/thanos/receive.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 3ebf1e53b4..3f2d0aded3 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -334,9 +334,14 @@ func runReceive( if lastDownscalePrepareTimestamp == 0 || n > 0 { lastDownscalePrepareTimestamp = time.Now().Unix() } - json.NewEncoder(w).Encode(struct { + err := json.NewEncoder(w).Encode(struct { Timestamp int64 `json:"timestamp"` - }{Timestamp: lastDownscalePrepareTimestamp}) + }{ + Timestamp: lastDownscalePrepareTimestamp, + }) + if err != nil { + level.Error(logger).Log("msg", "error writing downscale response", "err", err) + } })) g.Add(func() error { statusProber.Healthy() From ea0d891a7c555f070fa843ce5987ccbb9b38c41d Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Fri, 18 Oct 2024 00:19:39 -0700 Subject: [PATCH 17/18] simplify endpoint logic --- cmd/thanos/receive.go | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 3f2d0aded3..79eee48e88 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -5,7 +5,6 @@ package main import ( "context" - "encoding/json" "net/http" "os" "path" @@ -322,25 +321,13 @@ func runReceive( httpserver.WithGracePeriod(time.Duration(*conf.httpGracePeriod)), httpserver.WithTLSConfig(*conf.httpTLSConfig), ) - var lastDownscalePrepareTimestamp int64 = 0 srv.Handle("/-/downscale", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { n := dbs.GetTenantsLen() w.Header().Set("Tenant-Count", strconv.Itoa(n)) - w.WriteHeader(http.StatusOK) - if r.Method == http.MethodDelete { - return - } - w.Header().Set("Content-Type", "application/json") - if lastDownscalePrepareTimestamp == 0 || n > 0 { - lastDownscalePrepareTimestamp = time.Now().Unix() - } - err := json.NewEncoder(w).Encode(struct { - Timestamp int64 `json:"timestamp"` - }{ - Timestamp: lastDownscalePrepareTimestamp, - }) - if err != nil { - level.Error(logger).Log("msg", "error writing downscale response", "err", err) + if n > 0 { + w.WriteHeader(http.StatusTooEarly) + } else { + w.WriteHeader(http.StatusOK) } })) g.Add(func() error { From 662332aa5696ea97c41b45a2f7657cf9c214ed99 Mon Sep 17 00:00:00 2001 From: Yuchen Wang Date: Wed, 13 Nov 2024 20:52:08 -0800 Subject: [PATCH 18/18] return tenants in http header --- cmd/thanos/receive.go | 8 ++++++-- pkg/receive/multitsdb.go | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 25f6e2f76d..0460947724 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -7,7 +7,7 @@ import ( "context" "fmt" "net" - "net/http" + "net/http" "os" "path" "strconv" @@ -325,8 +325,12 @@ func runReceive( httpserver.WithTLSConfig(*conf.httpTLSConfig), ) srv.Handle("/-/downscale", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - n := dbs.GetTenantsLen() + tenants := dbs.GetTenants() + n := len(tenants) w.Header().Set("Tenant-Count", strconv.Itoa(n)) + for _, tname := range tenants { + w.Header().Add("Tenants", tname) + } if n > 0 { w.WriteHeader(http.StatusTooEarly) } else { diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index bbcbb1eb50..d40cf43c06 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -122,10 +122,14 @@ func NewMultiTSDB( return mt } -func (t *MultiTSDB) GetTenantsLen() int { +func (t *MultiTSDB) GetTenants() []string { t.mtx.RLock() + tenants := make([]string, 0, len(t.tenants)) + for tname := range t.tenants { + tenants = append(tenants, tname) + } defer t.mtx.RUnlock() - return len(t.tenants) + return tenants } type localClient struct {