From d2e966db8858a5d6b06786224c81a6d0eae59204 Mon Sep 17 00:00:00 2001 From: Oleg Kovalov Date: Wed, 8 Dec 2021 10:15:10 +0100 Subject: [PATCH 1/3] Update hedgedhttp to v0.7.0 --- go.mod | 2 +- go.sum | 4 +- tempodb/backend/azure/azure_helpers.go | 5 +- tempodb/backend/gcs/gcs.go | 5 +- tempodb/backend/s3/s3.go | 5 +- .../github.com/cristalhq/hedgedhttp/README.md | 29 ++++- .../github.com/cristalhq/hedgedhttp/hedged.go | 122 ++++++------------ .../github.com/cristalhq/hedgedhttp/stats.go | 73 +++++++++++ vendor/modules.txt | 2 +- 9 files changed, 153 insertions(+), 94 deletions(-) create mode 100644 vendor/github.com/cristalhq/hedgedhttp/stats.go diff --git a/go.mod b/go.mod index 071b08c6ec3..72e3f339585 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/aws/aws-sdk-go v1.42.9 github.com/cespare/xxhash v1.1.0 github.com/cortexproject/cortex v1.10.1-0.20211104100946-3f329a21cad4 - github.com/cristalhq/hedgedhttp v0.6.0 + github.com/cristalhq/hedgedhttp v0.7.0 github.com/drone/envsubst v1.0.3 github.com/dustin/go-humanize v1.0.0 github.com/go-kit/log v0.2.0 diff --git a/go.sum b/go.sum index 2e55346dab5..59f1eca3750 100644 --- a/go.sum +++ b/go.sum @@ -535,8 +535,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cristalhq/hedgedhttp v0.6.0 h1:32REZ0SZ1q0xoRNpHP5ab+Qd3VseyURXjN3HFpBqTiE= -github.com/cristalhq/hedgedhttp v0.6.0/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI= +github.com/cristalhq/hedgedhttp v0.7.0 h1:C2XPDC+AQH4QJt6vZI4jB5WNyF86QbSJD4C4fW3H3ro= +github.com/cristalhq/hedgedhttp v0.7.0/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI= github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ= github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= diff --git a/tempodb/backend/azure/azure_helpers.go b/tempodb/backend/azure/azure_helpers.go index c10dd2adf63..99b936f8d84 100644 --- a/tempodb/backend/azure/azure_helpers.go +++ b/tempodb/backend/azure/azure_helpers.go @@ -53,7 +53,10 @@ func GetContainerURL(ctx context.Context, cfg *Config, hedge bool) (blob.Contain // hedge if desired (0 means disabled) if hedge && cfg.HedgeRequestsAt != 0 { - transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, cfg.HedgeRequestsUpTo, transport) + transport, stats, err = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, cfg.HedgeRequestsUpTo, transport) + if err != nil { + return blob.ContainerURL{}, err + } instrumentation.PublishHedgedMetrics(stats) } diff --git a/tempodb/backend/gcs/gcs.go b/tempodb/backend/gcs/gcs.go index db5c9b91111..be4112bdf48 100644 --- a/tempodb/backend/gcs/gcs.go +++ b/tempodb/backend/gcs/gcs.go @@ -244,7 +244,10 @@ func createBucket(ctx context.Context, cfg *Config, hedge bool) (*storage.Bucket // hedge if desired (0 means disabled) if hedge && cfg.HedgeRequestsAt != 0 { - transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, cfg.HedgeRequestsUpTo, transport) + transport, stats, err = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, cfg.HedgeRequestsUpTo, transport) + if err != nil { + return nil, err + } instrumentation.PublishHedgedMetrics(stats) } diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go index ded4e7ca62e..a4d1a4f78e6 100644 --- a/tempodb/backend/s3/s3.go +++ b/tempodb/backend/s3/s3.go @@ -347,7 +347,10 @@ func createCore(cfg *Config, hedge bool) (*minio.Core, error) { var stats *hedgedhttp.Stats if hedge && cfg.HedgeRequestsAt != 0 { - transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, cfg.HedgeRequestsUpTo, transport) + transport, stats, err = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, cfg.HedgeRequestsUpTo, transport) + if err != nil { + return nil, err + } instrumentation.PublishHedgedMetrics(stats) } diff --git a/vendor/github.com/cristalhq/hedgedhttp/README.md b/vendor/github.com/cristalhq/hedgedhttp/README.md index 2b6aa491b18..aec2a1b3548 100644 --- a/vendor/github.com/cristalhq/hedgedhttp/README.md +++ b/vendor/github.com/cristalhq/hedgedhttp/README.md @@ -4,6 +4,7 @@ [![pkg-img]][pkg-url] [![reportcard-img]][reportcard-url] [![coverage-img]][coverage-url] +[![version-img]][version-url] Hedged HTTP client which helps to reduce tail latency at scale. @@ -21,6 +22,7 @@ Thanks to [Bohdan Storozhuk](https://github.com/storozhukbm) for the review and * Easy to integrate. * Optimized for speed. * Clean and tested code. +* Supports `http.Client` and `http.RoundTripper`. * Dependency-free. ## Install @@ -33,7 +35,30 @@ go get github.com/cristalhq/hedgedhttp ## Example -TODO +```go +ctx := context.Background() +req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://google.com", http.NoBody) +if err != nil { + panic(err) +} + +timeout := 10 * time.Millisecond +upto := 7 +client := &http.Client{Timeout: time.Second} +hedged, err := hedgedhttp.NewClient(timeout, upto, client) +if err != nil { + panic(err) +} + +// will take `upto` requests, with a `timeout` delay between them +resp, err := hedged.Do(req) +if err != nil { + panic(err) +} +defer resp.Body.Close() +``` + +Also see examples: [examples_test.go](https://github.com/cristalhq/hedgedhttp/blob/main/examples_test.go). ## Documentation @@ -51,3 +76,5 @@ See [these docs][pkg-url]. [reportcard-url]: https://goreportcard.com/report/cristalhq/hedgedhttp [coverage-img]: https://codecov.io/gh/cristalhq/hedgedhttp/branch/main/graph/badge.svg [coverage-url]: https://codecov.io/gh/cristalhq/hedgedhttp +[version-img]: https://img.shields.io/github/v/release/cristalhq/hedgedhttp +[version-url]: https://github.com/cristalhq/hedgedhttp/releases diff --git a/vendor/github.com/cristalhq/hedgedhttp/hedged.go b/vendor/github.com/cristalhq/hedgedhttp/hedged.go index 9361f992757..56d65b0b1c4 100644 --- a/vendor/github.com/cristalhq/hedgedhttp/hedged.go +++ b/vendor/github.com/cristalhq/hedgedhttp/hedged.go @@ -2,11 +2,11 @@ package hedgedhttp import ( "context" + "errors" "fmt" "net/http" "strings" "sync" - "sync/atomic" "time" ) @@ -15,47 +15,56 @@ const infiniteTimeout = 30 * 24 * time.Hour // domain specific infinite // NewClient returns a new http.Client which implements hedged requests pattern. // Given Client starts a new request after a timeout from previous request. // Starts no more than upto requests. -func NewClient(timeout time.Duration, upto int, client *http.Client) *http.Client { - newClient, _ := NewClientAndStats(timeout, upto, client) - return newClient +func NewClient(timeout time.Duration, upto int, client *http.Client) (*http.Client, error) { + newClient, _, err := NewClientAndStats(timeout, upto, client) + if err != nil { + return nil, err + } + return newClient, nil } // NewClientAndStats returns a new http.Client which implements hedged requests pattern // And Stats object that can be queried to obtain client's metrics. // Given Client starts a new request after a timeout from previous request. // Starts no more than upto requests. -func NewClientAndStats(timeout time.Duration, upto int, client *http.Client) (*http.Client, *Stats) { +func NewClientAndStats(timeout time.Duration, upto int, client *http.Client) (*http.Client, *Stats, error) { if client == nil { client = &http.Client{ Timeout: 5 * time.Second, } } - newTransport, metrics := NewRoundTripperAndStats(timeout, upto, client.Transport) + newTransport, metrics, err := NewRoundTripperAndStats(timeout, upto, client.Transport) + if err != nil { + return nil, nil, err + } client.Transport = newTransport - return client, metrics + return client, metrics, nil } // NewRoundTripper returns a new http.RoundTripper which implements hedged requests pattern. // Given RoundTripper starts a new request after a timeout from previous request. // Starts no more than upto requests. -func NewRoundTripper(timeout time.Duration, upto int, rt http.RoundTripper) http.RoundTripper { - newRT, _ := NewRoundTripperAndStats(timeout, upto, rt) - return newRT +func NewRoundTripper(timeout time.Duration, upto int, rt http.RoundTripper) (http.RoundTripper, error) { + newRT, _, err := NewRoundTripperAndStats(timeout, upto, rt) + if err != nil { + return nil, err + } + return newRT, nil } // NewRoundTripperAndStats returns a new http.RoundTripper which implements hedged requests pattern // And Stats object that can be queried to obtain client's metrics. // Given RoundTripper starts a new request after a timeout from previous request. // Starts no more than upto requests. -func NewRoundTripperAndStats(timeout time.Duration, upto int, rt http.RoundTripper) (http.RoundTripper, *Stats) { +func NewRoundTripperAndStats(timeout time.Duration, upto int, rt http.RoundTripper) (http.RoundTripper, *Stats, error) { switch { case timeout < 0: - panic("hedgedhttp: timeout cannot be negative") + return nil, nil, errors.New("hedgedhttp: timeout cannot be negative") case upto < 1: - panic("hedgedhttp: upto must be greater than 0") + return nil, nil, errors.New("hedgedhttp: upto must be greater than 0") } if rt == nil { @@ -72,7 +81,7 @@ func NewRoundTripperAndStats(timeout time.Duration, upto int, rt http.RoundTripp upto: upto, metrics: &Stats{}, } - return hedged, hedged.metrics + return hedged, hedged.metrics, nil } type hedgedTransport struct { @@ -107,7 +116,7 @@ func (ht *hedgedTransport) RoundTrip(req *http.Request) (*http.Response, error) for sent := 0; len(errOverall.Errors) < ht.upto; sent++ { if sent < ht.upto { idx := sent - subReq, cancel := reqWithCtx(req, mainCtx) + subReq, cancel := reqWithCtx(req, mainCtx, idx != 0) cancels[idx] = cancel runInPool(func() { @@ -174,80 +183,21 @@ type indexedResp struct { Resp *http.Response } -func reqWithCtx(r *http.Request, ctx context.Context) (*http.Request, func()) { +func reqWithCtx(r *http.Request, ctx context.Context, isHedged bool) (*http.Request, context.CancelFunc) { ctx, cancel := context.WithCancel(ctx) + if isHedged { + ctx = context.WithValue(ctx, hedgedRequest{}, struct{}{}) + } req := r.WithContext(ctx) return req, cancel } -// atomicCounter is a false sharing safe counter. -type atomicCounter struct { - count uint64 - _ [7]uint64 -} +type hedgedRequest struct{} -type cacheLine [64]byte - -// Stats object that can be queried to obtain certain metrics and get better observability. -type Stats struct { - _ cacheLine - requestedRoundTrips atomicCounter - actualRoundTrips atomicCounter - failedRoundTrips atomicCounter - canceledByUserRoundTrips atomicCounter - canceledSubRequests atomicCounter - _ cacheLine -} - -func (s *Stats) requestedRoundTripsInc() { atomic.AddUint64(&s.requestedRoundTrips.count, 1) } -func (s *Stats) actualRoundTripsInc() { atomic.AddUint64(&s.actualRoundTrips.count, 1) } -func (s *Stats) failedRoundTripsInc() { atomic.AddUint64(&s.failedRoundTrips.count, 1) } -func (s *Stats) canceledByUserRoundTripsInc() { atomic.AddUint64(&s.canceledByUserRoundTrips.count, 1) } -func (s *Stats) canceledSubRequestsInc() { atomic.AddUint64(&s.canceledSubRequests.count, 1) } - -// RequestedRoundTrips returns count of requests that were requested by client. -func (m *Stats) RequestedRoundTrips() uint64 { - return atomic.LoadUint64(&m.requestedRoundTrips.count) -} - -// ActualRoundTrips returns count of requests that were actually sent. -func (m *Stats) ActualRoundTrips() uint64 { - return atomic.LoadUint64(&m.actualRoundTrips.count) -} - -// FailedRoundTrips returns count of requests that failed. -func (m *Stats) FailedRoundTrips() uint64 { - return atomic.LoadUint64(&m.failedRoundTrips.count) -} - -// CanceledByUserRoundTrips returns count of requests that were canceled by user, using request context. -func (m *Stats) CanceledByUserRoundTrips() uint64 { - return atomic.LoadUint64(&m.canceledByUserRoundTrips.count) -} - -// CanceledSubRequests returns count of hedged sub-requests that were canceled by transport. -func (m *Stats) CanceledSubRequests() uint64 { - return atomic.LoadUint64(&m.canceledSubRequests.count) -} - -// StatsSnapshot is a snapshot of Stats. -type StatsSnapshot struct { - RequestedRoundTrips uint64 // count of requests that were requested by client - ActualRoundTrips uint64 // count of requests that were actually sent - FailedRoundTrips uint64 // count of requests that failed - CanceledByUserRoundTrips uint64 // count of requests that were canceled by user, using request context - CanceledSubRequests uint64 // count of hedged sub-requests that were canceled by transport -} - -// Snapshot of the stats. -func (m *Stats) Snapshot() StatsSnapshot { - return StatsSnapshot{ - RequestedRoundTrips: m.RequestedRoundTrips(), - ActualRoundTrips: m.ActualRoundTrips(), - FailedRoundTrips: m.FailedRoundTrips(), - CanceledByUserRoundTrips: m.CanceledByUserRoundTrips(), - CanceledSubRequests: m.CanceledSubRequests(), - } +// IsHedgedRequest reports when a request is hedged. +func IsHedgedRequest(r *http.Request) bool { + val := r.Context().Value(hedgedRequest{}) + return val != nil } var taskQueue = make(chan func()) @@ -255,7 +205,7 @@ var taskQueue = make(chan func()) func runInPool(task func()) { select { case taskQueue <- task: - // submited, everything is ok + // submitted, everything is ok default: go func() { @@ -281,7 +231,7 @@ func runInPool(task func()) { // MultiError is an error type to track multiple errors. This is used to // accumulate errors in cases and return them as a single "error". -// Insiper by https://github.com/hashicorp/go-multierror +// Inspired by https://github.com/hashicorp/go-multierror type MultiError struct { Errors []error ErrorFormatFn ErrorFormatFunc @@ -338,7 +288,7 @@ func getTimer(duration time.Duration) *time.Timer { func returnTimer(timer *time.Timer) { timer.Stop() select { - case _ = <-timer.C: + case <-timer.C: default: } timerPool.Put(timer) diff --git a/vendor/github.com/cristalhq/hedgedhttp/stats.go b/vendor/github.com/cristalhq/hedgedhttp/stats.go new file mode 100644 index 00000000000..fceeb234a22 --- /dev/null +++ b/vendor/github.com/cristalhq/hedgedhttp/stats.go @@ -0,0 +1,73 @@ +package hedgedhttp + +import "sync/atomic" + +// atomicCounter is a false sharing safe counter. +type atomicCounter struct { + count uint64 + _ [7]uint64 +} + +type cacheLine [64]byte + +// Stats object that can be queried to obtain certain metrics and get better observability. +type Stats struct { + _ cacheLine + requestedRoundTrips atomicCounter + actualRoundTrips atomicCounter + failedRoundTrips atomicCounter + canceledByUserRoundTrips atomicCounter + canceledSubRequests atomicCounter + _ cacheLine +} + +func (s *Stats) requestedRoundTripsInc() { atomic.AddUint64(&s.requestedRoundTrips.count, 1) } +func (s *Stats) actualRoundTripsInc() { atomic.AddUint64(&s.actualRoundTrips.count, 1) } +func (s *Stats) failedRoundTripsInc() { atomic.AddUint64(&s.failedRoundTrips.count, 1) } +func (s *Stats) canceledByUserRoundTripsInc() { atomic.AddUint64(&s.canceledByUserRoundTrips.count, 1) } +func (s *Stats) canceledSubRequestsInc() { atomic.AddUint64(&s.canceledSubRequests.count, 1) } + +// RequestedRoundTrips returns count of requests that were requested by client. +func (s *Stats) RequestedRoundTrips() uint64 { + return atomic.LoadUint64(&s.requestedRoundTrips.count) +} + +// ActualRoundTrips returns count of requests that were actually sent. +func (s *Stats) ActualRoundTrips() uint64 { + return atomic.LoadUint64(&s.actualRoundTrips.count) +} + +// FailedRoundTrips returns count of requests that failed. +func (s *Stats) FailedRoundTrips() uint64 { + return atomic.LoadUint64(&s.failedRoundTrips.count) +} + +// CanceledByUserRoundTrips returns count of requests that were canceled by user, using request context. +func (s *Stats) CanceledByUserRoundTrips() uint64 { + return atomic.LoadUint64(&s.canceledByUserRoundTrips.count) +} + +// CanceledSubRequests returns count of hedged sub-requests that were canceled by transport. +func (s *Stats) CanceledSubRequests() uint64 { + return atomic.LoadUint64(&s.canceledSubRequests.count) +} + +// StatsSnapshot is a snapshot of Stats. +type StatsSnapshot struct { + RequestedRoundTrips uint64 // count of requests that were requested by client + ActualRoundTrips uint64 // count of requests that were actually sent + FailedRoundTrips uint64 // count of requests that failed + CanceledByUserRoundTrips uint64 // count of requests that were canceled by user, using request context + CanceledSubRequests uint64 // count of hedged sub-requests that were canceled by transport +} + +// Snapshot of the stats. +func (s *Stats) Snapshot() StatsSnapshot { + return StatsSnapshot{ + RequestedRoundTrips: s.RequestedRoundTrips(), + ActualRoundTrips: s.ActualRoundTrips(), + FailedRoundTrips: s.FailedRoundTrips(), + CanceledByUserRoundTrips: s.CanceledByUserRoundTrips(), + CanceledSubRequests: s.CanceledSubRequests(), + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 58076633a76..c5a62b11e8a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -287,7 +287,7 @@ github.com/cortexproject/cortex/pkg/util/process github.com/cortexproject/cortex/pkg/util/push github.com/cortexproject/cortex/pkg/util/spanlogger github.com/cortexproject/cortex/pkg/util/validation -# github.com/cristalhq/hedgedhttp v0.6.0 +# github.com/cristalhq/hedgedhttp v0.7.0 ## explicit; go 1.16 github.com/cristalhq/hedgedhttp # github.com/davecgh/go-spew v1.1.1 From 080b0d9370b0474620ed27dde5cf89e2144c72cd Mon Sep 17 00:00:00 2001 From: Oleg Kovalov Date: Wed, 8 Dec 2021 10:19:51 +0100 Subject: [PATCH 2/3] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 311d2a27c01..29a65301bce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ * [BUGFIX] Fixed issue where compaction sometimes dropped spans. [#1130](https://github.com/grafana/tempo/pull/1130) (@joe-elliott) * [BUGFIX] Ensure that the admin client jsonnet has correct S3 bucket property. (@hedss) * [BUGFIX] Publish tenant index age correctly for tenant index writers. [#1146](https://github.com/grafana/tempo/pull/1146) (@joe-elliott) +* [CHANGE] Upgrade cristalhq/hedgedhttp from v0.6.0 to v0.7.0 [#1159](https://github.com/grafana/tempo/pull/1159) (@cristaloleg) ## v1.2.1 / 2021-11-15 * [BUGFIX] Fix defaults for MaxBytesPerTrace (ingester.max-bytes-per-trace) and MaxSearchBytesPerTrace (ingester.max-search-bytes-per-trace) [#1109](https://github.com/grafana/tempo/pull/1109) (@bitprocessor) From a24af560eb058bee81e9caab88144cb0a7e6b7c9 Mon Sep 17 00:00:00 2001 From: Oleg Kovalov Date: Wed, 8 Dec 2021 10:46:26 +0100 Subject: [PATCH 3/3] Rearrange CHANGELOG entries --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29a65301bce..95126c5ea48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * [CHANGE] Search: Add new per-tenant limit `max_bytes_per_tag_values_query` to limit the size of tag-values response. [#1068](https://github.com/grafana/tempo/pull/1068) (@annanay25) * [CHANGE] Reduce MaxSearchBytesPerTrace `ingester.max-search-bytes-per-trace` default to 5KB [#1129](https://github.com/grafana/tempo/pull/1129) @annanay25 * [CHANGE] **BREAKING CHANGE** The OTEL GRPC receiver's default port changed from 55680 to 4317. [#1142](https://github.com/grafana/tempo/pull/1142) (@tete17) +* [CHANGE] Upgrade cristalhq/hedgedhttp from v0.6.0 to v0.7.0 [#1159](https://github.com/grafana/tempo/pull/1159) (@cristaloleg) * [ENHANCEMENT] Expose `upto` parameter on hedged requests for each backend with `hedge_requests_up_to`. [#1085](https://github.com/grafana/tempo/pull/1085) (@joe-elliott) * [ENHANCEMENT] Search: drop use of TagCache, extract tags and tag values on-demand [#1068](https://github.com/grafana/tempo/pull/1068) (@kvrhdn) * [ENHANCEMENT] Jsonnet: add `$._config.namespace` to filter by namespace in cortex metrics [#1098](https://github.com/grafana/tempo/pull/1098) (@mapno) @@ -25,7 +26,6 @@ * [BUGFIX] Fixed issue where compaction sometimes dropped spans. [#1130](https://github.com/grafana/tempo/pull/1130) (@joe-elliott) * [BUGFIX] Ensure that the admin client jsonnet has correct S3 bucket property. (@hedss) * [BUGFIX] Publish tenant index age correctly for tenant index writers. [#1146](https://github.com/grafana/tempo/pull/1146) (@joe-elliott) -* [CHANGE] Upgrade cristalhq/hedgedhttp from v0.6.0 to v0.7.0 [#1159](https://github.com/grafana/tempo/pull/1159) (@cristaloleg) ## v1.2.1 / 2021-11-15 * [BUGFIX] Fix defaults for MaxBytesPerTrace (ingester.max-bytes-per-trace) and MaxSearchBytesPerTrace (ingester.max-search-bytes-per-trace) [#1109](https://github.com/grafana/tempo/pull/1109) (@bitprocessor)