Improve http connection reuse

CHANGELOG.md

@@ -11,6 +11,7 @@
 * [ENHANCEMENT] Use multiple goroutines to unmarshal responses in parallel in the query frontend. [#3713](https://github.com/grafana/tempo/pull/3713) (@joe-elliott)
 * [BUGFIX] Fix metrics queries when grouping by attributes that may not exist [#3734](https://github.com/grafana/tempo/pull/3734) (@mdisibio)
 * [BUGFIX] max_global_traces_per_user: take into account ingestion.tenant_shard_size when converting to local limit [#3618](https://github.com/grafana/tempo/pull/3618) (@kvrhdn)
+* [BUGFIX] Fix http connection reuse on GCP and AWS by reading io.EOF through the http body. [#3760](https://github.com/grafana/tempo/pull/3760) (@bmteller)
 
 ## v2.5.0
 

tempodb/backend/gcs/gcs.go

@@ -477,20 +477,17 @@ func (rw *readerWriter) readRange(ctx context.Context, name string, offset int64
 	}
 	defer r.Close()
 
-	totalBytes := 0
-	for {
-		byteCount, err := r.Read(buffer[totalBytes:])
-		if errors.Is(err, io.EOF) {
-			return nil
-		}
-		if err != nil {
-			return err
-		}
-		if byteCount == 0 {
-			return nil
-		}
-		totalBytes += byteCount
+	/* bytes read == len(buffer) if and only if err == nil */
+	_, err = io.ReadFull(r, buffer)
+
+	if err == nil {
+		/* read EOF so connection can be reused */
+		var dummy [1]byte
+		_, _ = r.Read(dummy[:])
+		return nil
 	}
+
+	return err
 }
 
 func createBucket(ctx context.Context, cfg *Config, hedge bool) (*storage.BucketHandle, error) {

tempodb/backend/s3/s3.go

@@ -561,20 +561,17 @@ func (rw *readerWriter) readRange(ctx context.Context, objName string, offset in
 	}
 	defer reader.Close()
 
-	totalBytes := 0
-	for {
-		byteCount, err := reader.Read(buffer[totalBytes:])
-		if errors.Is(err, io.EOF) {
-			return nil
-		}
-		if err != nil {
-			return fmt.Errorf("error in range read from s3 backend: %w", err)
-		}
-		if byteCount == 0 {
-			return nil
-		}
-		totalBytes += byteCount
+	/* bytes read == len(buffer) if and only if err == nil */
+	_, err = io.ReadFull(reader, buffer)
+
+	if err == nil {
+		/* read EOF so connection can be reused */
+		var dummy [1]byte
+		_, _ = reader.Read(dummy[:])
+		return nil
 	}
+
+	return fmt.Errorf("error in range read from s3 backend: %w", err)
 }
 
 func fetchCreds(cfg *Config) (*credentials.Credentials, error) {
@@ -626,6 +623,10 @@ func createCore(cfg *Config, hedge bool) (*minio.Core, error) {
 		return nil, fmt.Errorf("create minio.DefaultTransport: %w", err)
 	}
 
+	/* minio sets MaxIdleConns to 100 but we should also increase per host to 100 */
+	customTransport.MaxIdleConnsPerHost = 100
+	customTransport.MaxIdleConns = 100
+
 	tlsConfig, err := cfg.GetTLSConfig()
 	if err != nil {
 		return nil, fmt.Errorf("failed to create TLS config: %w", err)