feat(logcli): add gzip compression option (#14598)

Passing `--compress` to logcli will enable (or more accurately not disable) compression on the `http.Transport`, allowing Loki to return gzip-compressed payloads.

This improves overall execution time and reduces data transfer by 10-15x.

Signed-off-by: Jason Tackaberry <[email protected]>

Author: jtackaberry

cmd/logcli/main.go

@@ -483,6 +483,7 @@ func newQueryClient(app *kingpin.Application) client.Client {
 	app.Flag("max-backoff", "Maximum backoff time between retries. Can also be set using LOKI_CLIENT_MAX_BACKOFF env var.").Default("0").Envar("LOKI_CLIENT_MAX_BACKOFF").IntVar(&client.BackoffConfig.MaxBackoff)
 	app.Flag("auth-header", "The authorization header used. Can also be set using LOKI_AUTH_HEADER env var.").Default("Authorization").Envar("LOKI_AUTH_HEADER").StringVar(&client.AuthHeader)
 	app.Flag("proxy-url", "The http or https proxy to use when making requests. Can also be set using LOKI_HTTP_PROXY_URL env var.").Default("").Envar("LOKI_HTTP_PROXY_URL").StringVar(&client.ProxyURL)
+	app.Flag("compress", "Request that Loki compress returned data in transit. Can also be set using LOKI_HTTP_COMPRESSION env var.").Default("false").Envar("LOKI_HTTP_COMPRESSION").BoolVar(&client.Compression)
 
 	return client
 }

docs/sources/query/logcli.md

@@ -371,6 +371,7 @@ Flags:
       --auth-header="Authorization"
                                 The authorization header used. Can also be set using LOKI_AUTH_HEADER env var.
       --proxy-url=""            The http or https proxy to use when making requests. Can also be set using LOKI_HTTP_PROXY_URL env var.
+      --compress                Request that Loki compress returned data in transit. Can also be set using LOKI_HTTP_COMPRESSION env var.
       --limit=30                Limit on number of entries to print. Setting it to 0 will fetch all entries.
       --since=1h                Lookback window.
       --from=FROM               Start looking for logs at this absolute time (inclusive)
@@ -465,6 +466,7 @@ Flags:
       --auth-header="Authorization"
                               The authorization header used. Can also be set using LOKI_AUTH_HEADER env var.
       --proxy-url=""          The http or https proxy to use when making requests. Can also be set using LOKI_HTTP_PROXY_URL env var.
+      --compress              Request that Loki compress returned data in transit. Can also be set using LOKI_HTTP_COMPRESSION env var.
       --limit=30              Limit on number of entries to print. Setting it to 0 will fetch all entries.
       --now=NOW               Time at which to execute the instant query.
       --forward               Scan forwards through logs.
@@ -525,6 +527,7 @@ Flags:
       --auth-header="Authorization"
                               The authorization header used. Can also be set using LOKI_AUTH_HEADER env var.
       --proxy-url=""          The http or https proxy to use when making requests. Can also be set using LOKI_HTTP_PROXY_URL env var.
+      --compress              Request that Loki compress returned data in transit. Can also be set using LOKI_HTTP_COMPRESSION env var.
       --since=1h              Lookback window.
       --from=FROM             Start looking for labels at this absolute time (inclusive)
       --to=TO                 Stop looking for labels at this absolute time (exclusive)
@@ -581,6 +584,7 @@ Flags:
       --auth-header="Authorization"
                               The authorization header used. Can also be set using LOKI_AUTH_HEADER env var.
       --proxy-url=""          The http or https proxy to use when making requests. Can also be set using LOKI_HTTP_PROXY_URL env var.
+      --compress              Request that Loki compress returned data in transit. Can also be set using LOKI_HTTP_COMPRESSION env var.
       --since=1h              Lookback window.
       --from=FROM             Start looking for logs at this absolute time (inclusive)
       --to=TO                 Stop looking for logs at this absolute time (exclusive)
@@ -633,6 +637,7 @@ Flags:
       --auth-header="Authorization"
                               The authorization header used. Can also be set using LOKI_AUTH_HEADER env var.
       --proxy-url=""          The http or https proxy to use when making requests. Can also be set using LOKI_HTTP_PROXY_URL env var.
+      --compress              Request that Loki compress returned data in transit. Can also be set using LOKI_HTTP_COMPRESSION env var.
 ```
 
 ### `stats` command reference
@@ -694,6 +699,7 @@ Flags:
       --auth-header="Authorization"
                               The authorization header used. Can also be set using LOKI_AUTH_HEADER env var.
       --proxy-url=""          The http or https proxy to use when making requests. Can also be set using LOKI_HTTP_PROXY_URL env var.
+      --compress              Request that Loki compress returned data in transit. Can also be set using LOKI_HTTP_COMPRESSION env var.
       --since=1h              Lookback window.
       --from=FROM             Start looking for logs at this absolute time (inclusive)
       --to=TO                 Stop looking for logs at this absolute time (exclusive)
@@ -761,6 +767,7 @@ Flags:
       --auth-header="Authorization"
                               The authorization header used. Can also be set using LOKI_AUTH_HEADER env var.
       --proxy-url=""          The http or https proxy to use when making requests. Can also be set using LOKI_HTTP_PROXY_URL env var.
+      --compress              Request that Loki compress returned data in transit. Can also be set using LOKI_HTTP_COMPRESSION env var.
       --since=1h              Lookback window.
       --from=FROM             Start looking for logs at this absolute time (inclusive)
       --to=TO                 Stop looking for logs at this absolute time (exclusive)
@@ -833,6 +840,7 @@ Flags:
       --auth-header="Authorization"
                               The authorization header used. Can also be set using LOKI_AUTH_HEADER env var.
       --proxy-url=""          The http or https proxy to use when making requests. Can also be set using LOKI_HTTP_PROXY_URL env var.
+      --compress              Request that Loki compress returned data in transit. Can also be set using LOKI_HTTP_COMPRESSION env var.
       --since=1h              Lookback window.
       --from=FROM             Start looking for logs at this absolute time (inclusive)
       --to=TO                 Stop looking for logs at this absolute time (exclusive)

pkg/logcli/client/client.go

@@ -88,6 +88,7 @@ type DefaultClient struct {
 	AuthHeader      string
 	ProxyURL        string
 	BackoffConfig   BackoffConfig
+	Compression     bool
 }
 
 // Query uses the /api/v1/query endpoint to execute an instant query
@@ -320,6 +321,16 @@ func (c *DefaultClient) doRequest(path, query string, quiet bool, out interface{
 	if c.Tripperware != nil {
 		client.Transport = c.Tripperware(client.Transport)
 	}
+	if c.Compression {
+		// NewClientFromConfig() above returns an http.Client that uses a transport which
+		// has compression explicitly disabled. Here we re-enable it. If the caller
+		// defines a custom Tripperware that isn't an http.Transport then this won't work,
+		// but in that case they control the transport anyway and can configure
+		// compression that way.
+		if transport, ok := client.Transport.(*http.Transport); ok {
+			transport.DisableCompression = false
+		}
+	}
 
 	var resp *http.Response