Add per-user query metrics for series and bytes returned

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ## master / unreleased
 * [FEATURE] Ruler: Add new `-ruler.query-stats-enabled` which when enabled will report the `cortex_ruler_query_seconds_total` as a per-user metric that tracks the sum of the wall time of executing queries in the ruler in seconds. #4317
-* [FEATURE] Query Frontend: Add `cortex_query_series` and `cortex_query_bytes` per-user metrics to expose the number of series and bytes returned by queries. These metrics can be enabled with the `-frontend.query-stats-enabled` flag (or its respective YAML config option `query_stats_enabled`). #4343
+* [FEATURE] Query Frontend: Add `cortex_query_fetched_series_per_query` and `cortex_query_fetched_chunks_bytes_per_query` per-user summaries to expose the number of series and bytes fetched as part of queries. These metrics can be enabled with the `-frontend.query-stats-enabled` flag (or its respective YAML config option `query_stats_enabled`). #4343
 * [CHANGE] Update Go version to 1.16.6. #4362
 * [CHANGE] Querier / ruler: Change `-querier.max-fetched-chunks-per-query` configuration to limit to maximum number of chunks that can be fetched in a single query. The number of chunks fetched by ingesters AND long-term storare combined should not exceed the value configured on `-querier.max-fetched-chunks-per-query`. #4260
 * [CHANGE] Memberlist: the `memberlist_kv_store_value_bytes` has been removed due to values no longer being stored in-memory as encoded bytes. #4345

pkg/distributor/query.go

@@ -385,8 +385,8 @@ func (d *Distributor) queryIngesterStream(ctx context.Context, replicationSet ri
 		resp.Timeseries = append(resp.Timeseries, series)
 	}
 
-	reqStats.AddSeries(uint64(len(resp.Chunkseries) + len(resp.Timeseries)))
-	reqStats.AddBytes(uint64(resp.ChunksSize()))
+	reqStats.AddFetchedSeries(uint64(len(resp.Chunkseries) + len(resp.Timeseries)))
+	reqStats.AddFetchedChunkBytes(uint64(resp.ChunksSize()))
 
 	return resp, nil
 }

pkg/frontend/transport/handler.go

@@ -85,14 +85,14 @@ func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logge
 		// understand query responses is prohibitively expensive.
 
 		h.querySeries = promauto.With(reg).NewSummaryVec(prometheus.SummaryOpts{
-			Name:       "cortex_query_series",
-			Help:       "Number of series returned by successful queries.",
+			Name:       "cortex_query_fetched_series_per_query",
+			Help:       "Number of series fetched to execute a query.",
 			Objectives: map[float64]float64{},
 		}, []string{"user"})
 
 		h.queryBytes = promauto.With(reg).NewSummaryVec(prometheus.SummaryOpts{
-			Name:       "cortex_query_bytes",
-			Help:       "Number of bytes returned by successful queries.",
+			Name:       "cortex_query_fetched_chunks_bytes_per_query",
+			Help:       "Size of all chunks fetched to execute a query in bytes.",
 			Objectives: map[float64]float64{},
 		}, []string{"user"})
 
@@ -187,8 +187,8 @@ func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, quer
 	}
 	userID := tenant.JoinTenantIDs(tenantIDs)
 	wallTime := stats.LoadWallTime()
-	numSeries := stats.LoadSeries()
-	numBytes := stats.LoadBytes()
+	numSeries := stats.LoadFetchedSeries()
+	numBytes := stats.LoadFetchedChunkBytes()
 
 	// Track stats.
 	f.querySeconds.WithLabelValues(userID).Add(wallTime.Seconds())
@@ -204,8 +204,8 @@ func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, quer
 		"path", r.URL.Path,
 		"response_time", queryResponseTime,
 		"query_wall_time_seconds", wallTime.Seconds(),
-		"query_series", numSeries,
-		"query_bytes", numBytes,
+		"fetched_series_count", numSeries,
+		"fetched_chunk_bytes", numBytes,
 	}, formatQueryString(queryString)...)
 
 	level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...)

pkg/frontend/transport/handler_test.go

@@ -82,8 +82,8 @@ func TestHandler_ServeHTTP(t *testing.T) {
 			count, err := promtest.GatherAndCount(
 				reg,
 				"cortex_query_seconds_total",
-				"cortex_query_series",
-				"cortex_query_bytes",
+				"cortex_query_fetched_series_per_query",
+				"cortex_query_fetched_chunks_bytes_per_query",
 			)
 
 			assert.NoError(t, err)

pkg/querier/blocks_store_queryable.go

@@ -659,13 +659,13 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores(
 			numSeries := len(mySeries)
 			chunkBytes := countChunkBytes(mySeries...)
 
-			reqStats.AddSeries(uint64(numSeries))
-			reqStats.AddBytes(uint64(chunkBytes))
+			reqStats.AddFetchedSeries(uint64(numSeries))
+			reqStats.AddFetchedChunkBytes(uint64(chunkBytes))
 
 			level.Debug(spanLog).Log("msg", "received series from store-gateway",
 				"instance", c.RemoteAddress(),
-				"num series", numSeries,
-				"bytes series", chunkBytes,
+				"fetched series", numSeries,
+				"fetched chunk bytes", chunkBytes,
 				"requested blocks", strings.Join(convertULIDsToString(blockIDs), " "),
 				"queried blocks", strings.Join(convertULIDsToString(myQueriedBlocks), " "))
 

pkg/querier/stats/stats.go

@@ -54,36 +54,36 @@ func (s *Stats) LoadWallTime() time.Duration {
 	return time.Duration(atomic.LoadInt64((*int64)(&s.WallTime)))
 }
 
-func (s *Stats) AddSeries(series uint64) {
+func (s *Stats) AddFetchedSeries(series uint64) {
 	if s == nil {
 		return
 	}
 
-	atomic.AddUint64(&s.NumSeries, series)
+	atomic.AddUint64(&s.FetchedSeriesCount, series)
 }
 
-func (s *Stats) LoadSeries() uint64 {
+func (s *Stats) LoadFetchedSeries() uint64 {
 	if s == nil {
 		return 0
 	}
 
-	return atomic.LoadUint64(&s.NumSeries)
+	return atomic.LoadUint64(&s.FetchedSeriesCount)
 }
 
-func (s *Stats) AddBytes(bytes uint64) {
+func (s *Stats) AddFetchedChunkBytes(bytes uint64) {
 	if s == nil {
 		return
 	}
 
-	atomic.AddUint64(&s.NumBytes, bytes)
+	atomic.AddUint64(&s.FetchedChunkBytes, bytes)
 }
 
-func (s *Stats) LoadBytes() uint64 {
+func (s *Stats) LoadFetchedChunkBytes() uint64 {
 	if s == nil {
 		return 0
 	}
 
-	return atomic.LoadUint64(&s.NumBytes)
+	return atomic.LoadUint64(&s.FetchedChunkBytes)
 }
 
 // Merge the provide Stats into this one.
@@ -93,8 +93,8 @@ func (s *Stats) Merge(other *Stats) {
 	}
 
 	s.AddWallTime(other.LoadWallTime())
-	s.AddSeries(other.LoadSeries())
-	s.AddBytes(other.LoadBytes())
+	s.AddFetchedSeries(other.LoadFetchedSeries())
+	s.AddFetchedChunkBytes(other.LoadFetchedChunkBytes())
 }
 
 func ShouldTrackHTTPGRPCResponse(r *httpgrpc.HTTPResponse) bool {

pkg/querier/stats/stats.proto

@@ -13,8 +13,8 @@ option (gogoproto.unmarshaler_all) = true;
 message Stats {
   // The sum of all wall time spent in the querier to execute the query.
   google.protobuf.Duration wall_time = 1 [(gogoproto.stdduration) = true, (gogoproto.nullable) = false];
-  // The number of series included in the query response
-  uint64 num_series = 2;
-  // The number of bytes of the chunks included in the query response
-  uint64 num_bytes = 3;
+  // The number of series fetched for the query
+  uint64 fetched_series_count = 2;
+  // The number of bytes of the chunks fetched for the query
+  uint64 fetched_chunk_bytes = 3;
 }