Skip to content

Commit

Permalink
e2e client support for cardinality (#6852)
Browse files Browse the repository at this point in the history 
* Forbid to import anything else from mimir

Signed-off-by: Dimitar Dimitrov <[email protected]>

* e2e client support for cardinality

The necessary change for this was to add a dedicated package for cardinality API types, otherwise we end up importing the prometheus world and that may come with some side effects (comment)

Signed-off-by: Dimitar Dimitrov <[email protected]>

---------

Signed-off-by: Dimitar Dimitrov <[email protected]>
  • Loading branch information
@dimitarvdimitrov
dimitarvdimitrov authored Dec 14, 2023
1 parent 5d0316a commit 7e7e2c3
Show file tree
Hide file tree
Showing 6 changed files with 153 additions and 161 deletions.
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,7 @@ lint: check-makefiles
faillint -paths "github.com/grafana/mimir/pkg/..." ./pkg/ruler/rulespb/...
faillint -paths "github.com/grafana/mimir/pkg/..." ./pkg/storage/sharding/...
faillint -paths "github.com/grafana/mimir/pkg/..." ./pkg/querier/engine/...
faillint -paths "github.com/grafana/mimir/pkg/..." ./pkg/querier/api/...
faillint -paths "github.com/grafana/mimir/pkg/..." ./pkg/util/globalerror

# Ensure all errors are report as APIError
Expand Down
36 changes: 30 additions & 6 deletions integration/e2emimir/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/grafana/mimir/pkg/distributor"
"github.com/grafana/mimir/pkg/frontend/querymiddleware"
"github.com/grafana/mimir/pkg/mimirpb"
"github.com/grafana/mimir/pkg/querier/api"
)

var ErrNotFound = errors.New("not found")
Expand Down Expand Up @@ -298,7 +299,7 @@ func (c *Client) LabelNames(start, end time.Time) ([]string, error) {
}

// LabelNamesAndValues returns distinct label values per label name.
func (c *Client) LabelNamesAndValues(selector string, limit int) (*http.Response, error) {
func (c *Client) LabelNamesAndValues(selector string, limit int) (*api.LabelNamesCardinalityResponse, error) {
body := make(url.Values)
if len(selector) > 0 {
body.Set("selector", selector)
Expand All @@ -318,12 +319,23 @@ func (c *Client) LabelNamesAndValues(selector string, limit int) (*http.Response
ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
defer cancel()

// Execute HTTP request
return c.httpClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode)
}
var lvalsResp api.LabelNamesCardinalityResponse
err = json.NewDecoder(resp.Body).Decode(&lvalsResp)
if err != nil {
return nil, fmt.Errorf("error decoding label values response: %w", err)
}
return &lvalsResp, nil
}

// LabelValuesCardinality returns all values and series total count for each label name.
func (c *Client) LabelValuesCardinality(labelNames []string, selector string, limit int) (*http.Response, error) {
func (c *Client) LabelValuesCardinality(labelNames []string, selector string, limit int) (*api.LabelValuesCardinalityResponse, error) {
body := make(url.Values)
if len(selector) > 0 {
body.Set("selector", selector)
Expand All @@ -346,8 +358,20 @@ func (c *Client) LabelValuesCardinality(labelNames []string, selector string, li
ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
defer cancel()

// Execute HTTP request
return c.httpClient.Do(req.WithContext(ctx))
resp, err := c.httpClient.Do(req.WithContext(ctx))
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode)
}

var lvalsResp api.LabelValuesCardinalityResponse
err = json.NewDecoder(resp.Body).Decode(&lvalsResp)
if err != nil {
return nil, fmt.Errorf("error decoding label values response: %w", err)
}
return &lvalsResp, nil
}

// GetPrometheusMetadata fetches the metadata from the Prometheus endpoint /api/v1/metadata.
Expand Down
99 changes: 30 additions & 69 deletions integration/querier_label_name_values_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
package integration

import (
"encoding/json"
"fmt"
"net/http"
"sort"
Expand All @@ -20,6 +19,7 @@ import (
"github.com/stretchr/testify/require"

"github.com/grafana/mimir/integration/e2emimir"
"github.com/grafana/mimir/pkg/querier/api"
)

// Define cardinality 'env' and 'job' label sets.
Expand All @@ -29,28 +29,17 @@ var cardinalityJobLabelValues = []string{"distributor", "ingester", "store-gatew
func TestQuerierLabelNamesAndValues(t *testing.T) {
const numSeriesToPush = 1000

// Define response types.
type labelNamesAndValuesCardinality struct {
LabelName string `json:"label_name"`
LabelValuesCount int `json:"label_values_count"`
}
type labelNamesAndValuesResponse struct {
LabelValuesCountTotal int `json:"label_values_count_total"`
LabelNamesCount int `json:"label_names_count"`
Cardinality []labelNamesAndValuesCardinality `json:"cardinality"`
}

// Test cases
tests := map[string]struct {
selector string
limit int
expectedResult labelNamesAndValuesResponse
expectedResult api.LabelNamesCardinalityResponse
}{
"obtain label names and values with default selector and limit": {
expectedResult: labelNamesAndValuesResponse{
expectedResult: api.LabelNamesCardinalityResponse{
LabelValuesCountTotal: 1008,
LabelNamesCount: 3,
Cardinality: []labelNamesAndValuesCardinality{
Cardinality: []*api.LabelNamesCardinalityItem{
{LabelName: labels.MetricName, LabelValuesCount: 1000},
{LabelName: "job", LabelValuesCount: 5},
{LabelName: "env", LabelValuesCount: 3},
Expand All @@ -59,10 +48,10 @@ func TestQuerierLabelNamesAndValues(t *testing.T) {
},
"apply request selector": {
selector: "{job=~'store-.*'}",
expectedResult: labelNamesAndValuesResponse{
expectedResult: api.LabelNamesCardinalityResponse{
LabelValuesCountTotal: 204,
LabelNamesCount: 3,
Cardinality: []labelNamesAndValuesCardinality{
Cardinality: []*api.LabelNamesCardinalityItem{
{LabelName: labels.MetricName, LabelValuesCount: 200},
{LabelName: "env", LabelValuesCount: 3},
{LabelName: "job", LabelValuesCount: 1},
Expand All @@ -71,21 +60,21 @@ func TestQuerierLabelNamesAndValues(t *testing.T) {
},
"limit cardinality response elements": {
limit: 1,
expectedResult: labelNamesAndValuesResponse{
expectedResult: api.LabelNamesCardinalityResponse{
LabelValuesCountTotal: 1008,
LabelNamesCount: 3,
Cardinality: []labelNamesAndValuesCardinality{
Cardinality: []*api.LabelNamesCardinalityItem{
{LabelName: labels.MetricName, LabelValuesCount: 1000},
},
},
},
"apply request selector and limit": {
selector: "{job=~'store-.*'}",
limit: 1,
expectedResult: labelNamesAndValuesResponse{
expectedResult: api.LabelNamesCardinalityResponse{
LabelValuesCountTotal: 204,
LabelNamesCount: 3,
Cardinality: []labelNamesAndValuesCardinality{
Cardinality: []*api.LabelNamesCardinalityItem{
{LabelName: labels.MetricName, LabelValuesCount: 200},
},
},
Expand Down Expand Up @@ -178,57 +167,34 @@ func TestQuerierLabelNamesAndValues(t *testing.T) {
})

// Fetch label names and values.
res, err := client.LabelNamesAndValues(tc.selector, tc.limit)
lbNamesAndValuesResp, err := client.LabelNamesAndValues(tc.selector, tc.limit)
require.NoError(t, err)
require.Equal(t, http.StatusOK, res.StatusCode)

// Test results.
var lbNamesAndValuesResp labelNamesAndValuesResponse
require.NoError(t, json.NewDecoder(res.Body).Decode(&lbNamesAndValuesResp))

require.Equal(t, tc.expectedResult, lbNamesAndValuesResp)
require.Equal(t, tc.expectedResult, *lbNamesAndValuesResp)
})
}
}

func TestQuerierLabelValuesCardinality(t *testing.T) {
const numSeriesToPush = 1000

// Define response types.
type labelValuesCardinality struct {
LabelValue string `json:"label_value"`
SeriesCount uint64 `json:"series_count"`
}

type labelNamesCardinality struct {
LabelName string `json:"label_name"`
LabelValuesCount uint64 `json:"label_values_count"`
SeriesCount uint64 `json:"series_count"`
Cardinality []labelValuesCardinality `json:"cardinality"`
}

type labelValuesCardinalityResponse struct {
SeriesCountTotal uint64 `json:"series_count_total"`
Labels []labelNamesCardinality `json:"labels"`
}

// Test cases
tests := map[string]struct {
labelNames []string
selector string
limit int
expectedResult labelValuesCardinalityResponse
expectedResult api.LabelValuesCardinalityResponse
}{
"obtain labels cardinality with default selector and limit": {
labelNames: []string{"env", "job"},
expectedResult: labelValuesCardinalityResponse{
expectedResult: api.LabelValuesCardinalityResponse{
SeriesCountTotal: numSeriesToPush,
Labels: []labelNamesCardinality{
Labels: []api.LabelNamesCardinality{
{
LabelName: "env",
LabelValuesCount: 3,
SeriesCount: 1000,
Cardinality: []labelValuesCardinality{
Cardinality: []api.LabelValuesCardinality{
{LabelValue: "staging", SeriesCount: 334},
{LabelValue: "dev", SeriesCount: 333},
{LabelValue: "prod", SeriesCount: 333},
Expand All @@ -238,7 +204,7 @@ func TestQuerierLabelValuesCardinality(t *testing.T) {
LabelName: "job",
LabelValuesCount: 5,
SeriesCount: 1000,
Cardinality: []labelValuesCardinality{
Cardinality: []api.LabelValuesCardinality{
{LabelValue: "compactor", SeriesCount: 200},
{LabelValue: "distributor", SeriesCount: 200},
{LabelValue: "ingester", SeriesCount: 200},
Expand All @@ -251,14 +217,14 @@ func TestQuerierLabelValuesCardinality(t *testing.T) {
},
"obtain env label cardinality with default selector": {
labelNames: []string{"env"},
expectedResult: labelValuesCardinalityResponse{
expectedResult: api.LabelValuesCardinalityResponse{
SeriesCountTotal: numSeriesToPush,
Labels: []labelNamesCardinality{
Labels: []api.LabelNamesCardinality{
{
LabelName: "env",
LabelValuesCount: 3,
SeriesCount: 1000,
Cardinality: []labelValuesCardinality{
Cardinality: []api.LabelValuesCardinality{
{LabelValue: "staging", SeriesCount: 334},
{LabelValue: "dev", SeriesCount: 333},
{LabelValue: "prod", SeriesCount: 333},
Expand All @@ -270,14 +236,14 @@ func TestQuerierLabelValuesCardinality(t *testing.T) {
"obtain labels cardinality applying selector": {
labelNames: []string{"env", "job"},
selector: "{job=~'store-.*'}",
expectedResult: labelValuesCardinalityResponse{
expectedResult: api.LabelValuesCardinalityResponse{
SeriesCountTotal: numSeriesToPush,
Labels: []labelNamesCardinality{
Labels: []api.LabelNamesCardinality{
{
LabelName: "env",
LabelValuesCount: 3,
SeriesCount: 200,
Cardinality: []labelValuesCardinality{
Cardinality: []api.LabelValuesCardinality{
{LabelValue: "dev", SeriesCount: 67},
{LabelValue: "staging", SeriesCount: 67},
{LabelValue: "prod", SeriesCount: 66},
Expand All @@ -287,7 +253,7 @@ func TestQuerierLabelValuesCardinality(t *testing.T) {
LabelName: "job",
LabelValuesCount: 1,
SeriesCount: 200,
Cardinality: []labelValuesCardinality{
Cardinality: []api.LabelValuesCardinality{
{LabelValue: "store-gateway", SeriesCount: 200},
},
},
Expand All @@ -297,14 +263,14 @@ func TestQuerierLabelValuesCardinality(t *testing.T) {
"obtain labels cardinality with default and custom limit": {
labelNames: []string{"env", "job"},
limit: 2,
expectedResult: labelValuesCardinalityResponse{
expectedResult: api.LabelValuesCardinalityResponse{
SeriesCountTotal: numSeriesToPush,
Labels: []labelNamesCardinality{
Labels: []api.LabelNamesCardinality{
{
LabelName: "env",
LabelValuesCount: 3,
SeriesCount: 1000,
Cardinality: []labelValuesCardinality{
Cardinality: []api.LabelValuesCardinality{
{LabelValue: "staging", SeriesCount: 334},
{LabelValue: "dev", SeriesCount: 333},
},
Expand All @@ -313,7 +279,7 @@ func TestQuerierLabelValuesCardinality(t *testing.T) {
LabelName: "job",
LabelValuesCount: 5,
SeriesCount: 1000,
Cardinality: []labelValuesCardinality{
Cardinality: []api.LabelValuesCardinality{
{LabelValue: "compactor", SeriesCount: 200},
{LabelValue: "distributor", SeriesCount: 200},
},
Expand Down Expand Up @@ -401,9 +367,8 @@ func TestQuerierLabelValuesCardinality(t *testing.T) {
})

// Fetch label values cardinality.
res, err := client.LabelValuesCardinality(tc.labelNames, tc.selector, tc.limit)
lbValuesCardinalityResp, err := client.LabelValuesCardinality(tc.labelNames, tc.selector, tc.limit)
require.NoError(t, err)
require.Equal(t, http.StatusOK, res.StatusCode)

// Ensure all ingesters have been invoked.
ingesters := []*e2emimir.MimirService{ingester1, ingester2, ingester3}
Expand All @@ -415,15 +380,11 @@ func TestQuerierLabelValuesCardinality(t *testing.T) {
e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "route", "/cortex.Ingester/LabelValuesCardinality"))))
}

// Test results.
var lbValuesCardinalityResp labelValuesCardinalityResponse
require.NoError(t, json.NewDecoder(res.Body).Decode(&lbValuesCardinalityResp))

// Make sure the resultant label names are sorted
sort.Slice(lbValuesCardinalityResp.Labels, func(l, r int) bool {
return lbValuesCardinalityResp.Labels[l].LabelName < lbValuesCardinalityResp.Labels[r].LabelName
})
require.Equal(t, tc.expectedResult, lbValuesCardinalityResp)
require.Equal(t, tc.expectedResult, *lbValuesCardinalityResp)
})
}
}
37 changes: 37 additions & 0 deletions pkg/querier/api/api.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// SPDX-License-Identifier: AGPL-3.0-only

package api

import "github.com/prometheus/prometheus/model/labels"

type LabelValuesCardinalityResponse struct {
SeriesCountTotal uint64 `json:"series_count_total"`
Labels []LabelNamesCardinality `json:"labels"`
}

type LabelNamesCardinality struct {
LabelName string `json:"label_name"`
LabelValuesCount uint64 `json:"label_values_count"`
SeriesCount uint64 `json:"series_count"`
Cardinality []LabelValuesCardinality `json:"cardinality"`
}

type LabelValuesCardinality struct {
LabelValue string `json:"label_value"`
SeriesCount uint64 `json:"series_count"`
}

type LabelNamesCardinalityResponse struct {
LabelValuesCountTotal int `json:"label_values_count_total"`
LabelNamesCount int `json:"label_names_count"`
Cardinality []*LabelNamesCardinalityItem `json:"cardinality"`
}

type LabelNamesCardinalityItem struct {
LabelName string `json:"label_name"`
LabelValuesCount int `json:"label_values_count"`
}

type ActiveSeriesResponse struct {
Data []labels.Labels `json:"data"`
}
Loading

0 comments on commit 7e7e2c3

Please sign in to comment.