Fix mimir-continuous-test when changing configured num-series (#1775)

Signed-off-by: Marco Pracucci <[email protected]>
grafana · Apr 29, 2022 · c18b5ab · c18b5ab
1 parent dd90026
commit c18b5ab
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 23 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -95,7 +95,7 @@
 ### Tools
 
 * [FEATURE] Added a `markblocks` tool that creates `no-compact` and `delete` marks for the blocks. #1551
-* [FEATURE] Added `mimir-continuous-test` tool to continuously run smoke tests on live Mimir clusters. #1535 #1540 #1653 #1603 #1630 #1691 #1675 #1676 #1692 #1706 #1709 #1777 #1778
+* [FEATURE] Added `mimir-continuous-test` tool to continuously run smoke tests on live Mimir clusters. #1535 #1540 #1653 #1603 #1630 #1691 #1675 #1676 #1692 #1706 #1709 #1775 #1777 #1778
 * [FEATURE] Added `mimir-rules-action` GitHub action, located at `operations/mimir-rules-action/`, used to lint, prepare, verify, diff, and sync rules to a Mimir cluster. #1723
 
 ## 2.0.0

diff --git a/pkg/continuoustest/write_read_series.go b/pkg/continuoustest/write_read_series.go
@@ -22,6 +22,14 @@ const (
 	metricName    = "mimir_continuous_test_sine_wave"
 )
 
+var (
+	// We use max_over_time() with a 1s range selector in order to fetch only the samples we previously
+	// wrote and ensure the PromQL lookback period doesn't influence query results. This help to avoid
+	// false positives when finding the last written sample, or when restarting the testing tool with
+	// a different number of configured series to write and read.
+	queryMetricSum = fmt.Sprintf("sum(max_over_time(%s[1s]))", metricName)
+)
+
 type WriteReadSeriesTestConfig struct {
 	NumSeries   int
 	MaxQueryAge time.Duration
@@ -202,13 +210,12 @@ func (t *WriteReadSeriesTest) runRangeQueryAndVerifyResult(ctx context.Context,
 	}
 
 	step := getQueryStep(start, end, writeInterval)
-	query := fmt.Sprintf("sum(%s)", metricName)
 
-	logger := log.With(t.logger, "query", query, "start", start.UnixMilli(), "end", end.UnixMilli(), "step", step, "results_cache", strconv.FormatBool(resultsCacheEnabled))
+	logger := log.With(t.logger, "query", queryMetricSum, "start", start.UnixMilli(), "end", end.UnixMilli(), "step", step, "results_cache", strconv.FormatBool(resultsCacheEnabled))
 	level.Debug(logger).Log("msg", "Running range query")
 
 	t.metrics.queriesTotal.Inc()
-	matrix, err := t.client.QueryRange(ctx, query, start, end, step, WithResultsCacheEnabled(resultsCacheEnabled))
+	matrix, err := t.client.QueryRange(ctx, queryMetricSum, start, end, step, WithResultsCacheEnabled(resultsCacheEnabled))
 	if err != nil {
 		t.metrics.queriesFailedTotal.Inc()
 		level.Warn(logger).Log("msg", "Failed to execute range query", "err", err)
@@ -232,13 +239,11 @@ func (t *WriteReadSeriesTest) runInstantQueryAndVerifyResult(ctx context.Context
 		return
 	}
 
-	query := fmt.Sprintf("sum(%s)", metricName)
-
-	logger := log.With(t.logger, "query", query, "ts", ts.UnixMilli(), "results_cache", strconv.FormatBool(resultsCacheEnabled))
+	logger := log.With(t.logger, "query", queryMetricSum, "ts", ts.UnixMilli(), "results_cache", strconv.FormatBool(resultsCacheEnabled))
 	level.Debug(logger).Log("msg", "Running instant query")
 
 	t.metrics.queriesTotal.Inc()
-	vector, err := t.client.Query(ctx, query, ts, WithResultsCacheEnabled(resultsCacheEnabled))
+	vector, err := t.client.Query(ctx, queryMetricSum, ts, WithResultsCacheEnabled(resultsCacheEnabled))
 	if err != nil {
 		t.metrics.queriesFailedTotal.Inc()
 		level.Warn(logger).Log("msg", "Failed to execute instant query", "err", err)
@@ -275,9 +280,6 @@ func (t *WriteReadSeriesTest) nextWriteTimestamp(now time.Time) time.Time {
 }
 
 func (t *WriteReadSeriesTest) findPreviouslyWrittenTimeRange(ctx context.Context, now time.Time) (from, to time.Time) {
-	// We use max_over_time() with a 1s range selector in order to fetch only the samples we previously
-	// wrote and ensure the PromQL lookback period doesn't influence query results.
-	query := fmt.Sprintf("sum(max_over_time(%s[1s]))", metricName)
 	end := alignTimestampToInterval(now, writeInterval)
 	step := writeInterval
 
@@ -290,10 +292,10 @@ func (t *WriteReadSeriesTest) findPreviouslyWrittenTimeRange(ctx context.Context
 			return
 		}
 
-		logger := log.With(t.logger, "query", query, "start", start, "end", end, "step", step)
+		logger := log.With(t.logger, "query", queryMetricSum, "start", start, "end", end, "step", step)
 		level.Debug(logger).Log("msg", "Executing query to find previously written samples")
 
-		matrix, err := t.client.QueryRange(ctx, query, start, end, step, WithResultsCacheEnabled(false))
+		matrix, err := t.client.QueryRange(ctx, queryMetricSum, start, end, step, WithResultsCacheEnabled(false))
 		if err != nil {
 			level.Warn(logger).Log("msg", "Failed to execute range query used to find previously written samples", "err", err)
 			return

diff --git a/pkg/continuoustest/write_read_series_test.go b/pkg/continuoustest/write_read_series_test.go
@@ -42,10 +42,10 @@ func TestWriteReadSeriesTest_Run(t *testing.T) {
 		assert.Equal(t, int64(1000), test.lastWrittenTimestamp.Unix())
 
 		client.AssertNumberOfCalls(t, "QueryRange", 4)
-		client.AssertCalled(t, "QueryRange", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything)
+		client.AssertCalled(t, "QueryRange", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything)
 
 		client.AssertNumberOfCalls(t, "Query", 4)
-		client.AssertCalled(t, "Query", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), mock.Anything)
+		client.AssertCalled(t, "Query", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), mock.Anything)
 
 		assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
 			# HELP mimir_continuous_test_writes_total Total number of attempted write requests.
@@ -81,10 +81,10 @@ func TestWriteReadSeriesTest_Run(t *testing.T) {
 		assert.Equal(t, int64(980), test.lastWrittenTimestamp.Unix())
 
 		client.AssertNumberOfCalls(t, "QueryRange", 4)
-		client.AssertCalled(t, "QueryRange", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(980, 0), time.Unix(980, 0), writeInterval, mock.Anything)
+		client.AssertCalled(t, "QueryRange", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(980, 0), time.Unix(980, 0), writeInterval, mock.Anything)
 
 		client.AssertNumberOfCalls(t, "Query", 4)
-		client.AssertCalled(t, "Query", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(980, 0), mock.Anything)
+		client.AssertCalled(t, "Query", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(980, 0), mock.Anything)
 
 		assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
 			# HELP mimir_continuous_test_writes_total Total number of attempted write requests.
@@ -123,10 +123,10 @@ func TestWriteReadSeriesTest_Run(t *testing.T) {
 		assert.Equal(t, int64(1000), test.lastWrittenTimestamp.Unix())
 
 		client.AssertNumberOfCalls(t, "QueryRange", 4)
-		client.AssertCalled(t, "QueryRange", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(960, 0), time.Unix(1000, 0), writeInterval, mock.Anything)
+		client.AssertCalled(t, "QueryRange", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(960, 0), time.Unix(1000, 0), writeInterval, mock.Anything)
 
 		client.AssertNumberOfCalls(t, "Query", 4)
-		client.AssertCalled(t, "Query", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), mock.Anything)
+		client.AssertCalled(t, "Query", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), mock.Anything)
 
 		assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
 			# HELP mimir_continuous_test_writes_total Total number of attempted write requests.
@@ -259,10 +259,10 @@ func TestWriteReadSeriesTest_Run(t *testing.T) {
 		assert.Equal(t, int64(1000), test.lastWrittenTimestamp.Unix())
 
 		client.AssertNumberOfCalls(t, "QueryRange", 4)
-		client.AssertCalled(t, "QueryRange", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything)
+		client.AssertCalled(t, "QueryRange", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything)
 
 		client.AssertNumberOfCalls(t, "Query", 4)
-		client.AssertCalled(t, "Query", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), mock.Anything)
+		client.AssertCalled(t, "Query", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), mock.Anything)
 
 		assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
 			# HELP mimir_continuous_test_writes_total Total number of attempted write requests.
@@ -313,10 +313,10 @@ func TestWriteReadSeriesTest_Run(t *testing.T) {
 		assert.Equal(t, int64(1000), test.lastWrittenTimestamp.Unix())
 
 		client.AssertNumberOfCalls(t, "QueryRange", 4)
-		client.AssertCalled(t, "QueryRange", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything)
+		client.AssertCalled(t, "QueryRange", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything)
 
 		client.AssertNumberOfCalls(t, "Query", 4)
-		client.AssertCalled(t, "Query", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), mock.Anything)
+		client.AssertCalled(t, "Query", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), mock.Anything)
 
 		assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
 			# HELP mimir_continuous_test_writes_total Total number of attempted write requests.