Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

query-tee: include human-readable timestamps in comparison failure messages #9665

Merged
merged 2 commits into from
Oct 18, 2024
Merged

query-tee: include human-readable timestamps in comparison failure messages #9665

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b6a19fd
query-tee: include human-readable timestamps in comparison failure me…
charleskorn Oct 18, 2024
76ec72a
Add changelog entry
charleskorn Oct 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@

* [CHANGE] Don't compare responses for cancelled requests. #9640
* [FEATURE] Added `-proxy.compare-skip-samples-before` to skip samples before the given time when comparing responses. The time can be in RFC3339 format (or) RFC3339 without the timezone and seconds (or) date only. #9515
* [ENHANCEMENT] Added human-readable timestamps to comparison failure messages. #9665

### Documentation

Expand Down
8 changes: 6 additions & 2 deletions tools/querytee/response_comparator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -567,18 +567,22 @@ func compareSamplePair(expected, actual model.SamplePair, queryEvaluationTime ti
}

if expected.Timestamp != actual.Timestamp {
return fmt.Errorf("expected timestamp %v but got %v", expected.Timestamp, actual.Timestamp)
return fmt.Errorf("expected timestamp %v (%v) but got %v (%v)", expected.Timestamp, formatTimestamp(expected.Timestamp), actual.Timestamp, formatTimestamp(actual.Timestamp))
}
if opts.SkipRecentSamples > 0 && queryEvaluationTime.Sub(expected.Timestamp.Time()) < opts.SkipRecentSamples {
return nil
}
if !compareSampleValue(float64(expected.Value), float64(actual.Value), opts) {
return fmt.Errorf("expected value %s for timestamp %v but got %s", expected.Value, expected.Timestamp, actual.Value)
return fmt.Errorf("expected value %s for timestamp %v (%v) but got %s", expected.Value, expected.Timestamp, formatTimestamp(expected.Timestamp), actual.Value)
}

return nil
}

func formatTimestamp(t model.Time) string {
return t.Time().UTC().Format(time.RFC3339Nano)
}

func compareSampleValue(first, second float64, opts SampleComparisonOptions) bool {
if (math.IsNaN(first) && math.IsNaN(second)) ||
(math.IsInf(first, 1) && math.IsInf(second, 1)) ||
Expand Down
24 changes: 12 additions & 12 deletions tools/querytee/response_comparator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ Actual result for series:
actual: json.RawMessage(`[
{"metric":{"foo":"bar"},"values":[[1,"1"],[3,"2"]]}
]`),
err: `float sample pair does not match for metric {foo="bar"}: expected timestamp 2 but got 3
err: `float sample pair does not match for metric {foo="bar"}: expected timestamp 2 (1970-01-01T00:00:02Z) but got 3 (1970-01-01T00:00:03Z)
Expected result for series:
{foo="bar"} =>
1 @[1]
Expand All @@ -103,7 +103,7 @@ Actual result for series:
actual: json.RawMessage(`[
{"metric":{"foo":"bar"},"values":[[1,"1"],[2,"3"]]}
]`),
err: `float sample pair does not match for metric {foo="bar"}: expected value 2 for timestamp 2 but got 3
err: `float sample pair does not match for metric {foo="bar"}: expected value 2 for timestamp 2 (1970-01-01T00:00:02Z) but got 3
Expected result for series:
{foo="bar"} =>
1 @[1]
Expand Down Expand Up @@ -133,7 +133,7 @@ Actual result for series:
{"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]},
{"metric":{"oops":"bar"},"values":[[1,"1"],[2,"3"]]}
]`),
err: `float sample pair does not match for metric {oops="bar"}: expected value 2 for timestamp 2 but got 3
err: `float sample pair does not match for metric {oops="bar"}: expected value 2 for timestamp 2 (1970-01-01T00:00:02Z) but got 3
Expected result for series:
{oops="bar"} =>
1 @[1]
Expand Down Expand Up @@ -759,7 +759,7 @@ func TestCompareVector(t *testing.T) {
actual: json.RawMessage(`[
{"metric":{"foo":"bar"},"value":[2,"1"]}
]`),
err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected timestamp 1 but got 2`),
err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected timestamp 1 (1970-01-01T00:00:01Z) but got 2 (1970-01-01T00:00:02Z)`),
},
{
name: "difference in float sample value",
Expand All @@ -769,7 +769,7 @@ func TestCompareVector(t *testing.T) {
actual: json.RawMessage(`[
{"metric":{"foo":"bar"},"value":[1,"2"]}
]`),
err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected value 1 for timestamp 1 but got 2`),
err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected value 1 for timestamp 1 (1970-01-01T00:00:01Z) but got 2`),
},
{
name: "correct float samples",
Expand Down Expand Up @@ -1100,13 +1100,13 @@ func TestCompareScalar(t *testing.T) {
name: "difference in timestamp",
expected: json.RawMessage(`[1,"1"]`),
actual: json.RawMessage(`[2,"1"]`),
err: errors.New("expected timestamp 1 but got 2"),
err: errors.New("expected timestamp 1 (1970-01-01T00:00:01Z) but got 2 (1970-01-01T00:00:02Z)"),
},
{
name: "difference in value",
expected: json.RawMessage(`[1,"1"]`),
actual: json.RawMessage(`[1,"2"]`),
err: errors.New("expected value 1 for timestamp 1 but got 2"),
err: errors.New("expected value 1 for timestamp 1 (1970-01-01T00:00:01Z) but got 2"),
},
{
name: "correct values",
Expand Down Expand Up @@ -1365,7 +1365,7 @@ func TestCompareSamplesResponse(t *testing.T) {
"status": "success",
"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"773054.789"]}]}
}`),
err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected value 773054.5916666666 for timestamp 1 but got 773054.789`),
err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected value 773054.5916666666 for timestamp 1 (1970-01-01T00:00:01Z) but got 773054.789`),
},
{
name: "should fail if large values are significantly different, over the tolerance without using relative error",
Expand All @@ -1378,7 +1378,7 @@ func TestCompareSamplesResponse(t *testing.T) {
"status": "success",
"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"4.923488536785281e+41"]}]}
}`),
err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected value 492348853678528200000000000000000000000000 for timestamp 1 but got 492348853678528100000000000000000000000000`),
err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected value 492348853678528200000000000000000000000000 for timestamp 1 (1970-01-01T00:00:01Z) but got 492348853678528100000000000000000000000000`),
},
{
name: "should not fail if large values are significantly different, over the tolerance using relative error",
Expand Down Expand Up @@ -2203,7 +2203,7 @@ func TestCompareSamplesResponse(t *testing.T) {
}`),
skipSamplesBefore: 95 * 1000,
// 9 @[90] is not compared. foo=bar2 does not fail.
err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected timestamp 97 but got 100
err: errors.New(`float sample pair does not match for metric {foo="bar"}: expected timestamp 97 (1970-01-01T00:01:37Z) but got 100 (1970-01-01T00:01:40Z)
Expected result for series:
{foo="bar"} =>
7 @[97]
Expand Down Expand Up @@ -2386,7 +2386,7 @@ Count: 2.000000, Sum: 3.000000, Buckets: [[0,2):2] @[100]`),
"data": {"resultType":"scalar","result":[100,"1"]}
}`),
skipSamplesBefore: 95 * 1000,
err: errors.New(`expected timestamp 90 but got 100`),
err: errors.New(`expected timestamp 90 (1970-01-01T00:01:30Z) but got 100 (1970-01-01T00:01:40Z)`),
},
{
name: "actual is skippable but not the expected, causing an error",
Expand All @@ -2399,7 +2399,7 @@ Count: 2.000000, Sum: 3.000000, Buckets: [[0,2):2] @[100]`),
"data": {"resultType":"scalar","result":[90,"1"]}
}`),
skipSamplesBefore: 95 * 1000,
err: errors.New(`expected timestamp 100 but got 90`),
err: errors.New(`expected timestamp 100 (1970-01-01T00:01:40Z) but got 90 (1970-01-01T00:01:30Z)`),
},
{
name: "both expected and actual are skippable",
Expand Down
Loading