Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix broken tests and add an extra one #2625

Merged
merged 3 commits into from
Aug 1, 2022
Merged

Fix broken tests and add an extra one #2625

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9999dbf
Fix TestRampingArrivalRateGlobalIters and its setupExecutor() helper
na-- Aug 1, 2022
deea53e
Fix thresholds test by moving it to cmd/
na-- Aug 1, 2022
ad33cbb
Add an integration test with failing thresholds
na-- Aug 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions cmd/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,3 +389,93 @@ func TestSubMetricThresholdNoData(t *testing.T) {
{ tag:xyz }........: 0 0/s
two..................: 42`)
}

func TestSetupTeardownThresholds(t *testing.T) {
t.Parallel()
tb := httpmultibin.NewHTTPMultiBin(t)

script := []byte(tb.Replacer.Replace(`
import http from "k6/http";
import { check } from "k6";
import { Counter } from "k6/metrics";

let statusCheck = { "status is 200": (r) => r.status === 200 }
let myCounter = new Counter("setup_teardown");

export let options = {
iterations: 5,
thresholds: {
"setup_teardown": ["count == 2"],
"iterations": ["count == 5"],
"http_reqs": ["count == 7"],
},
};

export function setup() {
check(http.get("HTTPBIN_IP_URL"), statusCheck) && myCounter.add(1);
};

export default function () {
check(http.get("HTTPBIN_IP_URL"), statusCheck);
};

export function teardown() {
check(http.get("HTTPBIN_IP_URL"), statusCheck) && myCounter.add(1);
};
`))

ts := newGlobalTestState(t)
require.NoError(t, afero.WriteFile(ts.fs, filepath.Join(ts.cwd, "test.js"), script, 0o644))
ts.args = []string{"k6", "run", "test.js"}

newRootCommand(ts.globalState).execute()

require.Len(t, ts.loggerHook.Drain(), 0)
stdOut := ts.stdOut.String()
require.Contains(t, stdOut, `✓ http_reqs......................: 7`)
require.Contains(t, stdOut, `✓ iterations.....................: 5`)
require.Contains(t, stdOut, `✓ setup_teardown.................: 2`)
}

func TestThresholdsFailed(t *testing.T) {
t.Parallel()
tb := httpmultibin.NewHTTPMultiBin(t)

script := []byte(tb.Replacer.Replace(`
export let options = {
scenarios: {
sc1: {
executor: 'per-vu-iterations',
vus: 1, iterations: 1,
},
sc2: {
executor: 'shared-iterations',
vus: 1, iterations: 2,
},
},
thresholds: {
'iterations': ['count == 3'],
'iterations{scenario:sc1}': ['count == 2'],
'iterations{scenario:sc2}': ['count == 1'],
'iterations{scenario:sc3}': ['count == 0'],
},
};

export default function () {};
`))

ts := newGlobalTestState(t)
require.NoError(t, afero.WriteFile(ts.fs, filepath.Join(ts.cwd, "test.js"), script, 0o644))
ts.args = []string{"k6", "run", "test.js"}
ts.expectedExitCode = 99 // ThresholdsHaveFailed

newRootCommand(ts.globalState).execute()

assert.True(t, testutils.LogContains(ts.loggerHook.Drain(), logrus.ErrorLevel, `some thresholds have failed`))
stdOut := ts.stdOut.String()
t.Logf(stdOut)
require.Contains(t, stdOut, ` ✓ iterations...........: 3`)
require.Contains(t, stdOut, ` ✗ { scenario:sc1 }...: 1`)
require.Contains(t, stdOut, ` ✗ { scenario:sc2 }...: 2`)
require.Contains(t, stdOut, ` ✓ { scenario:sc3 }...: 0 0/s`)
}
67 changes: 0 additions & 67 deletions core/engine_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -794,73 +794,6 @@ func TestRunTags(t *testing.T) {
}
}

func TestSetupTeardownThresholds(t *testing.T) {
t.Parallel()
tb := httpmultibin.NewHTTPMultiBin(t)

script := []byte(tb.Replacer.Replace(`
import http from "k6/http";
import { check } from "k6";
import { Counter } from "k6/metrics";

let statusCheck = { "status is 200": (r) => r.status === 200 }
let myCounter = new Counter("setup_teardown");

export let options = {
iterations: 5,
thresholds: {
"setup_teardown": ["count == 2"],
"iterations": ["count == 5"],
"http_reqs": ["count == 7"],
},
};

export function setup() {
check(http.get("HTTPBIN_IP_URL"), statusCheck) && myCounter.add(1);
};

export default function () {
check(http.get("HTTPBIN_IP_URL"), statusCheck);
};

export function teardown() {
check(http.get("HTTPBIN_IP_URL"), statusCheck) && myCounter.add(1);
};
`))

registry := metrics.NewRegistry()
builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
runner, err := js.New(
&lib.RuntimeState{
Logger: testutils.NewLogger(t),
BuiltinMetrics: builtinMetrics,
Registry: registry,
},
&loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: script},
nil,
)
require.NoError(t, err)

test := newTestEngine(t, nil, runner, nil, lib.Options{
SystemTags: &metrics.DefaultSystemTagSet,
SetupTimeout: types.NullDurationFrom(3 * time.Second),
TeardownTimeout: types.NullDurationFrom(3 * time.Second),
VUs: null.IntFrom(3),
})
defer test.wait()

errC := make(chan error)
go func() { errC <- test.run() }()

select {
case <-time.After(10 * time.Second):
t.Fatal("Test timed out")
case err := <-errC:
require.NoError(t, err)
require.False(t, test.engine.IsTainted())
}
}

func TestSetupException(t *testing.T) {
t.Parallel()

Expand Down
5 changes: 1 addition & 4 deletions lib/executor/common_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,7 @@ func setupExecutor(t testing.TB, config lib.ExecutorConfig, es *lib.ExecutionSta
}
es.SetInitVUFunc(initVUFunc)

et, err := lib.NewExecutionTuple(es.Options.ExecutionSegment, es.Options.ExecutionSegmentSequence)
require.NoError(t, err)

maxPlannedVUs := lib.GetMaxPlannedVUs(config.GetExecutionRequirements(et))
maxPlannedVUs := lib.GetMaxPlannedVUs(config.GetExecutionRequirements(es.ExecutionTuple))
initializeVUs(ctx, t, logEntry, es, maxPlannedVUs, initVUFunc)

executor, err := config.NewExecutor(es, logEntry)
Expand Down
4 changes: 2 additions & 2 deletions lib/executor/ramping_arrival_rate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -713,7 +713,7 @@ func TestRampingArrivalRateGlobalIters(t *testing.T) {
BaseConfig: BaseConfig{GracefulStop: types.NullDurationFrom(100 * time.Millisecond)},
TimeUnit: types.NullDurationFrom(950 * time.Millisecond),
StartRate: null.IntFrom(0),
PreAllocatedVUs: null.IntFrom(2),
PreAllocatedVUs: null.IntFrom(4),
MaxVUs: null.IntFrom(5),
Stages: []Stage{
{
Expand All @@ -733,7 +733,7 @@ func TestRampingArrivalRateGlobalIters(t *testing.T) {
}{
{"0,1/4,3/4,1", "0:1/4", []uint64{1, 6, 11, 16}},
{"0,1/4,3/4,1", "1/4:3/4", []uint64{0, 2, 4, 5, 7, 9, 10, 12, 14, 15, 17, 19, 20}},
{"0,1/4,3/4,1", "3/4:1", []uint64{3, 8, 13}},
{"0,1/4,3/4,1", "3/4:1", []uint64{3, 8, 13, 18}},
}

for _, tc := range testCases {
Expand Down