From fcc3ce70a3d03a9601a6e350d12359ed3a5c2ec8 Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Tue, 14 Jun 2022 09:44:02 +0300 Subject: [PATCH 1/8] test_runner: add Subtest to tap protocol output --- lib/internal/test_runner/tap_stream.js | 4 ++++ lib/internal/test_runner/test.js | 1 + 2 files changed, 5 insertions(+) diff --git a/lib/internal/test_runner/tap_stream.js b/lib/internal/test_runner/tap_stream.js index a6bfbb3367cd79..176676f5a74d35 100644 --- a/lib/internal/test_runner/tap_stream.js +++ b/lib/internal/test_runner/tap_stream.js @@ -71,6 +71,10 @@ class TapStream extends Readable { return `TODO${reason ? ` ${tapEscape(reason)}` : ''}`; } + subTest(indent, name) { + this.#tryPush(`${indent}# Subtest: ${tapEscape(name)}\n`); + } + details(indent, duration, error) { let details = `${indent} ---\n`; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index cac379b32582a4..e7180ff1df811b 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -280,6 +280,7 @@ class Test extends AsyncResource { } start() { + this.reporter.subTest(this.indent, this.name); // If there is enough available concurrency to run the test now, then do // it. Otherwise, return a Promise to the caller and mark the test as // pending for later execution. From 9c7d73ad58665f14b1496078499fdb6a93950668 Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Tue, 14 Jun 2022 12:02:32 +0300 Subject: [PATCH 2/8] add tests --- test/fixtures/test-runner/index.test.tap | 13 ++++++ test/fixtures/test-runner/nested.test.cli.tap | 13 ++++++ test/fixtures/test-runner/nested.test.js | 12 ++++++ test/fixtures/test-runner/nested.test.tap | 41 +++++++++++++++++++ .../test-runner/{ => search-files}/index.js | 0 .../random.cjs => search-files/index.test.js} | 0 .../node_modules/test-nm.js | 0 .../{ => search-files}/random.test.mjs | 0 .../{ => search-files}/subdir/subdir_test.js | 0 .../test-runner/search-files/test/random.cjs | 4 ++ test/parallel/test-runner-cli.js | 2 +- test/parallel/test-runner-tap-protocol.js | 35 ++++++++++++++++ 12 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/test-runner/index.test.tap create mode 100644 test/fixtures/test-runner/nested.test.cli.tap create mode 100644 test/fixtures/test-runner/nested.test.js create mode 100644 test/fixtures/test-runner/nested.test.tap rename test/fixtures/test-runner/{ => search-files}/index.js (100%) rename test/fixtures/test-runner/{test/random.cjs => search-files/index.test.js} (100%) rename test/fixtures/test-runner/{ => search-files}/node_modules/test-nm.js (100%) rename test/fixtures/test-runner/{ => search-files}/random.test.mjs (100%) rename test/fixtures/test-runner/{ => search-files}/subdir/subdir_test.js (100%) create mode 100644 test/fixtures/test-runner/search-files/test/random.cjs create mode 100644 test/parallel/test-runner-tap-protocol.js diff --git a/test/fixtures/test-runner/index.test.tap b/test/fixtures/test-runner/index.test.tap new file mode 100644 index 00000000000000..e1a0f6a14ea80e --- /dev/null +++ b/test/fixtures/test-runner/index.test.tap @@ -0,0 +1,13 @@ +TAP version 13 +# Subtest: this should pass +ok 1 - this should pass + --- + duration_ms: + ... +1..1 +# tests 1 +# pass 1 +# fail 0 +# skipped 0 +# todo 0 +# duration_ms: diff --git a/test/fixtures/test-runner/nested.test.cli.tap b/test/fixtures/test-runner/nested.test.cli.tap new file mode 100644 index 00000000000000..408706ec4748f4 --- /dev/null +++ b/test/fixtures/test-runner/nested.test.cli.tap @@ -0,0 +1,13 @@ +TAP version 13 +# Subtest: $FILE_PATH +ok 1 - $FILE_PATH + --- + duration_ms: + ... +1..1 +# tests 1 +# pass 1 +# fail 0 +# skipped 0 +# todo 0 +# duration_ms: diff --git a/test/fixtures/test-runner/nested.test.js b/test/fixtures/test-runner/nested.test.js new file mode 100644 index 00000000000000..05edea12ffc896 --- /dev/null +++ b/test/fixtures/test-runner/nested.test.js @@ -0,0 +1,12 @@ +'use strict'; +const test = require('node:test'); + +test('level 1', async (t) => { + await t.test('level 2.1', async (t) => { + await t.test('level 3', () => {}); + }); + await t.test('level 2.2', async (t) => { + await t.test('level 3', () => {}); + }); + await t.test('level 2.4', () => {}); +}); diff --git a/test/fixtures/test-runner/nested.test.tap b/test/fixtures/test-runner/nested.test.tap new file mode 100644 index 00000000000000..8da20ee1733b61 --- /dev/null +++ b/test/fixtures/test-runner/nested.test.tap @@ -0,0 +1,41 @@ +TAP version 13 +# Subtest: level 1 + # Subtest: level 2.1 + # Subtest: level 3 + ok 1 - level 3 + --- + duration_ms: + ... + 1..1 + ok 1 - level 2.1 + --- + duration_ms: + ... + # Subtest: level 2.2 + # Subtest: level 3 + ok 1 - level 3 + --- + duration_ms: + ... + 1..1 + ok 2 - level 2.2 + --- + duration_ms: + ... + # Subtest: level 2.4 + ok 3 - level 2.4 + --- + duration_ms: + ... + 1..3 +ok 1 - level 1 + --- + duration_ms: + ... +1..1 +# tests 1 +# pass 1 +# fail 0 +# skipped 0 +# todo 0 +# duration_ms: diff --git a/test/fixtures/test-runner/index.js b/test/fixtures/test-runner/search-files/index.js similarity index 100% rename from test/fixtures/test-runner/index.js rename to test/fixtures/test-runner/search-files/index.js diff --git a/test/fixtures/test-runner/test/random.cjs b/test/fixtures/test-runner/search-files/index.test.js similarity index 100% rename from test/fixtures/test-runner/test/random.cjs rename to test/fixtures/test-runner/search-files/index.test.js diff --git a/test/fixtures/test-runner/node_modules/test-nm.js b/test/fixtures/test-runner/search-files/node_modules/test-nm.js similarity index 100% rename from test/fixtures/test-runner/node_modules/test-nm.js rename to test/fixtures/test-runner/search-files/node_modules/test-nm.js diff --git a/test/fixtures/test-runner/random.test.mjs b/test/fixtures/test-runner/search-files/random.test.mjs similarity index 100% rename from test/fixtures/test-runner/random.test.mjs rename to test/fixtures/test-runner/search-files/random.test.mjs diff --git a/test/fixtures/test-runner/subdir/subdir_test.js b/test/fixtures/test-runner/search-files/subdir/subdir_test.js similarity index 100% rename from test/fixtures/test-runner/subdir/subdir_test.js rename to test/fixtures/test-runner/search-files/subdir/subdir_test.js diff --git a/test/fixtures/test-runner/search-files/test/random.cjs b/test/fixtures/test-runner/search-files/test/random.cjs new file mode 100644 index 00000000000000..2a722c504b9fa5 --- /dev/null +++ b/test/fixtures/test-runner/search-files/test/random.cjs @@ -0,0 +1,4 @@ +'use strict'; +const test = require('node:test'); + +test('this should pass'); diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js index 7bd95372a2d68b..00dfa0155fcc98 100644 --- a/test/parallel/test-runner-cli.js +++ b/test/parallel/test-runner-cli.js @@ -4,7 +4,7 @@ const assert = require('assert'); const { spawnSync } = require('child_process'); const { join } = require('path'); const fixtures = require('../common/fixtures'); -const testFixtures = fixtures.path('test-runner'); +const testFixtures = fixtures.path('test-runner/search-files'); { // File not found. diff --git a/test/parallel/test-runner-tap-protocol.js b/test/parallel/test-runner-tap-protocol.js new file mode 100644 index 00000000000000..236b1bb001c284 --- /dev/null +++ b/test/parallel/test-runner-tap-protocol.js @@ -0,0 +1,35 @@ +'use strict'; +require('../common'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const fixtures = require('../common/fixtures'); + +function removeDuration(str) { + return str.replace(/duration_ms:? .+/g, 'duration_ms: '); +} + +{ + // sanity + const child = spawnSync(process.execPath, [fixtures.path('test-runner/index.test.js')]); + const stdout = removeDuration(child.stdout.toString()); + const expected = fixtures.readSync('test-runner/index.test.tap').toString(); + + assert.strictEqual(stdout, expected); +} +{ + // nested tests + const child = spawnSync(process.execPath, [fixtures.path('test-runner/nested.test.js')]); + const stdout = removeDuration(child.stdout.toString()); + const expected = fixtures.readSync('test-runner/nested.test.tap').toString(); + + assert.strictEqual(stdout, expected); +} +{ + // using test runner cli + const file = fixtures.path('test-runner/nested.test.js'); + const child = spawnSync(process.execPath, ['--test', fixtures.path('test-runner/nested.test.js')]); + const stdout = removeDuration(child.stdout.toString()); + const expected = fixtures.readSync('test-runner/nested.test.cli.tap').toString().replace(/\$FILE_PATH/g, file); + + assert.strictEqual(stdout, expected); +} \ No newline at end of file From 2d0153bc02c9bea0c70a120b3055ed4dd2f4d7b3 Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Tue, 14 Jun 2022 12:16:45 +0300 Subject: [PATCH 3/8] Revert "add tests" This reverts commit fcd4f7205f5bb8c6c486fb0b9898ceb74acc78bf. --- .../test-runner/{search-files => }/index.js | 0 test/fixtures/test-runner/index.test.tap | 13 ------ test/fixtures/test-runner/nested.test.cli.tap | 13 ------ test/fixtures/test-runner/nested.test.js | 12 ------ test/fixtures/test-runner/nested.test.tap | 41 ------------------- .../node_modules/test-nm.js | 0 .../{search-files => }/random.test.mjs | 0 .../test-runner/search-files/index.test.js | 4 -- .../{search-files => }/subdir/subdir_test.js | 0 .../{search-files => }/test/random.cjs | 0 test/parallel/test-runner-cli.js | 2 +- test/parallel/test-runner-tap-protocol.js | 35 ---------------- 12 files changed, 1 insertion(+), 119 deletions(-) rename test/fixtures/test-runner/{search-files => }/index.js (100%) delete mode 100644 test/fixtures/test-runner/index.test.tap delete mode 100644 test/fixtures/test-runner/nested.test.cli.tap delete mode 100644 test/fixtures/test-runner/nested.test.js delete mode 100644 test/fixtures/test-runner/nested.test.tap rename test/fixtures/test-runner/{search-files => }/node_modules/test-nm.js (100%) rename test/fixtures/test-runner/{search-files => }/random.test.mjs (100%) delete mode 100644 test/fixtures/test-runner/search-files/index.test.js rename test/fixtures/test-runner/{search-files => }/subdir/subdir_test.js (100%) rename test/fixtures/test-runner/{search-files => }/test/random.cjs (100%) delete mode 100644 test/parallel/test-runner-tap-protocol.js diff --git a/test/fixtures/test-runner/search-files/index.js b/test/fixtures/test-runner/index.js similarity index 100% rename from test/fixtures/test-runner/search-files/index.js rename to test/fixtures/test-runner/index.js diff --git a/test/fixtures/test-runner/index.test.tap b/test/fixtures/test-runner/index.test.tap deleted file mode 100644 index e1a0f6a14ea80e..00000000000000 --- a/test/fixtures/test-runner/index.test.tap +++ /dev/null @@ -1,13 +0,0 @@ -TAP version 13 -# Subtest: this should pass -ok 1 - this should pass - --- - duration_ms: - ... -1..1 -# tests 1 -# pass 1 -# fail 0 -# skipped 0 -# todo 0 -# duration_ms: diff --git a/test/fixtures/test-runner/nested.test.cli.tap b/test/fixtures/test-runner/nested.test.cli.tap deleted file mode 100644 index 408706ec4748f4..00000000000000 --- a/test/fixtures/test-runner/nested.test.cli.tap +++ /dev/null @@ -1,13 +0,0 @@ -TAP version 13 -# Subtest: $FILE_PATH -ok 1 - $FILE_PATH - --- - duration_ms: - ... -1..1 -# tests 1 -# pass 1 -# fail 0 -# skipped 0 -# todo 0 -# duration_ms: diff --git a/test/fixtures/test-runner/nested.test.js b/test/fixtures/test-runner/nested.test.js deleted file mode 100644 index 05edea12ffc896..00000000000000 --- a/test/fixtures/test-runner/nested.test.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict'; -const test = require('node:test'); - -test('level 1', async (t) => { - await t.test('level 2.1', async (t) => { - await t.test('level 3', () => {}); - }); - await t.test('level 2.2', async (t) => { - await t.test('level 3', () => {}); - }); - await t.test('level 2.4', () => {}); -}); diff --git a/test/fixtures/test-runner/nested.test.tap b/test/fixtures/test-runner/nested.test.tap deleted file mode 100644 index 8da20ee1733b61..00000000000000 --- a/test/fixtures/test-runner/nested.test.tap +++ /dev/null @@ -1,41 +0,0 @@ -TAP version 13 -# Subtest: level 1 - # Subtest: level 2.1 - # Subtest: level 3 - ok 1 - level 3 - --- - duration_ms: - ... - 1..1 - ok 1 - level 2.1 - --- - duration_ms: - ... - # Subtest: level 2.2 - # Subtest: level 3 - ok 1 - level 3 - --- - duration_ms: - ... - 1..1 - ok 2 - level 2.2 - --- - duration_ms: - ... - # Subtest: level 2.4 - ok 3 - level 2.4 - --- - duration_ms: - ... - 1..3 -ok 1 - level 1 - --- - duration_ms: - ... -1..1 -# tests 1 -# pass 1 -# fail 0 -# skipped 0 -# todo 0 -# duration_ms: diff --git a/test/fixtures/test-runner/search-files/node_modules/test-nm.js b/test/fixtures/test-runner/node_modules/test-nm.js similarity index 100% rename from test/fixtures/test-runner/search-files/node_modules/test-nm.js rename to test/fixtures/test-runner/node_modules/test-nm.js diff --git a/test/fixtures/test-runner/search-files/random.test.mjs b/test/fixtures/test-runner/random.test.mjs similarity index 100% rename from test/fixtures/test-runner/search-files/random.test.mjs rename to test/fixtures/test-runner/random.test.mjs diff --git a/test/fixtures/test-runner/search-files/index.test.js b/test/fixtures/test-runner/search-files/index.test.js deleted file mode 100644 index 2a722c504b9fa5..00000000000000 --- a/test/fixtures/test-runner/search-files/index.test.js +++ /dev/null @@ -1,4 +0,0 @@ -'use strict'; -const test = require('node:test'); - -test('this should pass'); diff --git a/test/fixtures/test-runner/search-files/subdir/subdir_test.js b/test/fixtures/test-runner/subdir/subdir_test.js similarity index 100% rename from test/fixtures/test-runner/search-files/subdir/subdir_test.js rename to test/fixtures/test-runner/subdir/subdir_test.js diff --git a/test/fixtures/test-runner/search-files/test/random.cjs b/test/fixtures/test-runner/test/random.cjs similarity index 100% rename from test/fixtures/test-runner/search-files/test/random.cjs rename to test/fixtures/test-runner/test/random.cjs diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js index 00dfa0155fcc98..7bd95372a2d68b 100644 --- a/test/parallel/test-runner-cli.js +++ b/test/parallel/test-runner-cli.js @@ -4,7 +4,7 @@ const assert = require('assert'); const { spawnSync } = require('child_process'); const { join } = require('path'); const fixtures = require('../common/fixtures'); -const testFixtures = fixtures.path('test-runner/search-files'); +const testFixtures = fixtures.path('test-runner'); { // File not found. diff --git a/test/parallel/test-runner-tap-protocol.js b/test/parallel/test-runner-tap-protocol.js deleted file mode 100644 index 236b1bb001c284..00000000000000 --- a/test/parallel/test-runner-tap-protocol.js +++ /dev/null @@ -1,35 +0,0 @@ -'use strict'; -require('../common'); -const assert = require('assert'); -const { spawnSync } = require('child_process'); -const fixtures = require('../common/fixtures'); - -function removeDuration(str) { - return str.replace(/duration_ms:? .+/g, 'duration_ms: '); -} - -{ - // sanity - const child = spawnSync(process.execPath, [fixtures.path('test-runner/index.test.js')]); - const stdout = removeDuration(child.stdout.toString()); - const expected = fixtures.readSync('test-runner/index.test.tap').toString(); - - assert.strictEqual(stdout, expected); -} -{ - // nested tests - const child = spawnSync(process.execPath, [fixtures.path('test-runner/nested.test.js')]); - const stdout = removeDuration(child.stdout.toString()); - const expected = fixtures.readSync('test-runner/nested.test.tap').toString(); - - assert.strictEqual(stdout, expected); -} -{ - // using test runner cli - const file = fixtures.path('test-runner/nested.test.js'); - const child = spawnSync(process.execPath, ['--test', fixtures.path('test-runner/nested.test.js')]); - const stdout = removeDuration(child.stdout.toString()); - const expected = fixtures.readSync('test-runner/nested.test.cli.tap').toString().replace(/\$FILE_PATH/g, file); - - assert.strictEqual(stdout, expected); -} \ No newline at end of file From 1682cbb3e14b5647fddd7990425df142604c2689 Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Tue, 14 Jun 2022 13:09:07 +0300 Subject: [PATCH 4/8] fix tests for real --- lib/internal/test_runner/test.js | 2 +- test/message/test_runner_no_refs.out | 2 + test/message/test_runner_only_tests.out | 23 ++++++ test/message/test_runner_output.out | 71 +++++++++++++++++++ .../test_runner_unresolved_promise.out | 3 + 5 files changed, 100 insertions(+), 1 deletion(-) diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index e7180ff1df811b..1643acdaa013b1 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -280,7 +280,6 @@ class Test extends AsyncResource { } start() { - this.reporter.subTest(this.indent, this.name); // If there is enough available concurrency to run the test now, then do // it. Otherwise, return a Promise to the caller and mark the test as // pending for later execution. @@ -296,6 +295,7 @@ class Test extends AsyncResource { } async run() { + this.reporter.subTest(this.indent, this.name); this.parent.activeSubtests++; this.startTime = hrtime(); diff --git a/test/message/test_runner_no_refs.out b/test/message/test_runner_no_refs.out index c5407e3bd330c0..078ab7c390e488 100644 --- a/test/message/test_runner_no_refs.out +++ b/test/message/test_runner_no_refs.out @@ -1,4 +1,6 @@ TAP version 13 +# Subtest: does not keep event loop alive + # Subtest: +does not keep event loop alive not ok 1 - +does not keep event loop alive --- duration_ms: * diff --git a/test/message/test_runner_only_tests.out b/test/message/test_runner_only_tests.out index 76e6337bb8953d..b65958b010f8ad 100644 --- a/test/message/test_runner_only_tests.out +++ b/test/message/test_runner_only_tests.out @@ -1,76 +1,97 @@ TAP version 13 +# Subtest: only = undefined ok 1 - only = undefined # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = undefined, skip = string ok 2 - only = undefined, skip = string # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = undefined, skip = true ok 3 - only = undefined, skip = true # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = undefined, skip = false ok 4 - only = undefined, skip = false # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = false ok 5 - only = false # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = false, skip = string ok 6 - only = false, skip = string # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = false, skip = true ok 7 - only = false, skip = true # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = false, skip = false ok 8 - only = false, skip = false # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = true, skip = string ok 9 - only = true, skip = string # SKIP skip message --- duration_ms: * ... +# Subtest: only = true, skip = true ok 10 - only = true, skip = true # SKIP --- duration_ms: * ... +# Subtest: only = true, with subtests + # Subtest: running subtest 1 ok 1 - running subtest 1 --- duration_ms: * ... + # Subtest: running subtest 2 ok 2 - running subtest 2 --- duration_ms: * ... + # Subtest: skipped subtest 1 ok 3 - skipped subtest 1 # SKIP 'only' option not set --- duration_ms: * ... + # Subtest: skipped subtest 2 ok 4 - skipped subtest 2 # SKIP 'only' option not set --- duration_ms: * ... + # Subtest: running subtest 3 ok 5 - running subtest 3 --- duration_ms: * ... + # Subtest: running subtest 4 + # Subtest: running sub-subtest 1 ok 1 - running sub-subtest 1 --- duration_ms: * ... + # Subtest: running sub-subtest 2 ok 2 - running sub-subtest 2 --- duration_ms: * ... + # Subtest: skipped sub-subtest 1 ok 3 - skipped sub-subtest 1 # SKIP 'only' option not set --- duration_ms: * ... + # Subtest: skipped sub-subtest 2 ok 4 - skipped sub-subtest 2 # SKIP 'only' option not set --- duration_ms: * @@ -80,10 +101,12 @@ ok 10 - only = true, skip = true # SKIP --- duration_ms: * ... + # Subtest: skipped subtest 3 ok 7 - skipped subtest 3 # SKIP 'only' option not set --- duration_ms: * ... + # Subtest: skipped subtest 4 ok 8 - skipped subtest 4 # SKIP --- duration_ms: * diff --git a/test/message/test_runner_output.out b/test/message/test_runner_output.out index 6c2fa7eb14cc72..e637ac9fb73cf8 100644 --- a/test/message/test_runner_output.out +++ b/test/message/test_runner_output.out @@ -1,12 +1,15 @@ TAP version 13 +# Subtest: sync pass todo ok 1 - sync pass todo # TODO --- duration_ms: * ... +# Subtest: sync pass todo with message ok 2 - sync pass todo with message # TODO this is a passing todo --- duration_ms: * ... +# Subtest: sync fail todo not ok 3 - sync fail todo # TODO --- duration_ms: * @@ -23,6 +26,7 @@ not ok 3 - sync fail todo # TODO * * ... +# Subtest: sync fail todo with message not ok 4 - sync fail todo with message # TODO this is a failing todo --- duration_ms: * @@ -41,19 +45,23 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo * * ... +# Subtest: sync skip pass ok 5 - sync skip pass # SKIP --- duration_ms: * ... +# Subtest: sync skip pass with message ok 6 - sync skip pass with message # SKIP this is skipped --- duration_ms: * ... +# Subtest: sync pass ok 7 - sync pass --- duration_ms: * ... # this test should pass +# Subtest: sync throw fail not ok 8 - sync throw fail --- duration_ms: * @@ -70,14 +78,17 @@ not ok 8 - sync throw fail * * ... +# Subtest: async skip pass ok 9 - async skip pass # SKIP --- duration_ms: * ... +# Subtest: async pass ok 10 - async pass --- duration_ms: * ... +# Subtest: async throw fail not ok 11 - async throw fail --- duration_ms: * @@ -94,6 +105,7 @@ not ok 11 - async throw fail * * ... +# Subtest: async skip fail not ok 12 - async skip fail # SKIP --- duration_ms: * @@ -110,6 +122,7 @@ not ok 12 - async skip fail # SKIP * * ... +# Subtest: async assertion fail not ok 13 - async assertion fail --- duration_ms: * @@ -130,10 +143,12 @@ not ok 13 - async assertion fail * * ... +# Subtest: resolve pass ok 14 - resolve pass --- duration_ms: * ... +# Subtest: reject fail not ok 15 - reject fail --- duration_ms: * @@ -150,26 +165,33 @@ not ok 15 - reject fail * * ... +# Subtest: unhandled rejection - passes but warns ok 16 - unhandled rejection - passes but warns --- duration_ms: * ... +# Subtest: async unhandled rejection - passes but warns ok 17 - async unhandled rejection - passes but warns --- duration_ms: * ... +# Subtest: immediate throw - passes but warns ok 18 - immediate throw - passes but warns --- duration_ms: * ... +# Subtest: immediate reject - passes but warns ok 19 - immediate reject - passes but warns --- duration_ms: * ... +# Subtest: immediate resolve pass ok 20 - immediate resolve pass --- duration_ms: * ... +# Subtest: subtest sync throw fail + # Subtest: +sync throw fail not ok 1 - +sync throw fail --- duration_ms: * @@ -197,6 +219,7 @@ not ok 21 - subtest sync throw fail error: '1 subtest failed' code: 'ERR_TEST_FAILURE' ... +# Subtest: sync throw non-error fail not ok 22 - sync throw non-error fail --- duration_ms: * @@ -204,6 +227,11 @@ not ok 22 - sync throw non-error fail error: 'Symbol(thrown symbol from sync throw non-error fail)' code: 'ERR_TEST_FAILURE' ... +# Subtest: level 0a + # Subtest: level 1a + # Subtest: level 1b + # Subtest: level 1c + # Subtest: level 1d ok 1 - level 1a --- duration_ms: * @@ -225,6 +253,10 @@ ok 23 - level 0a --- duration_ms: * ... +# Subtest: top level + # Subtest: +long running + # Subtest: +short running + # Subtest: ++short running not ok 1 - +long running --- duration_ms: * @@ -249,18 +281,22 @@ not ok 24 - top level error: '1 subtest failed' code: 'ERR_TEST_FAILURE' ... +# Subtest: invalid subtest - pass but subtest fails ok 25 - invalid subtest - pass but subtest fails --- duration_ms: * ... +# Subtest: sync skip option ok 26 - sync skip option # SKIP --- duration_ms: * ... +# Subtest: sync skip option with message ok 27 - sync skip option with message # SKIP this is skipped --- duration_ms: * ... +# Subtest: sync skip option is false fail not ok 28 - sync skip option is false fail --- duration_ms: * @@ -276,59 +312,73 @@ not ok 28 - sync skip option is false fail * * ... +# Subtest: ok 29 - --- duration_ms: * ... +# Subtest: functionOnly ok 30 - functionOnly --- duration_ms: * ... +# Subtest: ok 31 - --- duration_ms: * ... +# Subtest: test with only a name provided ok 32 - test with only a name provided --- duration_ms: * ... +# Subtest: ok 33 - --- duration_ms: * ... +# Subtest: ok 34 - # SKIP --- duration_ms: * ... +# Subtest: test with a name and options provided ok 35 - test with a name and options provided # SKIP --- duration_ms: * ... +# Subtest: functionAndOptions ok 36 - functionAndOptions # SKIP --- duration_ms: * ... +# Subtest: escaped description \\ \# \\\#\\ ok 37 - escaped description \\ \# \\\#\\ --- duration_ms: * ... +# Subtest: escaped skip message ok 38 - escaped skip message # SKIP \#skip --- duration_ms: * ... +# Subtest: escaped todo message ok 39 - escaped todo message # TODO \#todo --- duration_ms: * ... +# Subtest: escaped diagnostic ok 40 - escaped diagnostic --- duration_ms: * ... # \#diagnostic +# Subtest: callback pass ok 41 - callback pass --- duration_ms: * ... +# Subtest: callback fail not ok 42 - callback fail --- duration_ms: * @@ -339,18 +389,22 @@ not ok 42 - callback fail * * ... +# Subtest: sync t is this in test ok 43 - sync t is this in test --- duration_ms: * ... +# Subtest: async t is this in test ok 44 - async t is this in test --- duration_ms: * ... +# Subtest: callback t is this in test ok 45 - callback t is this in test --- duration_ms: * ... +# Subtest: callback also returns a Promise not ok 46 - callback also returns a Promise --- duration_ms: * @@ -358,6 +412,7 @@ not ok 46 - callback also returns a Promise error: 'passed a callback but also returned a Promise' code: 'ERR_TEST_FAILURE' ... +# Subtest: callback throw not ok 47 - callback throw --- duration_ms: * @@ -373,6 +428,7 @@ not ok 47 - callback throw * * ... +# Subtest: callback called twice not ok 48 - callback called twice --- duration_ms: * @@ -383,10 +439,12 @@ not ok 48 - callback called twice * * ... +# Subtest: callback called twice in different ticks ok 49 - callback called twice in different ticks --- duration_ms: * ... +# Subtest: callback called twice in future tick not ok 50 - callback called twice in future tick --- duration_ms: * @@ -396,6 +454,19 @@ not ok 50 - callback called twice in future tick stack: |- * ... +# Subtest: callback async throw +# Subtest: callback async throw after done +# Subtest: only is set but not in only mode + # Subtest: running subtest 1 + # Subtest: running subtest 2 + # Subtest: running subtest 3 + # Subtest: running subtest 4 +# Subtest: custom inspect symbol fail +# Subtest: custom inspect symbol that throws fail +# Subtest: subtest sync throw fails + # Subtest: sync throw fails at first + # Subtest: sync throw fails at second +# Subtest: invalid subtest fail not ok 51 - callback async throw --- duration_ms: * diff --git a/test/message/test_runner_unresolved_promise.out b/test/message/test_runner_unresolved_promise.out index 98f52966c33bcb..58b3a4ed89a616 100644 --- a/test/message/test_runner_unresolved_promise.out +++ b/test/message/test_runner_unresolved_promise.out @@ -1,8 +1,10 @@ TAP version 13 +# Subtest: pass ok 1 - pass --- duration_ms: * ... +# Subtest: never resolving promise not ok 2 - never resolving promise --- duration_ms: * @@ -12,6 +14,7 @@ not ok 2 - never resolving promise stack: |- * ... +# Subtest: fail not ok 3 - fail --- duration_ms: * From 99db83624a81247765e530b8577b28dca37aeedd Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Tue, 14 Jun 2022 20:21:29 +0300 Subject: [PATCH 5/8] CR --- lib/internal/test_runner/test.js | 9 +++++++- test/message/test_runner_output.out | 35 +++++++++++++++-------------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 1643acdaa013b1..27cea6e1789c31 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -162,6 +162,7 @@ class Test extends AsyncResource { processReadySubtestRange(canSend) { const start = this.waitingOn; const end = start + this.readySubtests.size; + let shouldReportSubtest = this.parent !== null; for (let i = start; i < end; i++) { const subtest = this.readySubtests.get(i); @@ -183,6 +184,11 @@ class Test extends AsyncResource { return; } + if (shouldReportSubtest) { + this.reporter.subTest(this.indent, this.name); + shouldReportSubtest = false; + } + // Report the subtest's results and remove it from the ready map. subtest.finalize(); this.readySubtests.delete(i); @@ -295,7 +301,6 @@ class Test extends AsyncResource { } async run() { - this.reporter.subTest(this.indent, this.name); this.parent.activeSubtests++; this.startTime = hrtime(); @@ -417,6 +422,8 @@ class Test extends AsyncResource { // Output this test's results and update the parent's waiting counter. if (this.subtests.length > 0) { this.reporter.plan(this.subtests[0].indent, this.subtests.length); + } else { + this.reporter.subTest(this.indent, this.name); } this.report(); diff --git a/test/message/test_runner_output.out b/test/message/test_runner_output.out index e637ac9fb73cf8..8c95eda5c84067 100644 --- a/test/message/test_runner_output.out +++ b/test/message/test_runner_output.out @@ -229,21 +229,22 @@ not ok 22 - sync throw non-error fail ... # Subtest: level 0a # Subtest: level 1a - # Subtest: level 1b - # Subtest: level 1c - # Subtest: level 1d ok 1 - level 1a --- duration_ms: * ... + # Subtest: level 1b ok 2 - level 1b --- duration_ms: * ... +# Subtest: level 0a + # Subtest: level 1c ok 3 - level 1c --- duration_ms: * ... + # Subtest: level 1d ok 4 - level 1d --- duration_ms: * @@ -255,8 +256,6 @@ ok 23 - level 0a ... # Subtest: top level # Subtest: +long running - # Subtest: +short running - # Subtest: ++short running not ok 1 - +long running --- duration_ms: * @@ -264,6 +263,8 @@ ok 23 - level 0a error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' ... + # Subtest: +short running + # Subtest: ++short running ok 1 - ++short running --- duration_ms: * @@ -455,18 +456,6 @@ not ok 50 - callback called twice in future tick * ... # Subtest: callback async throw -# Subtest: callback async throw after done -# Subtest: only is set but not in only mode - # Subtest: running subtest 1 - # Subtest: running subtest 2 - # Subtest: running subtest 3 - # Subtest: running subtest 4 -# Subtest: custom inspect symbol fail -# Subtest: custom inspect symbol that throws fail -# Subtest: subtest sync throw fails - # Subtest: sync throw fails at first - # Subtest: sync throw fails at second -# Subtest: invalid subtest fail not ok 51 - callback async throw --- duration_ms: * @@ -476,22 +465,28 @@ not ok 51 - callback async throw stack: |- * ... +# Subtest: callback async throw after done ok 52 - callback async throw after done --- duration_ms: * ... +# Subtest: only is set but not in only mode + # Subtest: running subtest 1 ok 1 - running subtest 1 --- duration_ms: * ... + # Subtest: running subtest 2 ok 2 - running subtest 2 --- duration_ms: * ... + # Subtest: running subtest 3 ok 3 - running subtest 3 --- duration_ms: * ... + # Subtest: running subtest 4 ok 4 - running subtest 4 --- duration_ms: * @@ -501,6 +496,7 @@ ok 53 - only is set but not in only mode --- duration_ms: * ... +# Subtest: custom inspect symbol fail not ok 54 - custom inspect symbol fail --- duration_ms: * @@ -508,6 +504,7 @@ not ok 54 - custom inspect symbol fail error: 'customized' code: 'ERR_TEST_FAILURE' ... +# Subtest: custom inspect symbol that throws fail not ok 55 - custom inspect symbol that throws fail --- duration_ms: * @@ -519,6 +516,8 @@ not ok 55 - custom inspect symbol that throws fail } code: 'ERR_TEST_FAILURE' ... +# Subtest: subtest sync throw fails + # Subtest: sync throw fails at first not ok 1 - sync throw fails at first --- duration_ms: * @@ -537,6 +536,7 @@ not ok 55 - custom inspect symbol that throws fail * * ... + # Subtest: sync throw fails at second not ok 2 - sync throw fails at second --- duration_ms: * @@ -563,6 +563,7 @@ not ok 56 - subtest sync throw fails error: '2 subtests failed' code: 'ERR_TEST_FAILURE' ... +# Subtest: invalid subtest fail not ok 57 - invalid subtest fail --- duration_ms: * From ea7383237ea2831903868bb7ced8eab7ccfe1c9c Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Tue, 14 Jun 2022 20:26:31 +0300 Subject: [PATCH 6/8] fix --- lib/internal/test_runner/test.js | 7 ++++--- test/message/test_runner_output.out | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 27cea6e1789c31..39db7f668608a2 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -93,6 +93,7 @@ class Test extends AsyncResource { this.reporter = new TapStream(); this.runOnlySubtests = this.only; this.testNumber = 0; + this.shouldReportSubtest = false; } else { const indent = parent.parent === null ? parent.indent : parent.indent + parent.indentString; @@ -104,6 +105,7 @@ class Test extends AsyncResource { this.reporter = parent.reporter; this.runOnlySubtests = !this.only; this.testNumber = parent.subtests.length + 1; + this.shouldReportSubtest = true; } if (isUint32(concurrency) && concurrency !== 0) { @@ -162,7 +164,6 @@ class Test extends AsyncResource { processReadySubtestRange(canSend) { const start = this.waitingOn; const end = start + this.readySubtests.size; - let shouldReportSubtest = this.parent !== null; for (let i = start; i < end; i++) { const subtest = this.readySubtests.get(i); @@ -184,9 +185,9 @@ class Test extends AsyncResource { return; } - if (shouldReportSubtest) { + if (this.shouldReportSubtest) { this.reporter.subTest(this.indent, this.name); - shouldReportSubtest = false; + this.shouldReportSubtest = false; } // Report the subtest's results and remove it from the ready map. diff --git a/test/message/test_runner_output.out b/test/message/test_runner_output.out index 8c95eda5c84067..2f6e2502749068 100644 --- a/test/message/test_runner_output.out +++ b/test/message/test_runner_output.out @@ -238,7 +238,6 @@ not ok 22 - sync throw non-error fail --- duration_ms: * ... -# Subtest: level 0a # Subtest: level 1c ok 3 - level 1c --- From 1366dffadb135d0df66d4dbc8fb1e06753f44f6b Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Wed, 15 Jun 2022 10:23:49 +0300 Subject: [PATCH 7/8] fix --- lib/internal/test_runner/tap_stream.js | 2 +- lib/internal/test_runner/test.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/internal/test_runner/tap_stream.js b/lib/internal/test_runner/tap_stream.js index 176676f5a74d35..ca0c101aa9260c 100644 --- a/lib/internal/test_runner/tap_stream.js +++ b/lib/internal/test_runner/tap_stream.js @@ -71,7 +71,7 @@ class TapStream extends Readable { return `TODO${reason ? ` ${tapEscape(reason)}` : ''}`; } - subTest(indent, name) { + subtest(indent, name) { this.#tryPush(`${indent}# Subtest: ${tapEscape(name)}\n`); } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 39db7f668608a2..388a3b13d0c6a4 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -186,7 +186,7 @@ class Test extends AsyncResource { } if (this.shouldReportSubtest) { - this.reporter.subTest(this.indent, this.name); + this.reporter.subtest(this.indent, this.name); this.shouldReportSubtest = false; } @@ -424,7 +424,7 @@ class Test extends AsyncResource { if (this.subtests.length > 0) { this.reporter.plan(this.subtests[0].indent, this.subtests.length); } else { - this.reporter.subTest(this.indent, this.name); + this.reporter.subtest(this.indent, this.name); } this.report(); From 7b00fa131e4732e98f7f613b6615abfcf1d4476d Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Thu, 16 Jun 2022 08:49:08 +0300 Subject: [PATCH 8/8] CR --- lib/internal/test_runner/test.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 388a3b13d0c6a4..2876b3d66640cd 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -93,7 +93,6 @@ class Test extends AsyncResource { this.reporter = new TapStream(); this.runOnlySubtests = this.only; this.testNumber = 0; - this.shouldReportSubtest = false; } else { const indent = parent.parent === null ? parent.indent : parent.indent + parent.indentString; @@ -105,7 +104,6 @@ class Test extends AsyncResource { this.reporter = parent.reporter; this.runOnlySubtests = !this.only; this.testNumber = parent.subtests.length + 1; - this.shouldReportSubtest = true; } if (isUint32(concurrency) && concurrency !== 0) { @@ -185,9 +183,8 @@ class Test extends AsyncResource { return; } - if (this.shouldReportSubtest) { + if (i === 1 && this.parent !== null) { this.reporter.subtest(this.indent, this.name); - this.shouldReportSubtest = false; } // Report the subtest's results and remove it from the ready map.