From 44d348e25298d2815df717118e0131862dadf9d8 Mon Sep 17 00:00:00 2001 From: cjihrig Date: Sat, 13 Jul 2024 11:10:59 -0400 Subject: [PATCH] test_runner: support running tests in process This commit introduces a new --experimental-test-isolation flag that, when set to 'none', causes the test runner to execute all tests in the same process. By default, this is the main test runner process, but if watch mode is enabled, it spawns a separate process that runs all of the tests. The default value of the new flag is 'process', which uses the existing behavior of running each test file in its own child process. It is worth noting that when the isolation mode is 'none', globals and all other top level logic (such as top level before() and after() hooks) is shared among all files. Co-authored-by: Moshe Atlow --- doc/api/cli.md | 20 +- doc/api/test.md | 20 +- doc/node.1 | 3 + lib/internal/main/test_runner.js | 6 +- lib/internal/test_runner/harness.js | 1 + lib/internal/test_runner/runner.js | 193 ++++++++++--- lib/internal/test_runner/utils.js | 44 +-- src/env-inl.h | 3 +- src/node_options.cc | 7 + src/node_options.h | 1 + .../test-runner/no-isolation/one.test.js | 32 +++ .../test-runner/no-isolation/two.test.js | 30 ++ test/fixtures/test-runner/snapshots/unit-2.js | 11 + test/parallel/test-runner-cli-concurrency.js | 14 + test/parallel/test-runner-cli-timeout.js | 8 + test/parallel/test-runner-cli.js | 271 ++++++++++-------- test/parallel/test-runner-coverage.js | 38 +++ .../test-runner-extraneous-async-activity.js | 18 ++ .../test-runner-force-exit-failure.js | 23 +- .../test-runner-no-isolation-filtering.js | 69 +++++ test/parallel/test-runner-no-isolation.mjs | 47 +++ test/parallel/test-runner-snapshot-tests.js | 72 +++++ 22 files changed, 724 insertions(+), 207 deletions(-) create mode 100644 test/fixtures/test-runner/no-isolation/one.test.js create mode 100644 test/fixtures/test-runner/no-isolation/two.test.js create mode 100644 test/fixtures/test-runner/snapshots/unit-2.js create mode 100644 test/parallel/test-runner-no-isolation-filtering.js create mode 100644 test/parallel/test-runner-no-isolation.mjs diff --git a/doc/api/cli.md b/doc/api/cli.md index 7111bf368173a4..6c4423badbed18 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -1065,6 +1065,20 @@ generated as part of the test runner output. If no tests are run, a coverage report is not generated. See the documentation on [collecting code coverage from tests][] for more details. +### `--experimental-test-isolation=mode` + + + +> Stability: 1.0 - Early development + +Configures the type of test isolation used in the test runner. When `mode` is +`'process'`, each test file is run in a separate child process. When `mode` is +`'none'`, all test files run in the same process as the test runner. The default +isolation mode is `'process'`. This flag is ignored if the `--test` flag is not +present. + ### `--experimental-test-module-mocks` The maximum number of test files that the test runner CLI will execute -concurrently. The default value is `os.availableParallelism() - 1`. +concurrently. If `--experimental-test-isolation` is set to `'none'`, this flag +is ignored and concurrency is one. Otherwise, concurrency defaults to +`os.availableParallelism() - 1`. ### `--test-coverage-exclude` @@ -2335,7 +2351,7 @@ added: v22.3.0 > Stability: 1.0 - Early development -Regenerates the snapshot file used by the test runner for [snapshot testing][]. +Regenerates the snapshot files used by the test runner for [snapshot testing][]. Node.js must be started with the `--experimental-test-snapshots` flag in order to use this functionality. diff --git a/doc/api/test.md b/doc/api/test.md index e2edeb86b72567..afb0e5f6368cb2 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -933,7 +933,7 @@ the [`--experimental-test-snapshots`][] command-line flag. Snapshot files are generated by starting Node.js with the [`--test-update-snapshots`][] command-line flag. A separate snapshot file is generated for each test file. By default, the snapshot file has the same name -as `process.argv[1]` with a `.snapshot` file extension. This behavior can be +as the test file with a `.snapshot` file extension. This behavior can be configured using the `snapshot.setResolveSnapshotPath()` function. Each snapshot assertion corresponds to an export in the snapshot file. @@ -1239,6 +1239,9 @@ added: - v18.9.0 - v16.19.0 changes: + - version: REPLACEME + pr-url: https://github.com/nodejs/node/pull/53927 + description: Added the `isolation` option. - version: REPLACEME pr-url: https://github.com/nodejs/node/pull/53866 description: Added the `globPatterns` option. @@ -1274,8 +1277,13 @@ changes: * `inspectPort` {number|Function} Sets inspector port of test child process. This can be a number, or a function that takes no arguments and returns a number. If a nullish value is provided, each process gets its own port, - incremented from the primary's `process.debugPort`. - **Default:** `undefined`. + incremented from the primary's `process.debugPort`. This option is ignored + if the `isolation` option is set to `'none'` as no child processes are + spawned. **Default:** `undefined`. + * `isolation` {string} Configures the type of test isolation. If set to + `'process'`, each test file is run in a separate child process. If set to + `'none'`, all test files run in the current process. The default isolation + mode is `'process'`. * `only`: {boolean} If truthy, the test context will only run tests that have the `only` option set * `setup` {Function} A function that accepts the `TestsStream` instance @@ -1727,9 +1735,9 @@ added: v22.3.0 * `fn` {Function} A function used to compute the location of the snapshot file. The function receives the path of the test file as its only argument. If the - `process.argv[1]` is not associated with a file (for example in the REPL), - the input is undefined. `fn()` must return a string specifying the location of - the snapshot file. + test is not associated with a file (for example in the REPL), the input is + undefined. `fn()` must return a string specifying the location of the snapshot + snapshot file. This function is used to customize the location of the snapshot file used for snapshot testing. By default, the snapshot filename is the same as the entry diff --git a/doc/node.1 b/doc/node.1 index 0d0473670535f8..fdffc7e1ae9d6c 100644 --- a/doc/node.1 +++ b/doc/node.1 @@ -185,6 +185,9 @@ Enable the experimental node:sqlite module. .It Fl -experimental-test-coverage Enable code coverage in the test runner. . +.It Fl -experimental-test-isolation Ns = Ns Ar mode +Configures the type of test isolation used in the test runner. +. .It Fl -experimental-test-module-mocks Enable module mocking in the test runner. . diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js index a42c1e5dff64cc..34f0abe557ef38 100644 --- a/lib/internal/main/test_runner.js +++ b/lib/internal/main/test_runner.js @@ -23,6 +23,7 @@ prepareMainThreadExecution(false); markBootstrapComplete(); const { + isolation, perFileTimeout, runnerConcurrency, shard, @@ -42,11 +43,12 @@ if (isUsingInspector()) { const options = { concurrency, inspectPort, - watch: watchMode, + isolation, setup: setupTestReporters, - timeout: perFileTimeout, shard, globPatterns: ArrayPrototypeSlice(process.argv, 1), + timeout: perFileTimeout, + watch: watchMode, }; debug('test runner configuration:', options); run(options).on('test:fail', (data) => { diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index dcaa75d95a640b..046b15f382eb38 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -308,4 +308,5 @@ module.exports = { after: hook('after'), beforeEach: hook('beforeEach'), afterEach: hook('afterEach'), + startSubtestAfterBootstrap, }; diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index a14cc97ce8690c..b4be0e500145aa 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -24,6 +24,7 @@ const { StringPrototypeIndexOf, StringPrototypeSlice, StringPrototypeStartsWith, + Symbol, TypedArrayPrototypeGetLength, TypedArrayPrototypeSubarray, } = primordials; @@ -44,18 +45,24 @@ const { ERR_TEST_FAILURE, }, } = require('internal/errors'); +const esmLoader = require('internal/modules/esm/loader'); const { validateArray, validateBoolean, validateFunction, validateObject, + validateOneOf, validateInteger, } = require('internal/validators'); const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector'); const { isRegExp } = require('internal/util/types'); -const { kEmptyObject } = require('internal/util'); +const { pathToFileURL } = require('internal/url'); +const { getCWDURL, kEmptyObject } = require('internal/util'); const { kEmitMessage } = require('internal/test_runner/tests_stream'); -const { createTestTree } = require('internal/test_runner/harness'); +const { + createTestTree, + startSubtestAfterBootstrap, +} = require('internal/test_runner/harness'); const { kAborted, kCancelledByParent, @@ -76,7 +83,11 @@ const { triggerUncaughtException, exitCodes: { kGenericUserError }, } = internalBinding('errors'); +let debug = require('internal/util/debuglog').debuglog('test_runner', (fn) => { + debug = fn; +}); +const kIsolatedProcessName = Symbol('kIsolatedProcessName'); const kFilterArgs = ['--test', '--experimental-test-coverage', '--watch']; const kFilterArgValues = ['--test-reporter', '--test-reporter-destination']; const kDiagnosticsFilterArgs = ['tests', 'suites', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms']; @@ -129,7 +140,12 @@ function getRunArgs(path, { forceExit, inspectPort, testNamePatterns, testSkipPa if (only === true) { ArrayPrototypePush(argv, '--test-only'); } - ArrayPrototypePush(argv, path); + + if (path === kIsolatedProcessName) { + ArrayPrototypePush(argv, '--test', ...ArrayPrototypeSlice(process.argv, 1)); + } else { + ArrayPrototypePush(argv, path); + } return argv; } @@ -325,7 +341,9 @@ class FileTest extends Test { function runTestFile(path, filesWatcher, opts) { const watchMode = filesWatcher != null; - const subtest = opts.root.createSubtest(FileTest, path, { __proto__: null, signal: opts.signal }, async (t) => { + const testPath = path === kIsolatedProcessName ? '' : path; + const testOpts = { __proto__: null, signal: opts.signal }; + const subtest = opts.root.createSubtest(FileTest, testPath, testOpts, async (t) => { const args = getRunArgs(path, opts); const stdio = ['pipe', 'pipe', 'pipe']; const env = { __proto__: null, ...process.env, NODE_TEST_CONTEXT: 'child-v8' }; @@ -417,10 +435,23 @@ function watchFiles(testFiles, opts) { const filesWatcher = { __proto__: null, watcher, runningProcesses, runningSubtests }; opts.root.harness.watching = true; + async function restartTestFile(file) { + const runningProcess = runningProcesses.get(file); + if (runningProcess) { + runningProcess.kill(); + await once(runningProcess, 'exit'); + } + if (!runningSubtests.size) { + // Reset the topLevel counter + opts.root.harness.counters.topLevel = 0; + } + await runningSubtests.get(file); + runningSubtests.set(file, runTestFile(file, filesWatcher, opts)); + } + watcher.on('changed', ({ owners, eventType }) => { if (!opts.hasFiles && eventType === 'rename') { const updatedTestFiles = createTestFileList(opts.globPatterns); - const newFileName = ArrayPrototypeFind(updatedTestFiles, (x) => !ArrayPrototypeIncludes(testFiles, x)); const previousFileName = ArrayPrototypeFind(testFiles, (x) => !ArrayPrototypeIncludes(updatedTestFiles, x)); @@ -433,25 +464,22 @@ function watchFiles(testFiles, opts) { testFiles = updatedTestFiles; } - watcher.unfilterFilesOwnedBy(owners); - PromisePrototypeThen(SafePromiseAllReturnVoid(testFiles, async (file) => { - if (!owners.has(file)) { - return; - } - const runningProcess = runningProcesses.get(file); - if (runningProcess) { - runningProcess.kill(); - await once(runningProcess, 'exit'); - } - if (!runningSubtests.size) { - // Reset the topLevel counter - opts.root.harness.counters.topLevel = 0; - } - await runningSubtests.get(file); - runningSubtests.set(file, runTestFile(file, filesWatcher, opts)); - }, undefined, (error) => { - triggerUncaughtException(error, true /* fromPromise */); - })); + if (opts.isolation === 'none') { + PromisePrototypeThen(restartTestFile(kIsolatedProcessName), undefined, (error) => { + triggerUncaughtException(error, true /* fromPromise */); + }); + } else { + watcher.unfilterFilesOwnedBy(owners); + PromisePrototypeThen(SafePromiseAllReturnVoid(testFiles, async (file) => { + if (!owners.has(file)) { + return; + } + + await restartTestFile(file); + }, undefined, (error) => { + triggerUncaughtException(error, true /* fromPromise */); + })); + } }); if (opts.signal) { kResistStopPropagation ??= require('internal/event_target').kResistStopPropagation; @@ -479,6 +507,7 @@ function run(options = kEmptyObject) { files, forceExit, inspectPort, + isolation = 'process', watch, setup, only, @@ -560,23 +589,19 @@ function run(options = kEmptyObject) { throw new ERR_INVALID_ARG_TYPE(name, ['string', 'RegExp'], value); }); } + validateOneOf(isolation, 'options.isolation', ['process', 'none']); - const root = createTestTree({ __proto__: null, concurrency, timeout, signal }); - - if (process.env.NODE_TEST_CONTEXT !== undefined) { - process.emitWarning('node:test run() is being called recursively within a test file. skipping running files.'); - root.postRun(); - return root.reporter; - } let testFiles = files ?? createTestFileList(globPatterns); if (shard) { testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1); } - let postRun = () => root.postRun(); - let teardown = () => root.harness.teardown(); + const root = createTestTree({ __proto__: null, concurrency, timeout, signal }); + let teardown; + let postRun; let filesWatcher; + let runFiles; const opts = { __proto__: null, root, @@ -588,21 +613,95 @@ function run(options = kEmptyObject) { globPatterns, only, forceExit, + isolation, }; - if (watch) { - filesWatcher = watchFiles(testFiles, opts); - postRun = undefined; - teardown = undefined; - } - const runFiles = () => { - root.harness.bootstrapPromise = null; - root.harness.allowTestsToRun = true; - return SafePromiseAllSettledReturnVoid(testFiles, (path) => { - const subtest = runTestFile(path, filesWatcher, opts); - filesWatcher?.runningSubtests.set(path, subtest); - return subtest; - }); - }; + + if (isolation === 'process') { + if (process.env.NODE_TEST_CONTEXT !== undefined) { + process.emitWarning('node:test run() is being called recursively within a test file. skipping running files.'); + root.postRun(); + return root.reporter; + } + + if (watch) { + filesWatcher = watchFiles(testFiles, opts); + } else { + postRun = () => root.postRun(); + teardown = () => root.harness.teardown(); + } + + runFiles = () => { + root.harness.bootstrapPromise = null; + root.harness.allowTestsToRun = true; + return SafePromiseAllSettledReturnVoid(testFiles, (path) => { + const subtest = runTestFile(path, filesWatcher, opts); + filesWatcher?.runningSubtests.set(path, subtest); + return subtest; + }); + }; + } else { + if (watch) { + filesWatcher = watchFiles(testFiles, opts); + runFiles = async () => { + root.harness.bootstrapPromise = null; + root.harness.allowTestsToRun = true; + const subtest = runTestFile(kIsolatedProcessName, filesWatcher, opts); + filesWatcher?.runningSubtests.set(kIsolatedProcessName, subtest); + return subtest; + }; + } else { + runFiles = async () => { + const { promise, resolve: finishBootstrap } = Promise.withResolvers(); + + await root.runInAsyncScope(async () => { + const parentURL = getCWDURL().href; + const cascadedLoader = esmLoader.getOrInitializeCascadedLoader(); + let topLevelTestCount = 0; + + root.harness.bootstrapPromise = promise; + + for (let i = 0; i < testFiles.length; ++i) { + const testFile = testFiles[i]; + const fileURL = pathToFileURL(testFile); + let threw = false; + let importError; + + root.entryFile = resolve(testFile); + debug('loading test file:', fileURL.href); + try { + await cascadedLoader.import(fileURL, parentURL, { __proto__: null }); + } catch (err) { + threw = true; + importError = err; + } + + debug( + 'loaded "%s": top level test count before = %d and after = %d', + testFile, + topLevelTestCount, + root.subtests.length, + ); + if (topLevelTestCount === root.subtests.length) { + // This file had no tests in it. Add the placeholder test. + const subtest = root.createSubtest(Test, testFile); + if (threw) { + subtest.fail(importError); + } + startSubtestAfterBootstrap(subtest); + } + + topLevelTestCount = root.subtests.length; + } + }); + + debug('beginning test execution'); + root.entryFile = null; + finishBootstrap(); + root.harness.allowTestsToRun = true; + root.processPendingSubtests(); + }; + } + } const setupPromise = PromiseResolve(setup?.(root.reporter)); PromisePrototypeThen(PromisePrototypeThen(PromisePrototypeThen(setupPromise, runFiles), postRun), teardown); diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index aae2a756800a0f..e05707bac24e63 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -198,17 +198,18 @@ function parseCommandLine() { const sourceMaps = getOptionValue('--enable-source-maps'); const updateSnapshots = getOptionValue('--test-update-snapshots'); const watchMode = getOptionValue('--watch'); + let testOnlyFlag = getOptionValue('--test-only'); + let testNamePatterns = mapPatternFlagToRegExArray('--test-name-pattern'); + let testSkipPatterns = mapPatternFlagToRegExArray('--test-skip-pattern'); const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child'; const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8'; let coverageExcludeGlobs; let coverageIncludeGlobs; let destinations; + let isolation; let perFileTimeout; let reporters; let runnerConcurrency; - let testNamePatterns; - let testSkipPatterns; - let testOnlyFlag; let shard; if (isChildProcessV8) { @@ -239,10 +240,20 @@ function parseCommandLine() { } if (isTestRunner) { + isolation = getOptionValue('--experimental-test-isolation'); perFileTimeout = getOptionValue('--test-timeout') || Infinity; - runnerConcurrency = getOptionValue('--test-concurrency') || true; - testOnlyFlag = false; - testNamePatterns = null; + + if (isolation === 'none') { + runnerConcurrency = 1; + } else { + runnerConcurrency = getOptionValue('--test-concurrency') || true; + + // This should not be necessary once lib/internal/test.js is not using + // command line flags directly. + testOnlyFlag = false; + testNamePatterns = null; + testSkipPatterns = null; + } const shardOption = getOptionValue('--test-shard'); if (shardOption) { @@ -264,16 +275,6 @@ function parseCommandLine() { } else { perFileTimeout = Infinity; runnerConcurrency = 1; - const testNamePatternFlag = getOptionValue('--test-name-pattern'); - testOnlyFlag = getOptionValue('--test-only'); - testNamePatterns = testNamePatternFlag?.length > 0 ? - ArrayPrototypeMap( - testNamePatternFlag, - (re) => convertStringToRegExp(re, '--test-name-pattern'), - ) : null; - const testSkipPatternFlag = getOptionValue('--test-skip-pattern'); - testSkipPatterns = testSkipPatternFlag?.length > 0 ? - ArrayPrototypeMap(testSkipPatternFlag, (re) => convertStringToRegExp(re, '--test-skip-pattern')) : null; } if (coverage) { @@ -288,6 +289,7 @@ function parseCommandLine() { coverageExcludeGlobs, coverageIncludeGlobs, forceExit, + isolation, perFileTimeout, runnerConcurrency, shard, @@ -304,6 +306,16 @@ function parseCommandLine() { return globalTestOptions; } +function mapPatternFlagToRegExArray(flagName) { + const patterns = getOptionValue(flagName); + + if (patterns?.length > 0) { + return ArrayPrototypeMap(patterns, (re) => convertStringToRegExp(re, flagName)); + } + + return null; +} + function countCompletedTest(test, harness = test.root.harness) { if (test.nesting === 0) { harness.counters.topLevel++; diff --git a/src/env-inl.h b/src/env-inl.h index a03358a3386e47..ecd5c368009df7 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -675,7 +675,8 @@ inline bool Environment::owns_inspector() const { inline bool Environment::should_create_inspector() const { return (flags_ & EnvironmentFlags::kNoCreateInspector) == 0 && - !options_->test_runner && !options_->watch_mode; + !(options_->test_runner && options_->test_isolation == "process") && + !options_->watch_mode; } inline bool Environment::should_wait_for_inspector_frontend() const { diff --git a/src/node_options.cc b/src/node_options.cc index 4b3b06bbc7ee1c..b6232a70578a0b 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -143,6 +143,10 @@ void EnvironmentOptions::CheckOptions(std::vector* errors, } if (test_runner) { + if (test_isolation != "process" && test_isolation != "none") { + errors->push_back("invalid value for --experimental-test-isolation"); + } + if (syntax_check_only) { errors->push_back("either --test or --check can be used, not both"); } @@ -646,6 +650,9 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { AddOption("--experimental-test-coverage", "enable code coverage in the test runner", &EnvironmentOptions::test_runner_coverage); + AddOption("--experimental-test-isolation", + "configures the type of test isolation used in the test runner", + &EnvironmentOptions::test_isolation); AddOption("--experimental-test-module-mocks", "enable module mocking in the test runner", &EnvironmentOptions::test_runner_module_mocks); diff --git a/src/node_options.h b/src/node_options.h index 77db2987656b3d..2a614ad27c08d6 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -185,6 +185,7 @@ class EnvironmentOptions : public Options { std::vector test_reporter_destination; bool test_only = false; bool test_udp_no_try_send = false; + std::string test_isolation = "process"; std::string test_shard; std::vector test_skip_pattern; std::vector coverage_include_pattern; diff --git a/test/fixtures/test-runner/no-isolation/one.test.js b/test/fixtures/test-runner/no-isolation/one.test.js new file mode 100644 index 00000000000000..69e0485a37127b --- /dev/null +++ b/test/fixtures/test-runner/no-isolation/one.test.js @@ -0,0 +1,32 @@ +'use strict'; +const { before, beforeEach, after, afterEach, test, suite } = require('node:test'); + +globalThis.GLOBAL_ORDER = []; + +before(function() { + GLOBAL_ORDER.push(`before one: ${this.name}`); +}); + +beforeEach(function() { + GLOBAL_ORDER.push(`beforeEach one: ${this.name}`); +}); + +after(function() { + GLOBAL_ORDER.push(`after one: ${this.name}`); +}); + +afterEach(function() { + GLOBAL_ORDER.push(`afterEach one: ${this.name}`); +}); + +suite('suite one', function() { + GLOBAL_ORDER.push(this.name); + + test('suite one - test', { only: true }, function() { + GLOBAL_ORDER.push(this.name); + }); +}); + +test('test one', function() { + GLOBAL_ORDER.push(this.name); +}); diff --git a/test/fixtures/test-runner/no-isolation/two.test.js b/test/fixtures/test-runner/no-isolation/two.test.js new file mode 100644 index 00000000000000..50ae6541ce156d --- /dev/null +++ b/test/fixtures/test-runner/no-isolation/two.test.js @@ -0,0 +1,30 @@ +'use strict'; +const { before, beforeEach, after, afterEach, test, suite } = require('node:test'); + +before(function() { + GLOBAL_ORDER.push(`before two: ${this.name}`); +}); + +beforeEach(function() { + GLOBAL_ORDER.push(`beforeEach two: ${this.name}`); +}); + +after(function() { + GLOBAL_ORDER.push(`after two: ${this.name}`); +}); + +afterEach(function() { + GLOBAL_ORDER.push(`afterEach two: ${this.name}`); +}); + +suite('suite two', function() { + GLOBAL_ORDER.push(this.name); + + before(function() { + GLOBAL_ORDER.push(`before suite two: ${this.name}`); + }); + + test('suite two - test', { only: true }, function() { + GLOBAL_ORDER.push(this.name); + }); +}); diff --git a/test/fixtures/test-runner/snapshots/unit-2.js b/test/fixtures/test-runner/snapshots/unit-2.js new file mode 100644 index 00000000000000..311378b2810136 --- /dev/null +++ b/test/fixtures/test-runner/snapshots/unit-2.js @@ -0,0 +1,11 @@ +'use strict'; +const { snapshot, test } = require('node:test'); +const { basename, join } = require('node:path'); + +snapshot.setResolveSnapshotPath((testFile) => { + return join(process.cwd(), `${basename(testFile)}.snapshot`); +}); + +test('has a snapshot', (t) => { + t.assert.snapshot('a snapshot from ' + __filename); +}); diff --git a/test/parallel/test-runner-cli-concurrency.js b/test/parallel/test-runner-cli-concurrency.js index fbabaf08e27279..b2aa0ac6c3c6c5 100644 --- a/test/parallel/test-runner-cli-concurrency.js +++ b/test/parallel/test-runner-cli-concurrency.js @@ -24,3 +24,17 @@ test('concurrency of two', async () => { const cp = spawnSync(process.execPath, args, { cwd, env }); assert.match(cp.stderr.toString(), /concurrency: 2,/); }); + +test('isolation=none uses a concurrency of one', async () => { + const args = ['--test', '--experimental-test-isolation=none']; + const cp = spawnSync(process.execPath, args, { cwd, env }); + assert.match(cp.stderr.toString(), /concurrency: 1,/); +}); + +test('isolation=none overrides --test-concurrency', async () => { + const args = [ + '--test', '--experimental-test-isolation=none', '--test-concurrency=2', + ]; + const cp = spawnSync(process.execPath, args, { cwd, env }); + assert.match(cp.stderr.toString(), /concurrency: 1,/); +}); diff --git a/test/parallel/test-runner-cli-timeout.js b/test/parallel/test-runner-cli-timeout.js index b8998d397fa12c..53a3e4ce7ea48e 100644 --- a/test/parallel/test-runner-cli-timeout.js +++ b/test/parallel/test-runner-cli-timeout.js @@ -18,3 +18,11 @@ test('timeout of 10ms', async () => { const cp = spawnSync(process.execPath, args, { cwd, env }); assert.match(cp.stderr.toString(), /timeout: 10,/); }); + +test('isolation=none uses the --test-timeout flag', async () => { + const args = [ + '--test', '--experimental-test-isolation=none', '--test-timeout=10', + ]; + const cp = spawnSync(process.execPath, args, { cwd, env }); + assert.match(cp.stderr.toString(), /timeout: 10,/); +}); diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js index f165a509c995cc..d2d2eea8809404 100644 --- a/test/parallel/test-runner-cli.js +++ b/test/parallel/test-runner-cli.js @@ -7,92 +7,162 @@ const { join } = require('path'); const fixtures = require('../common/fixtures'); const testFixtures = fixtures.path('test-runner'); -{ - // File not found. - const args = ['--test', 'a-random-file-that-does-not-exist.js']; - const child = spawnSync(process.execPath, args); +for (const isolation of ['none', 'process']) { + { + // File not found. + const args = [ + '--test', + `--experimental-test-isolation=${isolation}`, + 'a-random-file-that-does-not-exist.js', + ]; + const child = spawnSync(process.execPath, args); - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stdout.toString(), ''); - assert.match(child.stderr.toString(), /^Could not find/); -} + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stdout.toString(), ''); + assert.match(child.stderr.toString(), /^Could not find/); + } -{ - // Default behavior. node_modules is ignored. Files that don't match the - // pattern are ignored except in test/ directories. - const args = ['--test']; - const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') }); + { + // Default behavior. node_modules is ignored. Files that don't match the + // pattern are ignored except in test/ directories. + const args = ['--test', `--experimental-test-isolation=${isolation}`]; + const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') }); - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); - const stdout = child.stdout.toString(); - assert.match(stdout, /ok 1 - this should pass/); - assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); - assert.match(stdout, /ok 4 - this should pass/); - assert.match(stdout, /ok 5 - this should be skipped/); - assert.match(stdout, /ok 6 - this should be executed/); -} + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, /not ok 2 - this should fail/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); + assert.match(stdout, /ok 4 - this should pass/); + assert.match(stdout, /ok 5 - this should be skipped/); + assert.match(stdout, /ok 6 - this should be executed/); + } -{ - // Same but with a prototype mutation in require scripts. - const args = ['--require', join(testFixtures, 'protoMutation.js'), '--test']; - const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') }); + { + // Same but with a prototype mutation in require scripts. + const args = [ + '--require', join(testFixtures, 'protoMutation.js'), + '--test', + `--experimental-test-isolation=${isolation}`, + ]; + const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') }); + + const stdout = child.stdout.toString(); + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, /not ok 2 - this should fail/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); + assert.match(stdout, /ok 4 - this should pass/); + assert.match(stdout, /ok 5 - this should be skipped/); + assert.match(stdout, /ok 6 - this should be executed/); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + } - const stdout = child.stdout.toString(); - assert.match(stdout, /ok 1 - this should pass/); - assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); - assert.match(stdout, /ok 4 - this should pass/); - assert.match(stdout, /ok 5 - this should be skipped/); - assert.match(stdout, /ok 6 - this should be executed/); - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); -} + { + // User specified files that don't match the pattern are still run. + const args = [ + '--test', + `--experimental-test-isolation=${isolation}`, + join(testFixtures, 'index.js'), + ]; + const child = spawnSync(process.execPath, args, { cwd: testFixtures }); -{ - // User specified files that don't match the pattern are still run. - const args = ['--test', join(testFixtures, 'index.js')]; - const child = spawnSync(process.execPath, args, { cwd: testFixtures }); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert.match(stdout, /not ok 1 - .+index\.js/); + } - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); - const stdout = child.stdout.toString(); - assert.match(stdout, /not ok 1 - .+index\.js/); -} + { + // Searches node_modules if specified. + const args = [ + '--test', + `--experimental-test-isolation=${isolation}`, + join(testFixtures, 'default-behavior/node_modules/*.js'), + ]; + const child = spawnSync(process.execPath, args); -{ - // Searches node_modules if specified. - const args = ['--test', join(testFixtures, 'default-behavior/node_modules/*.js')]; - const child = spawnSync(process.execPath, args); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert.match(stdout, /not ok 1 - .+test-nm\.js/); + } - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); - const stdout = child.stdout.toString(); - assert.match(stdout, /not ok 1 - .+test-nm\.js/); -} + { + // The current directory is used by default. + const args = ['--test', `--experimental-test-isolation=${isolation}`]; + const options = { cwd: join(testFixtures, 'default-behavior') }; + const child = spawnSync(process.execPath, args, options); -{ - // The current directory is used by default. - const args = ['--test']; - const options = { cwd: join(testFixtures, 'default-behavior') }; - const child = spawnSync(process.execPath, args, options); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, /not ok 2 - this should fail/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); + assert.match(stdout, /ok 4 - this should pass/); + assert.match(stdout, /ok 5 - this should be skipped/); + assert.match(stdout, /ok 6 - this should be executed/); + } - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); - const stdout = child.stdout.toString(); - assert.match(stdout, /ok 1 - this should pass/); - assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); - assert.match(stdout, /ok 4 - this should pass/); - assert.match(stdout, /ok 5 - this should be skipped/); - assert.match(stdout, /ok 6 - this should be executed/); + { + // Test combined stream outputs + const args = [ + '--test', + `--experimental-test-isolation=${isolation}`, + 'test/fixtures/test-runner/default-behavior/index.test.js', + 'test/fixtures/test-runner/nested.js', + 'test/fixtures/test-runner/invalid-tap.js', + ]; + const child = spawnSync(process.execPath, args); + + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert.match(stdout, /# Subtest: this should pass/); + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, / {2}---/); + assert.match(stdout, / {2}duration_ms: .*/); + assert.match(stdout, / {2}\.\.\./); + + assert.match(stdout, /# Subtest: .+invalid-tap\.js/); + assert.match(stdout, /invalid tap output/); + assert.match(stdout, /ok 2 - .+invalid-tap\.js/); + + assert.match(stdout, /# Subtest: level 0a/); + assert.match(stdout, / {4}# Subtest: level 1a/); + assert.match(stdout, / {4}ok 1 - level 1a/); + assert.match(stdout, / {4}# Subtest: level 1b/); + assert.match(stdout, / {4}not ok 2 - level 1b/); + assert.match(stdout, / {6}code: 'ERR_TEST_FAILURE'/); + assert.match(stdout, / {6}stack: |-'/); + assert.match(stdout, / {8}TestContext\. .*/); + assert.match(stdout, / {4}# Subtest: level 1c/); + assert.match(stdout, / {4}ok 3 - level 1c # SKIP aaa/); + assert.match(stdout, / {4}# Subtest: level 1d/); + assert.match(stdout, / {4}ok 4 - level 1d/); + assert.match(stdout, /not ok 3 - level 0a/); + assert.match(stdout, / {2}error: '1 subtest failed'/); + assert.match(stdout, /# Subtest: level 0b/); + assert.match(stdout, /not ok 4 - level 0b/); + assert.match(stdout, / {2}error: 'level 0b error'/); + assert.match(stdout, /# tests 8/); + assert.match(stdout, /# suites 0/); + assert.match(stdout, /# pass 4/); + assert.match(stdout, /# fail 3/); + assert.match(stdout, /# cancelled 0/); + assert.match(stdout, /# skipped 1/); + assert.match(stdout, /# todo 0/); + } } { @@ -115,57 +185,6 @@ const testFixtures = fixtures.path('test-runner'); } } -{ - // Test combined stream outputs - const args = [ - '--test', - 'test/fixtures/test-runner/default-behavior/index.test.js', - 'test/fixtures/test-runner/nested.js', - 'test/fixtures/test-runner/invalid-tap.js', - ]; - const child = spawnSync(process.execPath, args); - - - assert.strictEqual(child.status, 1); - assert.strictEqual(child.signal, null); - assert.strictEqual(child.stderr.toString(), ''); - const stdout = child.stdout.toString(); - assert.match(stdout, /# Subtest: this should pass/); - assert.match(stdout, /ok 1 - this should pass/); - assert.match(stdout, / {2}---/); - assert.match(stdout, / {2}duration_ms: .*/); - assert.match(stdout, / {2}\.\.\./); - - assert.match(stdout, /# Subtest: .+invalid-tap\.js/); - assert.match(stdout, /# invalid tap output/); - assert.match(stdout, /ok 2 - .+invalid-tap\.js/); - - assert.match(stdout, /# Subtest: level 0a/); - assert.match(stdout, / {4}# Subtest: level 1a/); - assert.match(stdout, / {4}ok 1 - level 1a/); - assert.match(stdout, / {4}# Subtest: level 1b/); - assert.match(stdout, / {4}not ok 2 - level 1b/); - assert.match(stdout, / {6}code: 'ERR_TEST_FAILURE'/); - assert.match(stdout, / {6}stack: |-'/); - assert.match(stdout, / {8}TestContext\. .*/); - assert.match(stdout, / {4}# Subtest: level 1c/); - assert.match(stdout, / {4}ok 3 - level 1c # SKIP aaa/); - assert.match(stdout, / {4}# Subtest: level 1d/); - assert.match(stdout, / {4}ok 4 - level 1d/); - assert.match(stdout, /not ok 3 - level 0a/); - assert.match(stdout, / {2}error: '1 subtest failed'/); - assert.match(stdout, /# Subtest: level 0b/); - assert.match(stdout, /not ok 4 - level 0b/); - assert.match(stdout, / {2}error: 'level 0b error'/); - assert.match(stdout, /# tests 8/); - assert.match(stdout, /# suites 0/); - assert.match(stdout, /# pass 4/); - assert.match(stdout, /# fail 3/); - assert.match(stdout, /# cancelled 0/); - assert.match(stdout, /# skipped 1/); - assert.match(stdout, /# todo 0/); -} - { // Test user logging in tests. const args = [ diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index 1283232a867246..ddf35ea7bed179 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -187,6 +187,44 @@ test('coverage is combined for multiple processes', skipIfNoInspector, () => { assert.strictEqual(result.status, 0); }); +test('coverage works with isolation=none', skipIfNoInspector, () => { + // There is a bug in coverage calculation. The branch % in the common.js + // fixture is different depending on the test isolation mode. The 'none' mode + // is closer to what c8 reports here, so the bug is likely in the code that + // merges reports from different processes. + let report = [ + '# start of coverage report', + '# -------------------------------------------------------------------', + '# file | line % | branch % | funcs % | uncovered lines', + '# -------------------------------------------------------------------', + '# common.js | 89.86 | 68.42 | 100.00 | 8 13-14 18 34-35 53', + '# first.test.js | 83.33 | 100.00 | 50.00 | 5-6', + '# second.test.js | 100.00 | 100.00 | 100.00 | ', + '# third.test.js | 100.00 | 100.00 | 100.00 | ', + '# -------------------------------------------------------------------', + '# all files | 92.11 | 76.00 | 88.89 |', + '# -------------------------------------------------------------------', + '# end of coverage report', + ].join('\n'); + + if (common.isWindows) { + report = report.replaceAll('/', '\\'); + } + + const fixture = fixtures.path('v8-coverage', 'combined_coverage'); + const args = [ + '--test', '--experimental-test-coverage', '--test-reporter', 'tap', '--experimental-test-isolation=none', + ]; + const result = spawnSync(process.execPath, args, { + env: { ...process.env, NODE_TEST_TMPDIR: tmpdir.path }, + cwd: fixture, + }); + + assert.strictEqual(result.stderr.toString(), ''); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.status, 0); +}); + test('coverage reports on lines, functions, and branches', skipIfNoInspector, async (t) => { const fixture = fixtures.path('test-runner', 'coverage.js'); const child = spawnSync(process.execPath, diff --git a/test/parallel/test-runner-extraneous-async-activity.js b/test/parallel/test-runner-extraneous-async-activity.js index 68db109b292f15..23f3194e02f106 100644 --- a/test/parallel/test-runner-extraneous-async-activity.js +++ b/test/parallel/test-runner-extraneous-async-activity.js @@ -48,3 +48,21 @@ const { spawnSync } = require('child_process'); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); } + +{ + const child = spawnSync(process.execPath, [ + '--test', + '--experimental-test-isolation=none', + fixtures.path('test-runner', 'async-error-in-test-hook.mjs'), + ]); + const stdout = child.stdout.toString(); + assert.match(stdout, /^# Error: Test hook "before" at .+async-error-in-test-hook\.mjs:3:1 generated asynchronous activity after the test ended/m); + assert.match(stdout, /^# Error: Test hook "beforeEach" at .+async-error-in-test-hook\.mjs:9:1 generated asynchronous activity after the test ended/m); + assert.match(stdout, /^# Error: Test hook "after" at .+async-error-in-test-hook\.mjs:15:1 generated asynchronous activity after the test ended/m); + assert.match(stdout, /^# Error: Test hook "afterEach" at .+async-error-in-test-hook\.mjs:21:1 generated asynchronous activity after the test ended/m); + assert.match(stdout, /^# pass 1$/m); + assert.match(stdout, /^# fail 0$/m); + assert.match(stdout, /^# cancelled 0$/m); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); +} diff --git a/test/parallel/test-runner-force-exit-failure.js b/test/parallel/test-runner-force-exit-failure.js index 1fff8f30d7e038..ce1f3208c5b4e6 100644 --- a/test/parallel/test-runner-force-exit-failure.js +++ b/test/parallel/test-runner-force-exit-failure.js @@ -4,12 +4,21 @@ const { match, doesNotMatch, strictEqual } = require('node:assert'); const { spawnSync } = require('node:child_process'); const fixtures = require('../common/fixtures'); const fixture = fixtures.path('test-runner/throws_sync_and_async.js'); -const r = spawnSync(process.execPath, ['--test', '--test-force-exit', fixture]); -strictEqual(r.status, 1); -strictEqual(r.signal, null); -strictEqual(r.stderr.toString(), ''); +for (const isolation of ['none', 'process']) { + const args = [ + '--test', + '--test-force-exit', + `--experimental-test-isolation=${isolation}`, + fixture, + ]; + const r = spawnSync(process.execPath, args); -const stdout = r.stdout.toString(); -match(stdout, /error: 'fails'/); -doesNotMatch(stdout, /this should not have a chance to be thrown/); + strictEqual(r.status, 1); + strictEqual(r.signal, null); + strictEqual(r.stderr.toString(), ''); + + const stdout = r.stdout.toString(); + match(stdout, /error: 'fails'/); + doesNotMatch(stdout, /this should not have a chance to be thrown/); +} diff --git a/test/parallel/test-runner-no-isolation-filtering.js b/test/parallel/test-runner-no-isolation-filtering.js new file mode 100644 index 00000000000000..f8fba1cbfffbef --- /dev/null +++ b/test/parallel/test-runner-no-isolation-filtering.js @@ -0,0 +1,69 @@ +'use strict'; +require('../common'); +const fixtures = require('../common/fixtures'); +const assert = require('node:assert'); +const { spawnSync } = require('node:child_process'); +const { test } = require('node:test'); + +const fixture1 = fixtures.path('test-runner', 'no-isolation', 'one.test.js'); +const fixture2 = fixtures.path('test-runner', 'no-isolation', 'two.test.js'); + +test('works with --test-only', () => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--test-only', + fixture1, + fixture2, + ]; + const child = spawnSync(process.execPath, args); + const stdout = child.stdout.toString(); + + assert.strictEqual(child.status, 0); + assert.strictEqual(child.signal, null); + assert.match(stdout, /# tests 2/); + assert.match(stdout, /# suites 2/); + assert.match(stdout, /# pass 2/); + assert.match(stdout, /ok 1 - suite one/); + assert.match(stdout, /ok 1 - suite one - test/); + assert.match(stdout, /ok 2 - suite two/); + assert.match(stdout, /ok 1 - suite two - test/); +}); + +test('works with --test-name-pattern', () => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--test-name-pattern=/test one/', + fixture1, + fixture2, + ]; + const child = spawnSync(process.execPath, args); + const stdout = child.stdout.toString(); + + assert.strictEqual(child.status, 0); + assert.strictEqual(child.signal, null); + assert.match(stdout, /# tests 1/); + assert.match(stdout, /# suites 0/); + assert.match(stdout, /# pass 1/); + assert.match(stdout, /ok 1 - test one/); +}); + +test('works with --test-skip-pattern', () => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--test-skip-pattern=/one/', + fixture1, + fixture2, + ]; + const child = spawnSync(process.execPath, args); + const stdout = child.stdout.toString(); + + assert.strictEqual(child.status, 0); + assert.strictEqual(child.signal, null); + assert.match(stdout, /# tests 1/); + assert.match(stdout, /# suites 1/); + assert.match(stdout, /# pass 1/); + assert.match(stdout, /ok 1 - suite two - test/); +}); diff --git a/test/parallel/test-runner-no-isolation.mjs b/test/parallel/test-runner-no-isolation.mjs new file mode 100644 index 00000000000000..9547f458e0bd3b --- /dev/null +++ b/test/parallel/test-runner-no-isolation.mjs @@ -0,0 +1,47 @@ +import { allowGlobals, mustCall, mustNotCall } from '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import { deepStrictEqual } from 'node:assert'; +import { run } from 'node:test'; + +const stream = run({ + files: [ + fixtures.path('test-runner', 'no-isolation', 'one.test.js'), + fixtures.path('test-runner', 'no-isolation', 'two.test.js'), + ], + isolation: 'none', +}); + +stream.on('test:fail', mustNotCall()); +stream.on('test:pass', mustCall(5)); +// eslint-disable-next-line no-unused-vars +for await (const _ of stream); +allowGlobals(globalThis.GLOBAL_ORDER); +deepStrictEqual(globalThis.GLOBAL_ORDER, [ + 'suite one', + 'suite two', + 'before one: ', + 'before two: ', + + 'beforeEach one: suite one - test', + 'beforeEach two: suite one - test', + 'suite one - test', + 'afterEach one: suite one - test', + 'afterEach two: suite one - test', + + 'beforeEach one: test one', + 'beforeEach two: test one', + 'test one', + 'afterEach one: test one', + 'afterEach two: test one', + + 'before suite two: suite two', + + 'beforeEach one: suite two - test', + 'beforeEach two: suite two - test', + 'suite two - test', + 'afterEach one: suite two - test', + 'afterEach two: suite two - test', + + 'after one: ', + 'after two: ', +]); diff --git a/test/parallel/test-runner-snapshot-tests.js b/test/parallel/test-runner-snapshot-tests.js index e00019ef49d4f6..62ebdd3cade2fb 100644 --- a/test/parallel/test-runner-snapshot-tests.js +++ b/test/parallel/test-runner-snapshot-tests.js @@ -339,3 +339,75 @@ test('t.assert.snapshot()', async (t) => { t.assert.match(child.stdout, /fail 0/); }); }); + +test('snapshots from multiple files (isolation=none)', async (t) => { + tmpdir.refresh(); + + const fixture = fixtures.path('test-runner', 'snapshots', 'unit.js'); + const fixture2 = fixtures.path('test-runner', 'snapshots', 'unit-2.js'); + + await t.test('fails prior to snapshot generation', async (t) => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--experimental-test-snapshots', + fixture, + fixture2, + ]; + const child = await common.spawnPromisified( + process.execPath, + args, + { cwd: tmpdir.path }, + ); + + t.assert.strictEqual(child.code, 1); + t.assert.strictEqual(child.signal, null); + t.assert.match(child.stdout, /# tests 6/); + t.assert.match(child.stdout, /# pass 0/); + t.assert.match(child.stdout, /# fail 6/); + t.assert.match(child.stdout, /Missing snapshots/); + }); + + await t.test('passes when regenerating snapshots', async (t) => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--experimental-test-snapshots', + '--test-update-snapshots', + fixture, + fixture2, + ]; + const child = await common.spawnPromisified( + process.execPath, + args, + { cwd: tmpdir.path }, + ); + + t.assert.strictEqual(child.code, 0); + t.assert.strictEqual(child.signal, null); + t.assert.match(child.stdout, /tests 6/); + t.assert.match(child.stdout, /pass 6/); + t.assert.match(child.stdout, /fail 0/); + }); + + await t.test('passes when snapshots exist', async (t) => { + const args = [ + '--test', + '--experimental-test-isolation=none', + '--experimental-test-snapshots', + fixture, + fixture2, + ]; + const child = await common.spawnPromisified( + process.execPath, + args, + { cwd: tmpdir.path }, + ); + + t.assert.strictEqual(child.code, 0); + t.assert.strictEqual(child.signal, null); + t.assert.match(child.stdout, /tests 6/); + t.assert.match(child.stdout, /pass 6/); + t.assert.match(child.stdout, /fail 0/); + }); +});