test_runner: add bail out

doc/api/cli.md

@@ -2254,6 +2254,17 @@ Starts the Node.js command line test runner. This flag cannot be combined with
 See the documentation on [running tests from the command line][]
 for more details.
 
+### `--test-bail`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+Instructs the test runner to bail out if a test failure occurs.
+See the documentation on [test bailout][] for more details.
+
 ### `--test-concurrency`
 
 <!-- YAML
@@ -3714,6 +3725,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12
 [single executable application]: single-executable-applications.md
 [snapshot testing]: test.md#snapshot-testing
 [syntax detection]: packages.md#syntax-detection
+[test bailout]: test.md#bailing-out
 [test reporters]: test.md#test-reporters
 [test runner execution model]: test.md#test-runner-execution-model
 [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones

doc/api/test.md

@@ -990,6 +990,22 @@ exports[`suite of snapshot tests > snapshot test 2`] = `
 Once the snapshot file is created, run the tests again without the
 `--test-update-snapshots` flag. The tests should pass now.
 
+## Bailing out
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--test-bail` flag provides a way to stop test execution
+as soon as a test fails.
+By enabling this flag, the test runner will cancel all remaining tests
+when it encounters the first failing test.
+
+Note: The bail option is not currently supported in watch mode.
+
 ## Test reporters
 
 <!-- YAML
@@ -1077,6 +1093,9 @@ const customReporter = new Transform({
       case 'test:fail':
         callback(null, `test ${event.data.name} failed`);
         break;
+      case 'test:bail':
+        callback(null, `test ${event.data.name} bailed out`);
+        break;
       case 'test:plan':
         callback(null, 'test plan');
         break;
@@ -1122,6 +1141,9 @@ const customReporter = new Transform({
       case 'test:fail':
         callback(null, `test ${event.data.name} failed`);
         break;
+      case 'test:bail':
+        callback(null, `test ${event.data.name} bailed out`);
+        break;
       case 'test:plan':
         callback(null, 'test plan');
         break;
@@ -1166,6 +1188,9 @@ export default async function * customReporter(source) {
       case 'test:fail':
         yield `test ${event.data.name} failed\n`;
         break;
+      case 'test:bail':
+        yield `test ${event.data.name} bailed out\n`;
+        break;
       case 'test:plan':
         yield 'test plan\n';
         break;
@@ -1206,6 +1231,9 @@ module.exports = async function * customReporter(source) {
       case 'test:fail':
         yield `test ${event.data.name} failed\n`;
         break;
+      case 'test:bail':
+        yield `test ${event.data.name} bailed out\n`;
+        break;
       case 'test:plan':
         yield 'test plan\n';
         break;
@@ -1289,6 +1317,10 @@ changes:
     parallel.
     If `false`, it would only run one test file at a time.
     **Default:** `false`.
+  * `bail`: {boolean} Determines whether the test runner stops execution after the first test failure.
+    If set to `true`, the runner cancels all remaining tests immediately upon encountering a failure,
+    following the [bailing out][] behavior.
+    **Default:** `false`.
   * `cwd`: {string} Specifies the current working directory to be used by the test runner.
     Serves as the base path for resolving files according to the [test runner execution model][].
     **Default:** `process.cwd()`.
@@ -1316,8 +1348,7 @@ changes:
     and can be used to setup listeners before any tests are run.
     **Default:** `undefined`.
   * `execArgv` {Array} An array of CLI flags to pass to the `node` executable when
-    spawning the subprocesses. This option has no effect when `isolation` is `'none`'.
-    **Default:** `[]`
+    spawning the subprocesses. This option has no effect when `isolation` is `'none''. **Default:** `\[]\`
   * `argv` {Array} An array of CLI flags to pass to each test file when spawning the
     subprocesses. This option has no effect when `isolation` is `'none'`.
     **Default:** `[]`.
@@ -3124,6 +3155,22 @@ generated for each test file in addition to a final cumulative summary.
 
 Emitted when no more tests are queued for execution in watch mode.
 
+### Event: `'test:bail'`
+
+* `data` {Object}
+  * `column` {number|undefined} The column number where the test is defined, or
+    `undefined` if the test was run through the REPL.
+  * `file` {string|undefined} The path of the test file,
+    `undefined` if test was run through the REPL.
+  * `line` {number|undefined} The line number where the test is defined, or
+    `undefined` if the test was run through the REPL.
+  * `name` {string} The test name.
+  * `nesting` {number} The nesting level of the test.
+
+Emitted when the test runner stops executing tests due to the [`--test-bail`][] flag.
+This event signals that the first failing test caused the suite to bail out,
+canceling all pending and currently running tests.
+
 ## Class: `TestContext`
 
 <!-- YAML
@@ -3618,6 +3665,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`--experimental-test-module-mocks`]: cli.md#--experimental-test-module-mocks
 [`--import`]: cli.md#--importmodule
 [`--no-experimental-strip-types`]: cli.md#--no-experimental-strip-types
+[`--test-bail`]: cli.md#--test-bail
 [`--test-concurrency`]: cli.md#--test-concurrency
 [`--test-coverage-exclude`]: cli.md#--test-coverage-exclude
 [`--test-coverage-include`]: cli.md#--test-coverage-include
@@ -3644,6 +3692,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`run()`]: #runoptions
 [`suite()`]: #suitename-options-fn
 [`test()`]: #testname-options-fn
+[bailing out]: #bailing-out
 [code coverage]: #collecting-code-coverage
 [describe options]: #describename-options-fn
 [it options]: #testname-options-fn

lib/internal/test_runner/harness.js

@@ -27,6 +27,7 @@ const {
   shouldColorizeTestFiles,
 } = require('internal/test_runner/utils');
 const { queueMicrotask } = require('internal/process/task_queues');
+const { AbortController, AbortSignal } = require('internal/abort_controller');
 const { bigint: hrtime } = process.hrtime;
 const resolvedPromise = PromiseResolve();
 const testResources = new SafeMap();
@@ -76,12 +77,19 @@ function createTestTree(rootTestOptions, globalOptions) {
 
       buildPhaseDeferred.resolve();
     },
+    abortController: new AbortController(),
+    abortTestTree() {
+      harness.abortController.abort();
+    },
   };
 
   harness.resetCounters();
   globalRoot = new Test({
     __proto__: null,
     ...rootTestOptions,
+    signal: rootTestOptions.signal ?
+      AbortSignal.any([rootTestOptions.signal, harness.abortController.signal]) :
+      harness.abortController.signal,
     harness,
     name: '<root>',
   });

lib/internal/test_runner/reporter/spec.js

@@ -9,7 +9,7 @@ const {
 const assert = require('assert');
 const Transform = require('internal/streams/transform');
 const colors = require('internal/util/colors');
-const { kSubtestsFailed } = require('internal/test_runner/test');
+const { kSubtestsFailed, kTestBailedOut } = require('internal/test_runner/test');
 const { getCoverageReport } = require('internal/test_runner/utils');
 const { relative } = require('path');
 const {
@@ -57,6 +57,7 @@ class SpecReporter extends Transform {
   #handleEvent({ type, data }) {
     switch (type) {
       case 'test:fail':
+        if (data.details?.error?.failureType === kTestBailedOut) break;
         if (data.details?.error?.failureType !== kSubtestsFailed) {
           ArrayPrototypePush(this.#failedTests, data);
         }
@@ -74,6 +75,8 @@ class SpecReporter extends Transform {
       case 'test:coverage':
         return getCoverageReport(indent(data.nesting), data.summary,
                                  reporterUnicodeSymbolMap['test:coverage'], colors.blue, true);
+      case 'test:bail':
+        return `${reporterColorMap[type]}${reporterUnicodeSymbolMap[type]}Bail out!${colors.white}\n`;
     }
   }
   _transform({ type, data }, encoding, callback) {

lib/internal/test_runner/reporter/tap.js

@@ -33,6 +33,7 @@ async function * tapReporter(source) {
   for await (const { type, data } of source) {
     switch (type) {
       case 'test:fail': {
+        if (data.details?.error?.failureType === lazyLoadTest().kTestBailedOut) break;
         yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo);
         const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
         yield reportDetails(data.nesting, data.details, location);
@@ -61,6 +62,9 @@ async function * tapReporter(source) {
       case 'test:coverage':
         yield getCoverageReport(indent(data.nesting), data.summary, '# ', '', true);
         break;
+      case 'test:bail':
+        yield `${indent(data.nesting)}Bail out!\n`;
+        break;
     }
   }
 }

lib/internal/test_runner/reporter/utils.js

@@ -24,6 +24,7 @@ const reporterUnicodeSymbolMap = {
   'test:coverage': '\u2139 ',
   'arrow:right': '\u25B6 ',
   'hyphen:minus': '\uFE63 ',
+  'test:bail': '\u2716 ',
 };
 
 const reporterColorMap = {
@@ -37,6 +38,9 @@ const reporterColorMap = {
   get 'test:diagnostic'() {
     return colors.blue;
   },
+  get 'test:bail'() {
+    return colors.red;
+  },
 };
 
 function indent(nesting) {

lib/internal/test_runner/runner.js

@@ -76,6 +76,7 @@ const {
   kSubtestsFailed,
   kTestCodeFailure,
   kTestTimeoutFailure,
+  kTestBailedOut,
   Test,
 } = require('internal/test_runner/test');
 
@@ -100,8 +101,12 @@ const kFilterArgs = ['--test', '--experimental-test-coverage', '--watch'];
 const kFilterArgValues = ['--test-reporter', '--test-reporter-destination'];
 const kDiagnosticsFilterArgs = ['tests', 'suites', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms'];
 
-const kCanceledTests = new SafeSet()
-  .add(kCancelledByParent).add(kAborted).add(kTestTimeoutFailure);
+const kCanceledTests = new SafeSet([
+  kCancelledByParent,
+  kAborted,
+  kTestTimeoutFailure,
+  kTestBailedOut,
+]);
 
 let kResistStopPropagation;
 
@@ -137,7 +142,8 @@ function getRunArgs(path, { forceExit,
                             only,
                             argv: suppliedArgs,
                             execArgv,
-                            cwd }) {
+                            cwd,
+                            bail }) {
   const argv = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
   if (forceExit === true) {
     ArrayPrototypePush(argv, '--test-force-exit');
@@ -154,6 +160,9 @@ function getRunArgs(path, { forceExit,
   if (only === true) {
     ArrayPrototypePush(argv, '--test-only');
   }
+  if (bail === true) {
+    ArrayPrototypePush(argv, '--test-bail');
+  }
 
   ArrayPrototypePushApply(argv, execArgv);
 
@@ -194,7 +203,14 @@ class FileTest extends Test {
   }
 
   #skipReporting() {
-    return this.#reportedChildren > 0 && (!this.error || this.error.failureType === kSubtestsFailed);
+    return (
+      (
+        this.#reportedChildren > 0 &&
+        (!this.error ||
+          this.error.failureType === kSubtestsFailed ||
+          this.error.failureType === kTestBailedOut
+        )
+      ));
   }
   #checkNestedComment(comment) {
     const firstSpaceIndex = StringPrototypeIndexOf(comment, ' ');
@@ -216,6 +232,12 @@ class FileTest extends Test {
     if (item.data.details?.error) {
       item.data.details.error = deserializeError(item.data.details.error);
     }
+    if (item.type === 'test:bail') {
+      this.root.harness.abortTestTree();
+      this.root.bailed = true;
+      this.bailed = true;
+      return;
+    }
     if (item.type === 'test:pass' || item.type === 'test:fail') {
       item.data.testNumber = isTopLevel ? (this.root.harness.counters.topLevel + 1) : item.data.testNumber;
       countCompletedTest({
@@ -478,6 +500,8 @@ function watchFiles(testFiles, opts) {
       // Reset the topLevel counter
       opts.root.harness.counters.topLevel = 0;
     }
+    // TODO(pmarchini): Reset the bailed flag to rerun the tests.
+    // This must be added only when we add support for bail in watch mode.
     await runningSubtests.get(file);
     runningSubtests.set(file, runTestFile(file, filesWatcher, opts));
   }
@@ -564,6 +588,7 @@ function run(options = kEmptyObject) {
     execArgv = [],
     argv = [],
     cwd = process.cwd(),
+    bail = false,
   } = options;
 
   if (files != null) {
@@ -663,6 +688,15 @@ function run(options = kEmptyObject) {
 
   validateStringArray(argv, 'options.argv');
   validateStringArray(execArgv, 'options.execArgv');
+  validateBoolean(bail, 'options.bail');
+  // TODO(pmarchini): watch mode with bail needs to be implemented
+  if (bail && watch) {
+    throw new ERR_INVALID_ARG_VALUE(
+      'options.bail',
+      watch,
+      'bail is not supported while watch mode is enabled',
+    );
+  }
 
   const rootTestOptions = { __proto__: null, concurrency, timeout, signal };
   const globalOptions = {
@@ -678,6 +712,7 @@ function run(options = kEmptyObject) {
     branchCoverage: branchCoverage,
     functionCoverage: functionCoverage,
     cwd,
+    bail,
   };
   const root = createTestTree(rootTestOptions, globalOptions);
   let testFiles = files ?? createTestFileList(globPatterns, cwd);
@@ -705,6 +740,7 @@ function run(options = kEmptyObject) {
     isolation,
     argv,
     execArgv,
+    bail,
   };
 
   if (isolation === 'process') {