test_runner: add initial CLI runner

This commit introduces an initial version of a CLI-based
test runner.

PR-URL: https://github.com/nodejs/node/pull/42658
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

doc/api/cli.md

@@ -1052,6 +1052,16 @@ minimum allocation from the secure heap. The minimum value is `2`.
 The maximum value is the lesser of `--secure-heap` or `2147483647`.
 The value given must be a power of two.
 
+### `--test`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Starts the Node.js command line test runner. This flag cannot be combined with
+`--check`, `--eval`, `--interactive`, or the inspector. See the documentation
+on [running tests from the command line][] for more details.
+
 ### `--test-only`
 
 <!-- YAML
@@ -2033,6 +2043,7 @@ $ node --max-old-space-size=1536 index.js
 [jitless]: https://v8.dev/blog/jitless
 [libuv threadpool documentation]: https://docs.libuv.org/en/latest/threadpool.html
 [remote code execution]: https://www.owasp.org/index.php/Code_Injection
+[running tests from the command line]: test.md#running-tests-from-the-command-line
 [security warning]: #warning-binding-inspector-to-a-public-ipport-combination-is-insecure
 [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 [ways that `TZ` is handled in other environments]: https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html

doc/api/test.md

@@ -219,6 +219,67 @@ test('a test that creates asynchronous activity', (t) => {
 });
 ```
 
+## Running tests from the command line
+
+The Node.js test runner can be invoked from the command line by passing the
+[`--test`][] flag:
+
+```bash
+node --test
+```
+
+By default, Node.js will recursively search the current directory for
+JavaScript source files matching a specific naming convention. Matching files
+are executed as test files. More information on the expected test file naming
+convention and behavior can be found in the [test runner execution model][]
+section.
+
+Alternatively, one or more paths can be provided as the final argument(s) to
+the Node.js command, as shown below.
+
+```bash
+node --test test1.js test2.mjs custom_test_dir/
+```
+
+In this example, the test runner will execute the files `test1.js` and
+`test2.mjs`. The test runner will also recursively search the
+`custom_test_dir/` directory for test files to execute.
+
+### Test runner execution model
+
+When searching for test files to execute, the test runner behaves as follows:
+
+* Any files explicitly provided by the user are executed.
+* If the user did not explicitly specify any paths, the current working
+  directory is recursively searched for files as specified in the following
+  steps.
+* `node_modules` directories are skipped unless explicitly provided by the
+  user.
+* If a directory named `test` is encountered, the test runner will search it
+  recursively for all all `.js`, `.cjs`, and `.mjs` files. All of these files
+  are treated as test files, and do not need to match the specific naming
+  convention detailed below. This is to accommodate projects that place all of
+  their tests in a single `test` directory.
+* In all other directories, `.js`, `.cjs`, and `.mjs` files matching the
+  following patterns are treated as test files:
+  * `^test$` - Files whose basename is the string `'test'`. Examples:
+    `test.js`, `test.cjs`, `test.mjs`.
+  * `^test-.+` - Files whose basename starts with the string `'test-'`
+    followed by one or more characters. Examples: `test-example.js`,
+    `test-another-example.mjs`.
+  * `.+[\.\-\_]test$` - Files whose basename ends with `.test`, `-test`, or
+    `_test`, preceded by one or more characters. Examples: `example.test.js`,
+    `example-test.cjs`, `example_test.mjs`.
+  * Other file types understood by Node.js such as `.node` and `.json` are not
+    automatically executed by the test runner, but are supported if explicitly
+    provided on the command line.
+
+Each matching test file is executed in a separate child process. If the child
+process finishes with an exit code of 0, the test is considered passing.
+Otherwise, the test is considered to be a failure. Test files must be
+executable by Node.js, but are not required to use the `node:test` module
+internally.
+
 ## `test([name][, options][, fn])`
 
 <!-- YAML
@@ -368,5 +429,7 @@ behaves in the same fashion as the top level [`test()`][] function.
 
 [TAP]: https://testanything.org/
 [`--test-only`]: cli.md#--test-only
+[`--test`]: cli.md#--test
 [`TestContext`]: #class-testcontext
 [`test()`]: #testname-options-fn
+[test runner execution model]: #test-runner-execution-model

doc/node.1

@@ -381,6 +381,9 @@ the secure heap. The default is 0. The value must be a power of two.
 .It Fl -secure-heap-min Ns = Ns Ar n
 Specify the minimum allocation from the OpenSSL secure heap. The default is 2. The value must be a power of two.
 .
+.It Fl -test
+Starts the Node.js command line test runner.
+.
 .It Fl -test-only
 Configures the test runner to only execute top level tests that have the `only`
 option set.

lib/internal/errors.js

@@ -207,6 +207,16 @@ function isErrorStackTraceLimitWritable() {
     desc.set !== undefined;
 }
 
+function inspectWithNoCustomRetry(obj, options) {
+  const utilInspect = lazyInternalUtilInspect();
+
+  try {
+    return utilInspect.inspect(obj, options);
+  } catch {
+    return utilInspect.inspect(obj, { ...options, customInspect: false });
+  }
+}
+
 // A specialized Error that includes an additional info property with
 // additional information about the error condition.
 // It has the properties present in a UVException but with a custom error
@@ -862,6 +872,7 @@ module.exports = {
   getMessage,
   hideInternalStackFrames,
   hideStackFrames,
+  inspectWithNoCustomRetry,
   isErrorStackTraceLimitWritable,
   isStackOverflowError,
   kEnhanceStackBeforeInspector,
@@ -1549,11 +1560,14 @@ E('ERR_TEST_FAILURE', function(error, failureType) {
   assert(typeof failureType === 'string',
          "The 'failureType' argument must be of type string.");
 
-  const msg = error?.message ?? lazyInternalUtilInspect().inspect(error);
+  let msg = error?.message ?? error;
 
-  this.failureType = error?.failureType ?? failureType;
+  if (typeof msg !== 'string') {
+    msg = inspectWithNoCustomRetry(msg);
+  }
+
+  this.failureType = failureType;
   this.cause = error;
-
   return msg;
 }, Error);
 E('ERR_TLS_CERT_ALTNAME_FORMAT', 'Invalid subject alternative name string',

lib/internal/main/test_runner.js

@@ -0,0 +1,157 @@
+'use strict';
+const {
+  ArrayFrom,
+  ArrayPrototypeFilter,
+  ArrayPrototypeIncludes,
+  ArrayPrototypePush,
+  ArrayPrototypeSlice,
+  ArrayPrototypeSort,
+  Promise,
+  SafeSet,
+} = primordials;
+const {
+  prepareMainThreadExecution,
+} = require('internal/bootstrap/pre_execution');
+const { spawn } = require('child_process');
+const { readdirSync, statSync } = require('fs');
+const console = require('internal/console/global');
+const {
+  codes: {
+    ERR_TEST_FAILURE,
+  },
+} = require('internal/errors');
+const test = require('internal/test_runner/harness');
+const { kSubtestsFailed } = require('internal/test_runner/test');
+const {
+  isSupportedFileType,
+  doesPathMatchFilter,
+} = require('internal/test_runner/utils');
+const { basename, join, resolve } = require('path');
+const kFilterArgs = ['--test'];
+
+prepareMainThreadExecution(false);
+markBootstrapComplete();
+
+// TODO(cjihrig): Replace this with recursive readdir once it lands.
+function processPath(path, testFiles, options) {
+  const stats = statSync(path);
+
+  if (stats.isFile()) {
+    if (options.userSupplied ||
+        (options.underTestDir && isSupportedFileType(path)) ||
+        doesPathMatchFilter(path)) {
+      testFiles.add(path);
+    }
+  } else if (stats.isDirectory()) {
+    const name = basename(path);
+
+    if (!options.userSupplied && name === 'node_modules') {
+      return;
+    }
+
+    // 'test' directories get special treatment. Recursively add all .js,
+    // .cjs, and .mjs files in the 'test' directory.
+    const isTestDir = name === 'test';
+    const { underTestDir } = options;
+    const entries = readdirSync(path);
+
+    if (isTestDir) {
+      options.underTestDir = true;
+    }
+
+    options.userSupplied = false;
+
+    for (let i = 0; i < entries.length; i++) {
+      processPath(join(path, entries[i]), testFiles, options);
+    }
+
+    options.underTestDir = underTestDir;
+  }
+}
+
+function createTestFileList() {
+  const cwd = process.cwd();
+  const hasUserSuppliedPaths = process.argv.length > 1;
+  const testPaths = hasUserSuppliedPaths ?
+    ArrayPrototypeSlice(process.argv, 1) : [cwd];
+  const testFiles = new SafeSet();
+
+  try {
+    for (let i = 0; i < testPaths.length; i++) {
+      const absolutePath = resolve(testPaths[i]);
+
+      processPath(absolutePath, testFiles, { userSupplied: true });
+    }
+  } catch (err) {
+    if (err?.code === 'ENOENT') {
+      console.error(`Could not find '${err.path}'`);
+      process.exit(1);
+    }
+
+    throw err;
+  }
+
+  return ArrayPrototypeSort(ArrayFrom(testFiles));
+}
+
+function filterExecArgv(arg) {
+  return !ArrayPrototypeIncludes(kFilterArgs, arg);
+}
+
+function runTestFile(path) {
+  return test(path, () => {
+    return new Promise((resolve, reject) => {
+      const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
+      ArrayPrototypePush(args, path);
+
+      const child = spawn(process.execPath, args);
+      // TODO(cjihrig): Implement a TAP parser to read the child's stdout
+      // instead of just displaying it all if the child fails.
+      let stdout = '';
+      let stderr = '';
+      let err;
+
+      child.on('error', (error) => {
+        err = error;
+      });
+
+      child.stdout.setEncoding('utf8');
+      child.stderr.setEncoding('utf8');
+
+      child.stdout.on('data', (chunk) => {
+        stdout += chunk;
+      });
+
+      child.stderr.on('data', (chunk) => {
+        stderr += chunk;
+      });
+
+      child.once('exit', (code, signal) => {
+        if (code !== 0 || signal !== null) {
+          if (!err) {
+            err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
+            err.exitCode = code;
+            err.signal = signal;
+            err.stdout = stdout;
+            err.stderr = stderr;
+            // The stack will not be useful since the failures came from tests
+            // in a child process.
+            err.stack = undefined;
+          }
+
+          return reject(err);
+        }
+
+        resolve();
+      });
+    });
+  });
+}
+
+(async function main() {
+  const testFiles = createTestFileList();
+
+  for (let i = 0; i < testFiles.length; i++) {
+    runTestFile(testFiles[i]);
+  }
+})();

lib/internal/test_runner/tap_stream.js

@@ -5,13 +5,13 @@ const {
   ArrayPrototypePush,
   ArrayPrototypeShift,
   ObjectEntries,
-  StringPrototypeReplace,
   StringPrototypeReplaceAll,
   StringPrototypeSplit,
+  RegExpPrototypeSymbolReplace,
 } = primordials;
+const { inspectWithNoCustomRetry } = require('internal/errors');
 const Readable = require('internal/streams/readable');
 const { isError } = require('internal/util');
-const { inspect } = require('internal/util/inspect');
 const kFrameStartRegExp = /^ {4}at /;
 const kLineBreakRegExp = /\n|\r\n/;
 const inspectOptions = { colors: false, breakLength: Infinity };
@@ -74,77 +74,8 @@ class TapStream extends Readable {
   details(indent, duration, error) {
     let details = `${indent}  ---\n`;
 
-    details += `${indent}  duration_ms: ${duration}\n`;
-
-    if (error !== null && typeof error === 'object') {
-      const entries = ObjectEntries(error);
-      const isErrorObj = isError(error);
-
-      for (let i = 0; i < entries.length; i++) {
-        const { 0: key, 1: value } = entries[i];
-
-        if (isError && (key === 'cause' || key === 'code')) {
-          continue;
-        }
-
-        details += `${indent}  ${key}: ${inspect(value, inspectOptions)}\n`;
-      }
-
-      if (isErrorObj) {
-        const { kTestCodeFailure } = lazyLoadTest();
-        const {
-          cause,
-          code,
-          failureType,
-          message,
-          stack,
-        } = error;
-        let errMsg = message ?? '<unknown error>';
-        let errStack = stack;
-        let errCode = code;
-
-        // If the ERR_TEST_FAILURE came from an error provided by user code,
-        // then try to unwrap the original error message and stack.
-        if (code === 'ERR_TEST_FAILURE' && failureType === kTestCodeFailure) {
-          errMsg = cause?.message ?? errMsg;
-          errStack = cause?.stack ?? errStack;
-          errCode = cause?.code ?? errCode;
-        }
-
-        details += `${indent}  error: ${inspect(errMsg, inspectOptions)}\n`;
-
-        if (errCode) {
-          details += `${indent}  code: ${errCode}\n`;
-        }
-
-        if (typeof errStack === 'string') {
-          const frames = [];
-
-          ArrayPrototypeForEach(
-            StringPrototypeSplit(errStack, kLineBreakRegExp),
-            (frame) => {
-              const processed = StringPrototypeReplace(
-                frame, kFrameStartRegExp, ''
-              );
-
-              if (processed.length > 0 && processed.length !== frame.length) {
-                ArrayPrototypePush(frames, processed);
-              }
-            }
-          );
-
-          if (frames.length > 0) {
-            const frameDelimiter = `\n${indent}    `;
-
-            details += `${indent}  stack: |-${frameDelimiter}`;
-            details += `${ArrayPrototypeJoin(frames, `${frameDelimiter}`)}\n`;
-          }
-        }
-      }
-    } else if (error !== null && error !== undefined) {
-      details += `${indent}  error: ${inspect(error, inspectOptions)}\n`;
-    }
-
+    details += jsToYaml(indent, 'duration_ms', duration);
+    details += jsToYaml(indent, null, error);
     details += `${indent}  ...\n`;
     this.#tryPush(details);
   }
@@ -190,4 +121,100 @@ function tapEscape(input) {
   );
 }
 
+function jsToYaml(indent, name, value) {
+  if (value === null || value === undefined) {
+    return '';
+  }
+
+  if (typeof value !== 'object') {
+    const prefix = `${indent}  ${name}: `;
+
+    if (typeof value !== 'string') {
+      return `${prefix}${inspectWithNoCustomRetry(value, inspectOptions)}\n`;
+    }
+
+    const lines = StringPrototypeSplit(value, kLineBreakRegExp);
+
+    if (lines.length === 1) {
+      return `${prefix}${inspectWithNoCustomRetry(value, inspectOptions)}\n`;
+    }
+
+    let str = `${prefix}|-\n`;
+
+    for (let i = 0; i < lines.length; i++) {
+      str += `${indent}    ${lines[i]}\n`;
+    }
+
+    return str;
+  }
+
+  const entries = ObjectEntries(value);
+  const isErrorObj = isError(value);
+  let result = '';
+
+  for (let i = 0; i < entries.length; i++) {
+    const { 0: key, 1: value } = entries[i];
+
+    if (isErrorObj && (key === 'cause' || key === 'code')) {
+      continue;
+    }
+
+    result += jsToYaml(indent, key, value);
+  }
+
+  if (isErrorObj) {
+    const { kTestCodeFailure } = lazyLoadTest();
+    const {
+      cause,
+      code,
+      failureType,
+      message,
+      stack,
+    } = value;
+    let errMsg = message ?? '<unknown error>';
+    let errStack = stack;
+    let errCode = code;
+
+    // If the ERR_TEST_FAILURE came from an error provided by user code,
+    // then try to unwrap the original error message and stack.
+    if (code === 'ERR_TEST_FAILURE' && failureType === kTestCodeFailure) {
+      errMsg = cause?.message ?? errMsg;
+      errStack = cause?.stack ?? errStack;
+      errCode = cause?.code ?? errCode;
+    }
+
+    result += jsToYaml(indent, 'error', errMsg);
+
+    if (errCode) {
+      result += jsToYaml(indent, 'code', errCode);
+    }
+
+    if (typeof errStack === 'string') {
+      const frames = [];
+
+      ArrayPrototypeForEach(
+        StringPrototypeSplit(errStack, kLineBreakRegExp),
+        (frame) => {
+          const processed = RegExpPrototypeSymbolReplace(
+            kFrameStartRegExp, frame, ''
+          );
+
+          if (processed.length > 0 && processed.length !== frame.length) {
+            ArrayPrototypePush(frames, processed);
+          }
+        }
+      );
+
+      if (frames.length > 0) {
+        const frameDelimiter = `\n${indent}    `;
+
+        result += `${indent}  stack: |-${frameDelimiter}`;
+        result += `${ArrayPrototypeJoin(frames, `${frameDelimiter}`)}\n`;
+      }
+    }
+  }
+
+  return result;
+}
+
 module.exports = { TapStream };

lib/internal/test_runner/test.js

@@ -12,12 +12,14 @@ const {
   codes: {
     ERR_TEST_FAILURE,
   },
+  kIsNodeError,
 } = require('internal/errors');
 const { getOptionValue } = require('internal/options');
 const { TapStream } = require('internal/test_runner/tap_stream');
 const { createDeferredPromise } = require('internal/util');
 const { isPromise } = require('internal/util/types');
 const { isUint32 } = require('internal/validators');
+const { cpus } = require('os');
 const { bigint: hrtime } = process.hrtime;
 const kCallbackAndPromisePresent = 'callbackAndPromisePresent';
 const kCancelledByParent = 'cancelledByParent';
@@ -27,7 +29,10 @@ const kSubtestsFailed = 'subtestsFailed';
 const kTestCodeFailure = 'testCodeFailure';
 const kDefaultIndent = '    ';
 const noop = FunctionPrototype;
-const testOnlyFlag = getOptionValue('--test-only');
+const isTestRunner = getOptionValue('--test');
+const testOnlyFlag = !isTestRunner && getOptionValue('--test-only');
+// TODO(cjihrig): Use uv_available_parallelism() once it lands.
+const rootConcurrency = isTestRunner ? cpus().length : 1;
 
 class TestContext {
   #test;
@@ -79,7 +84,7 @@ class Test extends AsyncResource {
     }
 
     if (parent === null) {
-      this.concurrency = 1;
+      this.concurrency = rootConcurrency;
       this.indent = '';
       this.indentString = kDefaultIndent;
       this.only = testOnlyFlag;
@@ -303,12 +308,14 @@ class Test extends AsyncResource {
 
           // If the callback is called a second time, let the user know, but
           // don't let them know more than once.
-          if (calledCount === 2) {
-            throw new ERR_TEST_FAILURE(
-              'callback invoked multiple times',
-              kMultipleCallbackInvocations
-            );
-          } else if (calledCount > 2) {
+          if (calledCount > 1) {
+            if (calledCount === 2) {
+              throw new ERR_TEST_FAILURE(
+                'callback invoked multiple times',
+                kMultipleCallbackInvocations
+              );
+            }
+
             return;
           }
 
@@ -335,7 +342,11 @@ class Test extends AsyncResource {
 
       this.pass();
     } catch (err) {
-      this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure));
+      if (err?.code === 'ERR_TEST_FAILURE' && kIsNodeError in err) {
+        this.fail(err);
+      } else {
+        this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure));
+      }
     }
 
     // Clean up the test. Then, try to report the results and execute any
@@ -436,4 +447,4 @@ class Test extends AsyncResource {
   }
 }
 
-module.exports = { kDefaultIndent, kTestCodeFailure, Test };
+module.exports = { kDefaultIndent, kSubtestsFailed, kTestCodeFailure, Test };

lib/internal/test_runner/utils.js

@@ -0,0 +1,15 @@
+'use strict';
+const { RegExpPrototypeExec } = primordials;
+const { basename } = require('path');
+const kSupportedFileExtensions = /\.[cm]?js$/;
+const kTestFilePattern = /((^test(-.+)?)|(.+[.\-_]test))\.[cm]?js$/;
+
+function doesPathMatchFilter(p) {
+  return RegExpPrototypeExec(kTestFilePattern, basename(p)) !== null;
+}
+
+function isSupportedFileType(p) {
+  return RegExpPrototypeExec(kSupportedFileExtensions, p) !== null;
+}
+
+module.exports = { isSupportedFileType, doesPathMatchFilter };

src/node.cc

@@ -517,6 +517,10 @@ MaybeLocal<Value> StartExecution(Environment* env, StartExecutionCallback cb) {
     return StartExecution(env, "internal/main/check_syntax");
   }
 
+  if (env->options()->test_runner) {
+    return StartExecution(env, "internal/main/test_runner");
+  }
+
   if (!first_argv.empty() && first_argv != "-") {
     return StartExecution(env, "internal/main/run_main_module");
   }

src/node_options.cc

@@ -143,6 +143,24 @@ void EnvironmentOptions::CheckOptions(std::vector<std::string>* errors) {
     errors->push_back("--heap-snapshot-near-heap-limit must not be negative");
   }
 
+  if (test_runner) {
+    if (syntax_check_only) {
+      errors->push_back("either --test or --check can be used, not both");
+    }
+
+    if (has_eval_string) {
+      errors->push_back("either --test or --eval can be used, not both");
+    }
+
+    if (force_repl) {
+      errors->push_back("either --test or --interactive can be used, not both");
+    }
+
+    if (debug_options_.inspector_enabled) {
+      errors->push_back("the inspector cannot be used with --test");
+    }
+  }
+
 #if HAVE_INSPECTOR
   if (!cpu_prof) {
     if (!cpu_prof_name.empty()) {
@@ -498,6 +516,9 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             "write warnings to file instead of stderr",
             &EnvironmentOptions::redirect_warnings,
             kAllowedInEnvironment);
+  AddOption("--test",
+            "launch test runner on startup",
+            &EnvironmentOptions::test_runner);
   AddOption("--test-only",
             "run tests with 'only' option set",
             &EnvironmentOptions::test_only,

src/node_options.h

@@ -148,6 +148,7 @@ class EnvironmentOptions : public Options {
 #endif  // HAVE_INSPECTOR
   std::string redirect_warnings;
   std::string diagnostic_dir;
+  bool test_runner = false;
   bool test_only = false;
   bool test_udp_no_try_send = false;
   bool throw_deprecation = false;

test/fixtures/test-runner/index.js

@@ -0,0 +1,2 @@
+'use strict';
+throw new Error('thrown from index.js');

test/fixtures/test-runner/index.test.js

@@ -0,0 +1,4 @@
+'use strict';
+const test = require('node:test');
+
+test('this should pass');

test/fixtures/test-runner/node_modules/test-nm.js

@@ -0,0 +1,2 @@
+'use strict';
+throw new Error('thrown from node_modules');

test/fixtures/test-runner/random.test.mjs

@@ -0,0 +1,5 @@
+import test from 'node:test';
+
+test('this should fail', () => {
+  throw new Error('this is a failing test');
+});

test/fixtures/test-runner/test/random.cjs

@@ -0,0 +1,4 @@
+'use strict';
+const test = require('node:test');
+
+test('this should pass');

test/message/test_runner_no_refs.out

@@ -3,8 +3,8 @@ TAP version 13
       ---
       duration_ms: *
       failureType: 'cancelledByParent'
-      error: "'test did not finish before its parent and was cancelled'"
-      code: ERR_TEST_FAILURE
+      error: 'test did not finish before its parent and was cancelled'
+      code: 'ERR_TEST_FAILURE'
       stack: |-
         *
       ...
@@ -13,8 +13,8 @@ not ok 1 - does not keep event loop alive
   ---
   duration_ms: *
   failureType: 'cancelledByParent'
-  error: "'test did not finish before its parent and was cancelled'"
-  code: ERR_TEST_FAILURE
+  error: 'test did not finish before its parent and was cancelled'
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...

test/message/test_runner_output.js

@@ -3,6 +3,7 @@
 require('../common');
 const assert = require('node:assert');
 const test = require('node:test');
+const util = require('util');
 
 test('sync pass todo', (t) => {
   t.todo();
@@ -296,3 +297,25 @@ test('only is set but not in only mode', { only: true }, async (t) => {
   t.runOnly(false);
   await t.test('running subtest 4');
 });
+
+test('custom inspect symbol fail', () => {
+  const obj = {
+    [util.inspect.custom]() {
+      return 'customized';
+    },
+    foo: 1
+  };
+
+  throw obj;
+});
+
+test('custom inspect symbol that throws fail', () => {
+  const obj = {
+    [util.inspect.custom]() {
+      throw new Error('bad-inspect');
+    },
+    foo: 1
+  };
+
+  throw obj;
+});

test/message/test_runner_output.out

@@ -12,7 +12,7 @@ not ok 3 - sync fail todo # TODO
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from sync fail todo'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -28,7 +28,7 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from sync fail todo with message'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -59,7 +59,7 @@ not ok 8 - sync throw fail
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from sync throw fail'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -83,7 +83,7 @@ not ok 11 - async throw fail
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from async throw fail'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -99,7 +99,7 @@ not ok 12 - async skip fail # SKIP
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from async throw fail'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -114,8 +114,12 @@ not ok 13 - async assertion fail
   ---
   duration_ms: *
   failureType: 'testCodeFailure'
-  error: 'Expected values to be strictly equal:\n\ntrue !== false\n'
-  code: ERR_ASSERTION
+  error: |-
+    Expected values to be strictly equal:
+
+    true !== false
+
+  code: 'ERR_ASSERTION'
   stack: |-
     *
     *
@@ -135,7 +139,7 @@ not ok 15 - reject fail
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'rejected from reject fail'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -171,7 +175,7 @@ ok 20 - immediate resolve pass
       duration_ms: *
       failureType: 'testCodeFailure'
       error: 'thrown from subtest sync throw fail'
-      code: ERR_TEST_FAILURE
+      code: 'ERR_TEST_FAILURE'
       stack: |-
         *
         *
@@ -190,15 +194,15 @@ not ok 21 - subtest sync throw fail
   ---
   duration_ms: *
   failureType: 'subtestsFailed'
-  error: "'1 subtest failed'"
-  code: ERR_TEST_FAILURE
+  error: '1 subtest failed'
+  code: 'ERR_TEST_FAILURE'
   ...
 not ok 22 - sync throw non-error fail
   ---
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'Symbol(thrown symbol from sync throw non-error fail)'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   ...
     ok 1 - level 1a
       ---
@@ -225,8 +229,8 @@ ok 23 - level 0a
       ---
       duration_ms: *
       failureType: 'cancelledByParent'
-      error: "'test did not finish before its parent and was cancelled'"
-      code: ERR_TEST_FAILURE
+      error: 'test did not finish before its parent and was cancelled'
+      code: 'ERR_TEST_FAILURE'
       ...
         ok 1 - ++short running
           ---
@@ -242,8 +246,8 @@ not ok 24 - top level
   ---
   duration_ms: *
   failureType: 'subtestsFailed'
-  error: "'1 subtest failed'"
-  code: ERR_TEST_FAILURE
+  error: '1 subtest failed'
+  code: 'ERR_TEST_FAILURE'
   ...
 ok 25 - invalid subtest - pass but subtest fails
   ---
@@ -262,7 +266,7 @@ not ok 28 - sync skip option is false fail
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'this should be executed'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -330,7 +334,7 @@ not ok 42 - callback fail
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'callback failure'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -351,15 +355,15 @@ not ok 46 - callback also returns a Promise
   ---
   duration_ms: *
   failureType: 'callbackAndPromisePresent'
-  error: "'passed a callback but also returned a Promise'"
-  code: ERR_TEST_FAILURE
+  error: 'passed a callback but also returned a Promise'
+  code: 'ERR_TEST_FAILURE'
   ...
 not ok 47 - callback throw
   ---
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from callback throw'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -373,8 +377,11 @@ not ok 48 - callback called twice
   ---
   duration_ms: *
   failureType: 'multipleCallbackInvocations'
-  error: "'callback invoked multiple times'"
-  code: ERR_TEST_FAILURE
+  error: 'callback invoked multiple times'
+  code: 'ERR_TEST_FAILURE'
+  stack: |-
+    *
+    *
   ...
 ok 49 - callback called twice in different ticks
   ---
@@ -383,9 +390,9 @@ ok 49 - callback called twice in different ticks
 not ok 50 - callback called twice in future tick
   ---
   duration_ms: *
-  failureType: 'multipleCallbackInvocations'
-  error: "'callback invoked multiple times'"
-  code: ERR_TEST_FAILURE
+  failureType: 'uncaughtException'
+  error: 'callback invoked multiple times'
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...
@@ -394,7 +401,7 @@ not ok 51 - callback async throw
   duration_ms: *
   failureType: 'uncaughtException'
   error: 'thrown from callback async throw'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...
@@ -423,25 +430,43 @@ ok 53 - only is set but not in only mode
   ---
   duration_ms: *
   ...
-not ok 54 - invalid subtest fail
+not ok 54 - custom inspect symbol fail
+  ---
+  duration_ms: *
+  failureType: 'testCodeFailure'
+  error: 'customized'
+  code: 'ERR_TEST_FAILURE'
+  ...
+not ok 55 - custom inspect symbol that throws fail
+  ---
+  duration_ms: *
+  failureType: 'testCodeFailure'
+  error: |-
+    {
+      foo: 1,
+      [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]]
+    }
+  code: 'ERR_TEST_FAILURE'
+  ...
+not ok 56 - invalid subtest fail
   ---
   duration_ms: *
   failureType: 'parentAlreadyFinished'
-  error: "'test could not be started because its parent finished'"
-  code: ERR_TEST_FAILURE
+  error: 'test could not be started because its parent finished'
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...
-1..54
+1..56
 # Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
-# Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: 'callback invoked multiple times'" and would have caused the test to fail, but instead triggered an uncaughtException event.
+# Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event.
-# tests 54
+# tests 56
 # pass 24
-# fail 15
+# fail 17
 # skipped 10
 # todo 5
 # duration_ms *

test/message/test_runner_unresolved_promise.out

@@ -7,8 +7,8 @@ not ok 2 - never resolving promise
   ---
   duration_ms: *
   failureType: 'cancelledByParent'
-  error: "'test did not finish before its parent and was cancelled'"
-  code: ERR_TEST_FAILURE
+  error: 'test did not finish before its parent and was cancelled'
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...
@@ -16,8 +16,8 @@ not ok 3 - fail
   ---
   duration_ms: *
   failureType: 'cancelledByParent'
-  error: "'test did not finish before its parent and was cancelled'"
-  code: ERR_TEST_FAILURE
+  error: 'test did not finish before its parent and was cancelled'
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...

test/parallel/test-runner-cli.js

@@ -0,0 +1,107 @@
+'use strict';
+require('../common');
+const assert = require('assert');
+const { spawnSync } = require('child_process');
+const { join } = require('path');
+const fixtures = require('../common/fixtures');
+const testFixtures = fixtures.path('test-runner');
+
+{
+  // File not found.
+  const args = ['--test', 'a-random-file-that-does-not-exist.js'];
+  const child = spawnSync(process.execPath, args);
+
+  assert.strictEqual(child.status, 1);
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.stdout.toString(), '');
+  assert(/^Could not find/.test(child.stderr.toString()));
+}
+
+{
+  // Default behavior. node_modules is ignored. Files that don't match the
+  // pattern are ignored except in test/ directories.
+  const args = ['--test', testFixtures];
+  const child = spawnSync(process.execPath, args);
+
+  assert.strictEqual(child.status, 1);
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.stderr.toString(), '');
+  const stdout = child.stdout.toString();
+  assert(/ok 1 - .+index\.test\.js/.test(stdout));
+  assert(/not ok 2 - .+random\.test\.mjs/.test(stdout));
+  assert(/not ok 1 - this should fail/.test(stdout));
+  assert(/ok 3 - .+subdir.+subdir_test\.js/.test(stdout));
+  assert(/ok 4 - .+random\.cjs/.test(stdout));
+}
+
+{
+  // User specified files that don't match the pattern are still run.
+  const args = ['--test', testFixtures, join(testFixtures, 'index.js')];
+  const child = spawnSync(process.execPath, args);
+
+  assert.strictEqual(child.status, 1);
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.stderr.toString(), '');
+  const stdout = child.stdout.toString();
+  assert(/not ok 1 - .+index\.js/.test(stdout));
+  assert(/ok 2 - .+index\.test\.js/.test(stdout));
+  assert(/not ok 3 - .+random\.test\.mjs/.test(stdout));
+  assert(/not ok 1 - this should fail/.test(stdout));
+  assert(/ok 4 - .+subdir.+subdir_test\.js/.test(stdout));
+}
+
+{
+  // Searches node_modules if specified.
+  const args = ['--test', join(testFixtures, 'node_modules')];
+  const child = spawnSync(process.execPath, args);
+
+  assert.strictEqual(child.status, 1);
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.stderr.toString(), '');
+  const stdout = child.stdout.toString();
+  assert(/not ok 1 - .+test-nm\.js/.test(stdout));
+}
+
+{
+  // The current directory is used by default.
+  const args = ['--test'];
+  const options = { cwd: testFixtures };
+  const child = spawnSync(process.execPath, args, options);
+
+  assert.strictEqual(child.status, 1);
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.stderr.toString(), '');
+  const stdout = child.stdout.toString();
+  assert(/ok 1 - .+index\.test\.js/.test(stdout));
+  assert(/not ok 2 - .+random\.test\.mjs/.test(stdout));
+  assert(/not ok 1 - this should fail/.test(stdout));
+  assert(/ok 3 - .+subdir.+subdir_test\.js/.test(stdout));
+  assert(/ok 4 - .+random\.cjs/.test(stdout));
+}
+
+{
+  // Flags that cannot be combined with --test.
+  const flags = [
+    ['--check', '--test'],
+    ['--interactive', '--test'],
+    ['--eval', 'console.log("should not print")', '--test'],
+    ['--print', 'console.log("should not print")', '--test'],
+  ];
+
+  if (process.features.inspector) {
+    flags.push(
+      ['--inspect', '--test'],
+      ['--inspect-brk', '--test'],
+    );
+  }
+
+  flags.forEach((args) => {
+    const child = spawnSync(process.execPath, args);
+
+    assert.notStrictEqual(child.status, 0);
+    assert.strictEqual(child.signal, null);
+    assert.strictEqual(child.stdout.toString(), '');
+    const stderr = child.stderr.toString();
+    assert(/--test/.test(stderr));
+  });
+}

test/parallel/test-runner-test-filter.js

@@ -0,0 +1,42 @@
+// Flags: --expose-internals
+'use strict';
+require('../common');
+const assert = require('assert');
+const { doesPathMatchFilter } = require('internal/test_runner/utils');
+
+// Paths expected to match
+[
+  'test.js',
+  'test.cjs',
+  'test.mjs',
+  'test-foo.js',
+  'test-foo.cjs',
+  'test-foo.mjs',
+  'foo.test.js',
+  'foo.test.cjs',
+  'foo.test.mjs',
+  'foo-test.js',
+  'foo-test.cjs',
+  'foo-test.mjs',
+  'foo_test.js',
+  'foo_test.cjs',
+  'foo_test.mjs',
+].forEach((p) => {
+  assert.strictEqual(doesPathMatchFilter(p), true);
+});
+
+// Paths expected not to match
+[
+  'test',
+  'test.djs',
+  'test.cs',
+  'test.mj',
+  'foo.js',
+  'test-foo.sj',
+  'test.foo.js',
+  'test_foo.js',
+  'testfoo.js',
+  'foo-test1.mjs',
+].forEach((p) => {
+  assert.strictEqual(doesPathMatchFilter(p), false);
+});