test_runner: expose expectFailure message

This change exposes the expectFailure message in the test runner and
adds edge cases for expectFailure ambiguity.

PR-URL: https://github.com/nodejs/node/pull/61563
Fixes: https://github.com/nodejs/node/issues/61570
Reviewed-By: Jordan Harband <ljharb@gmail.com>
Reviewed-By: Aviv Keller <me@aviv.sh>
Reviewed-By: Jacob Smith <jacob@frende.me>

doc/api/test.md

@@ -266,11 +266,12 @@ added:
  - v24.14.0
 -->
 
-This flips the pass/fail reporting for a specific test or suite: A flagged test/test-case must throw
-in order to "pass"; a test/test-case that does not throw, fails.
+This flips the pass/fail reporting for a specific test or suite: a flagged test
+case must throw in order to pass, and a flagged test case that does not throw
+fails.
 
-In the following, `doTheThing()` returns _currently_ `false` (`false` does not equal `true`, causing
-`strictEqual` to throw, so the test-case passes).
+In each of the following, `doTheThing()` fails to return `true`, but since the
+tests are flagged `expectFailure`, they pass.
 
 ```js
 it.expectFailure('should do the thing', () => {
@@ -280,6 +281,50 @@ it.expectFailure('should do the thing', () => {
 it('should do the thing', { expectFailure: true }, () => {
   assert.strictEqual(doTheThing(), true);
 });
+
+it('should do the thing', { expectFailure: 'feature not implemented' }, () => {
+  assert.strictEqual(doTheThing(), true);
+});
+```
+
+If the value of `expectFailure` is a
+[<RegExp>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp) |
+[<Function>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function) |
+[<Object>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object) |
+[<Error>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error),
+the tests will pass only if they throw a matching value.
+See [`assert.throws`][] for how each value type is handled.
+
+Each of the following tests fails _despite_ being flagged `expectFailure`
+because the failure does not match the specific **expected** failure.
+
+```js
+it('fails because regex does not match', {
+  expectFailure: /expected message/,
+}, () => {
+  throw new Error('different message');
+});
+
+it('fails because object matcher does not match', {
+  expectFailure: { code: 'ERR_EXPECTED' },
+}, () => {
+  const err = new Error('boom');
+  err.code = 'ERR_ACTUAL';
+  throw err;
+});
+```
+
+To supply both a reason and specific error for `expectFailure`, use `{ label, match }`.
+
+```js
+it('should fail with specific error and reason', {
+  expectFailure: {
+    label: 'reason for failure',
+    match: /error message/,
+  },
+}, () => {
+  assert.strictEqual(doTheThing(), true);
+});
 ```
 
 `skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo`
@@ -1687,6 +1732,18 @@ changes:
     thread. If `false`, only one test runs at a time.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `false`.
+  * `expectFailure` {boolean|string|RegExp|Function|Object|Error} If truthy, the
+    test is expected to fail. If a non-empty string is provided, that string is displayed
+    in the test results as the reason why the test is expected to fail. If a
+    [<RegExp>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp),
+    [<Function>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function),
+    [<Object>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object), or
+    [<Error>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error)
+    is provided directly (without wrapping in `{ match: … }`), the test passes
+    only if the thrown error matches, following the behavior of
+    [`assert.throws`][]. To provide both a reason and validation, pass an object
+    with `label` (string) and `match` (RegExp, Function, Object, or Error).
+    **Default:** `false`.
   * `only` {boolean} If truthy, and the test context is configured to run
     `only` tests, then this test will be run. Otherwise, the test is skipped.
     **Default:** `false`.
@@ -4184,6 +4241,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`NODE_V8_COVERAGE`]: cli.md#node_v8_coveragedir
 [`SuiteContext`]: #class-suitecontext
 [`TestContext`]: #class-testcontext
+[`assert.throws`]: assert.md#assertthrowsfn-error-message
 [`context.diagnostic`]: #contextdiagnosticmessage
 [`context.skip`]: #contextskipmessage
 [`context.todo`]: #contexttodomessage

lib/internal/test_runner/reporter/tap.js

@@ -87,7 +87,7 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure
   } else if (todo !== undefined) {
     line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
   } else if (expectFailure !== undefined) {
-    line += ' # EXPECTED FAILURE';
+    line += ` # EXPECTED FAILURE${typeof expectFailure === 'string' ? ` ${tapEscape(expectFailure)}` : ''}`;
   }
 
   line += '\n';

lib/internal/test_runner/test.js

@@ -1,5 +1,6 @@
 'use strict';
 const {
+  ArrayPrototypeEvery,
   ArrayPrototypePush,
   ArrayPrototypePushApply,
   ArrayPrototypeShift,
@@ -13,6 +14,7 @@ const {
   MathMax,
   Number,
   NumberPrototypeToFixed,
+  ObjectKeys,
   ObjectSeal,
   Promise,
   PromisePrototypeThen,
@@ -40,6 +42,7 @@ const {
   AbortError,
   codes: {
     ERR_INVALID_ARG_TYPE,
+    ERR_INVALID_ARG_VALUE,
     ERR_TEST_FAILURE,
   },
 } = require('internal/errors');
@@ -56,7 +59,8 @@ const {
   once: runOnce,
   setOwnProperty,
 } = require('internal/util');
-const { isPromise } = require('internal/util/types');
+const assert = require('assert');
+const { isPromise, isRegExp } = require('internal/util/types');
 const {
   validateAbortSignal,
   validateFunction,
@@ -492,6 +496,39 @@ class SuiteContext {
   }
 }
 
+function parseExpectFailure(expectFailure) {
+  if (expectFailure === undefined || expectFailure === false) {
+    return false;
+  }
+
+  if (typeof expectFailure === 'string') {
+    return { __proto__: null, label: expectFailure, match: undefined };
+  }
+
+  if (typeof expectFailure === 'function' || isRegExp(expectFailure)) {
+    return { __proto__: null, label: undefined, match: expectFailure };
+  }
+
+  if (typeof expectFailure !== 'object') {
+    return { __proto__: null, label: undefined, match: undefined };
+  }
+
+  const keys = ObjectKeys(expectFailure);
+  if (keys.length === 0) {
+    throw new ERR_INVALID_ARG_VALUE('options.expectFailure', expectFailure, 'must not be an empty object');
+  }
+
+  if (ArrayPrototypeEvery(keys, (k) => k === 'match' || k === 'label')) {
+    return {
+      __proto__: null,
+      label: expectFailure.label,
+      match: expectFailure.match,
+    };
+  }
+
+  return { __proto__: null, label: undefined, match: expectFailure };
+}
+
 class Test extends AsyncResource {
   reportedType = 'test';
   abortController;
@@ -641,7 +678,7 @@ class Test extends AsyncResource {
     this.plan = null;
     this.expectedAssertions = plan;
     this.cancelled = false;
-    this.expectFailure = expectFailure !== undefined && expectFailure !== false;
+    this.expectFailure = parseExpectFailure(expectFailure) || this.parent?.expectFailure;
     this.skipped = skip !== undefined && skip !== false;
     this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo;
     this.startTime = null;
@@ -955,7 +992,30 @@ class Test extends AsyncResource {
       return;
     }
 
-    if (this.expectFailure === true) {
+    if (this.expectFailure) {
+      if (typeof this.expectFailure === 'object' &&
+          this.expectFailure.match !== undefined) {
+        const { match: validation } = this.expectFailure;
+        try {
+          const errorToCheck = (
+            err?.code === 'ERR_TEST_FAILURE' &&
+            err?.failureType === kTestCodeFailure &&
+            err.cause
+          ) ?
+            err.cause :
+            err;
+          // eslint-disable-next-line no-restricted-syntax
+          assert.throws(() => { throw errorToCheck; }, validation);
+        } catch (e) {
+          this.passed = false;
+          this.error = new ERR_TEST_FAILURE(
+            'The test failed, but the error did not match the expected validation',
+            kTestCodeFailure,
+          );
+          this.error.cause = e;
+          return;
+        }
+      }
       this.passed = true;
     } else {
       this.passed = false;
@@ -965,7 +1025,7 @@ class Test extends AsyncResource {
   }
 
   pass() {
-    if (this.error == null && this.expectFailure === true && !this.skipped) {
+    if (this.error == null && this.expectFailure && !this.skipped) {
       this.passed = false;
       this.error = new ERR_TEST_FAILURE(
         'test was expected to fail but passed',
@@ -977,6 +1037,20 @@ class Test extends AsyncResource {
       return;
     }
 
+    if (this.skipped || this.isTodo) {
+      this.passed = true;
+      return;
+    }
+
+    if (this.expectFailure) {
+      this.passed = false;
+      this.error = new ERR_TEST_FAILURE(
+        'Test passed but was expected to fail',
+        kTestCodeFailure,
+      );
+      return;
+    }
+
     this.passed = true;
   }
 
@@ -1366,7 +1440,10 @@ class Test extends AsyncResource {
     } else if (this.isTodo) {
       directive = this.reporter.getTodo(this.message);
     } else if (this.expectFailure) {
-      directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure
+      const message = typeof this.expectFailure === 'object' ?
+        this.expectFailure.label :
+        this.expectFailure;
+      directive = this.reporter.getXFail(message);
     }
 
     if (this.reportedType) {

test/parallel/test-runner-expect-error-but-pass.js

@@ -8,10 +8,9 @@ if (!process.env.NODE_TEST_CONTEXT) {
 
   stream.on('test:pass', common.mustNotCall());
   stream.on('test:fail', common.mustCall((event) => {
-    assert.strictEqual(event.expectFailure, true);
     assert.strictEqual(event.details.error.code, 'ERR_TEST_FAILURE');
     assert.strictEqual(event.details.error.failureType, 'expectedFailure');
-    assert.strictEqual(event.details.error.cause, 'test was expected to fail but passed');
+    assert.strictEqual(event.details.error.message, 'test was expected to fail but passed');
   }, 1));
 } else {
   test('passing test', { expectFailure: true }, () => {});

test/parallel/test-runner-xfail.js

@@ -0,0 +1,259 @@
+'use strict';
+const common = require('../common');
+const { test, suite } = require('node:test');
+const { spawn } = require('child_process');
+const assert = require('node:assert');
+
+if (process.env.CHILD_PROCESS === 'true') {
+  test('fail with message string', { expectFailure: 'reason string' }, () => {
+    assert.fail('boom');
+  });
+
+  test('fail with label object', { expectFailure: { label: 'reason object' } }, () => {
+    assert.fail('boom');
+  });
+
+  test('fail with match regex', { expectFailure: { match: /boom/ } }, () => {
+    assert.fail('boom');
+  });
+
+  test('fail with match object', { expectFailure: { match: { message: 'boom' } } }, () => {
+    assert.fail('boom');
+  });
+
+  test('fail with match class', { expectFailure: { match: assert.AssertionError } }, () => {
+    assert.fail('boom');
+  });
+
+  test('fail with match error (wrong error)', { expectFailure: { match: /bang/ } }, () => {
+    assert.fail('boom'); // Should result in real failure because error doesn't match
+  });
+
+  test('unexpected pass', { expectFailure: true }, () => {
+    // Should result in real failure because it didn't fail
+  });
+
+  test('fail with empty string', { expectFailure: '' }, () => {
+    assert.fail('boom');
+  });
+
+  // 1. Matcher: RegExp
+  test('fails with regex matcher', { expectFailure: /expected error/ }, () => {
+    throw new Error('this is the expected error');
+  });
+
+  test('fails with regex matcher (mismatch)', { expectFailure: /expected error/ }, () => {
+    throw new Error('wrong error'); // Should fail the test
+  });
+
+  // 2. Matcher: Class
+  test('fails with class matcher', { expectFailure: RangeError }, () => {
+    throw new RangeError('out of bounds');
+  });
+
+  test('fails with class matcher (mismatch)', { expectFailure: RangeError }, () => {
+    throw new TypeError('wrong type'); // Should fail the test
+  });
+
+  // 3. Matcher: Function
+  test('fails with function matcher', {
+    expectFailure: (err) => err.code === 'ERR_FUNCTION',
+  }, () => {
+    const err = new Error('boom');
+    err.code = 'ERR_FUNCTION';
+    throw err;
+  });
+
+  test('fails with function matcher (mismatch)', {
+    expectFailure: (err) => err.code === 'ERR_FUNCTION',
+  }, () => {
+    const err = new Error('boom');
+    err.code = 'ERR_WRONG';
+    throw err; // Should fail
+  });
+
+  // 4. Matcher: Object (Properties)
+  test('fails with object matcher', { expectFailure: { code: 'ERR_TEST' } }, () => {
+    const err = new Error('boom');
+    err.code = 'ERR_TEST';
+    throw err;
+  });
+
+  test('fails with object matcher (mismatch)', { expectFailure: { code: 'ERR_TEST' } }, () => {
+    const err = new Error('boom');
+    err.code = 'ERR_WRONG';
+    throw err; // Should fail
+  });
+
+  // 5. Configuration Object: Reason + Validation
+  test('fails with config object (label + match)', {
+    expectFailure: {
+      label: 'Bug #124',
+      match: /boom/
+    }
+  }, () => {
+    throw new Error('boom');
+  });
+
+  test('fails with config object (label only)', {
+    expectFailure: { label: 'Bug #125' }
+  }, () => {
+    throw new Error('boom');
+  });
+
+  test('fails with config object (match only)', {
+    expectFailure: { match: /boom/ }
+  }, () => {
+    throw new Error('boom');
+  });
+
+  // 6. Edge Case: Empty Object (Should throw ERR_INVALID_ARG_VALUE during creation)
+  test('invalid empty object throws', () => {
+    // eslint-disable-next-line no-restricted-syntax
+    assert.throws(() => test('invalid empty object', { expectFailure: {} }, () => {}));
+  });
+
+  // 7. Primitives and Truthiness
+  test('fails with boolean true', { expectFailure: true }, () => {
+    throw new Error('any error');
+  });
+
+  // 8. Unexpected Pass (Enhanced)
+  test('unexpected pass (reason string)', { expectFailure: 'should fail' }, () => {
+    // Pass
+  });
+
+  test('unexpected pass (matcher)', { expectFailure: /boom/ }, () => {
+    // Pass
+  });
+
+  // 9. Test level expectFailure with subtests
+  test('test level expectFailure with subtests', { expectFailure: true }, async (t) => {
+    await t.test('subtest should pass if it throws', () => {
+      throw new Error('fail');
+    });
+
+    await t.test('subtest should fail if it does not throw', () => {
+      // This should fail the test because the suite expects failure,
+      // and this subtest inherited that expectation but didn't throw.
+    });
+  });
+
+  test('test level expectFailure with subtests and matcher', { expectFailure: /suite error/ }, async (t) => {
+    await t.test('subtest match suite expectation', () => {
+      throw new Error('suite error');
+    });
+  });
+
+  // 10. Suite level expectFailure
+  suite('suite level expectFailure', { expectFailure: true }, () => {
+    test('suite subtest should pass if it throws', () => {
+      throw new Error('fail');
+    });
+
+    test('suite subtest should fail if it does not throw', () => {
+      // This should fail the test because the suite expects failure,
+      // and this subtest inherited that expectation but didn't throw.
+    });
+  });
+
+  suite('suite level expectFailure with matcher', { expectFailure: /suite error/ }, () => {
+    test('suite subtest match suite expectation', () => {
+      throw new Error('suite error');
+    });
+  });
+
+  test('correct config with label and match object', {
+    expectFailure: {
+      label: 'Bug #124',
+      match: { code: 'ERR_TEST' }
+    }
+  }, () => {
+    const err = new Error('boom');
+    err.code = 'ERR_TEST';
+    throw err;
+  });
+
+  test('fails with ambiguous config (mixed props)', {
+    expectFailure: {
+      code: 'ERR_TEST',
+      label: 'Bug #124',
+    }
+  }, () => {
+    const err = new Error('boom');
+    err.code = 'ERR_TEST';
+    throw err;
+  });
+
+  test('fails with ambiguous config (mixed props with match)', {
+    expectFailure: {
+      code: 'ERR_TEST',
+      match: /message/
+    }
+  }, () => {
+    const err = new Error('message');
+    err.code = 'ERR_TEST';
+    throw err;
+  });
+
+} else {
+  const child = spawn(process.execPath, ['--test-reporter', 'tap', __filename], {
+    env: { ...process.env, CHILD_PROCESS: 'true' },
+    stdio: 'pipe',
+  });
+
+  let stdout = '';
+  child.stdout.setEncoding('utf8');
+  child.stdout.on('data', (chunk) => { stdout += chunk; });
+
+  child.on('close', common.mustCall((code) => {
+    // We expect exit code 1 because 'unexpected pass' and 'wrong error' should fail the test run
+    assert.strictEqual(code, 1);
+
+    assert.match(stdout, /ok \d+ - fail with message string # EXPECTED FAILURE reason string/);
+    assert.match(stdout, /ok \d+ - fail with label object # EXPECTED FAILURE reason object/);
+    assert.match(stdout, /ok \d+ - fail with match regex # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - fail with match object # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - fail with match class # EXPECTED FAILURE/);
+    assert.match(stdout, /not ok \d+ - fail with match error \(wrong error\) # EXPECTED FAILURE/);
+    assert.match(stdout, /not ok \d+ - unexpected pass # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - fail with empty string # EXPECTED FAILURE/);
+
+    // New tests verification
+    assert.match(stdout, /ok \d+ - fails with regex matcher # EXPECTED FAILURE/);
+    assert.match(stdout, /not ok \d+ - fails with regex matcher \(mismatch\) # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - fails with class matcher # EXPECTED FAILURE/);
+    assert.match(stdout, /not ok \d+ - fails with class matcher \(mismatch\) # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - fails with function matcher # EXPECTED FAILURE/);
+    assert.match(stdout, /not ok \d+ - fails with function matcher \(mismatch\) # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - fails with object matcher # EXPECTED FAILURE/);
+    assert.match(stdout, /not ok \d+ - fails with object matcher \(mismatch\) # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - fails with config object \(label \+ match\) # EXPECTED FAILURE Bug \\#124/);
+    assert.match(stdout, /ok \d+ - fails with config object \(label only\) # EXPECTED FAILURE Bug \\#125/);
+    assert.match(stdout, /ok \d+ - fails with config object \(match only\) # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - fails with boolean true # EXPECTED FAILURE/);
+    assert.match(stdout, /not ok \d+ - unexpected pass \(reason string\) # EXPECTED FAILURE should fail/);
+    assert.match(stdout, /not ok \d+ - unexpected pass \(matcher\) # EXPECTED FAILURE/);
+
+    // Suite inheritance verification
+    assert.match(stdout, /ok \d+ - test level expectFailure with subtests # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - subtest should pass if it throws # EXPECTED FAILURE/);
+    assert.match(stdout, /not ok \d+ - subtest should fail if it does not throw # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - test level expectFailure with subtests and matcher # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - subtest match suite expectation # EXPECTED FAILURE/);
+
+    // Suite level expectFailure verification
+    assert.match(stdout, /ok \d+ - suite level expectFailure # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - suite subtest should pass if it throws # EXPECTED FAILURE/);
+    assert.match(stdout, /not ok \d+ - suite subtest should fail if it does not throw # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - suite level expectFailure with matcher # EXPECTED FAILURE/);
+    assert.match(stdout, /ok \d+ - suite subtest match suite expectation # EXPECTED FAILURE/);
+
+    // Empty object error
+    assert.match(stdout, /ok \d+ - invalid empty object throws/);
+    assert.match(stdout, /ok \d+ - correct config with label and match object # EXPECTED FAILURE Bug \\#124/);
+    assert.match(stdout, /not ok \d+ - fails with ambiguous config \(mixed props\) # EXPECTED FAILURE/);
+    assert.match(stdout, /not ok \d+ - fails with ambiguous config \(mixed props with match\) # EXPECTED FAILURE/);
+
+  }));
+}