test_runner: add flaky option to retry on failure

Add a `flaky` option that re-runs a failing test until it passes,
intended for tests with unavoidable nondeterminism. Setting
`flaky: true` retries up to 20 times; `flaky: <positive integer>`
sets an explicit retry budget. The option is accepted on tests and
suites and via the it.flaky/test.flaky/describe.flaky/suite.flaky
shorthands; a test-case value overrides an inherited suite value
(nearest wins), and `flaky: false` opts a test out.

Only the final attempt is observable: intermediate failures emit no
test:fail, no per-test diagnostics, and nothing on the node.test
error channel. Each result carries a new `retryCount` field on the
test:pass, test:fail, and test:complete events (the number of retries
performed, `undefined` for non-flaky tests), reporters print a
`# FLAKY` directive, and the run summary gains a `flaky` counter.

beforeEach/afterEach re-run on every attempt while before/after run
once, so per-attempt state is reset and retries do not leak state.
An externally aborted test and an expectFailure are not retried. A
flaky test whose timeout is exhausted is reported as a failure rather
than cancelled.

Co-authored-by: vespa7 <98526766+vespa7@users.noreply.github.com>
Signed-off-by: sangwook <rewq5991@gmail.com>

Author: Han5991

doc/api/test.md

@@ -1868,6 +1868,16 @@ added:
 Shorthand for marking a suite as `TODO`. This is the same as
 [`suite([name], { todo: true }[, fn])`][suite options].
 
+## `suite.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+Shorthand for marking a suite as flaky. This is the same as
+[`suite([name], { flaky: true }[, fn])`][suite options].
+
 ## `suite.only([name][, options][, fn])`
 
 <!-- YAML
@@ -1939,6 +1949,14 @@ changes:
   * `todo` {boolean|string} If truthy, the test marked as `TODO`. If a string
     is provided, that string is displayed in the test results as the reason why
     the test is `TODO`. **Default:** `false`.
+  * `flaky` {boolean|number} If `true`, the test (or, for a suite, each of its
+    test-cases) is retried until it passes, up to a default of 20 retries. If a
+    positive integer is provided, it is retried up to that many times. A
+    non-positive or non-integer value throws. Each retry re-runs the test's
+    `beforeEach` and `afterEach` hooks. A test-case's own value overrides one
+    inherited from its suite; `flaky: false` opts out. Retries are intended for
+    tests that fail intermittently due to transient conditions; be aware that
+    non-idempotent state may leak between attempts. **Default:** `false`.
   * `timeout` {number} A number of milliseconds the test will fail after.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `Infinity`.
@@ -2000,6 +2018,16 @@ same as [`test([name], { todo: true }[, fn])`][it options].
 Shorthand for marking a test as `only`,
 same as [`test([name], { only: true }[, fn])`][it options].
 
+## `test.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+Shorthand for marking a test as flaky,
+same as [`test([name], { flaky: true }[, fn])`][it options].
+
 ## `describe([name][, options][, fn])`
 
 Alias for [`suite()`][].
@@ -2027,6 +2055,16 @@ added:
 Shorthand for marking a suite as `only`. This is the same as
 [`describe([name], { only: true }[, fn])`][describe options].
 
+## `describe.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+Shorthand for marking a suite as flaky. This is the same as
+[`describe([name], { flaky: true }[, fn])`][describe options].
+
 ## `it([name][, options][, fn])`
 
 <!-- YAML
@@ -2066,6 +2104,16 @@ added:
 Shorthand for marking a test as `only`,
 same as [`it([name], { only: true }[, fn])`][it options].
 
+## `it.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+Shorthand for marking a test as flaky,
+same as [`it([name], { flaky: true }[, fn])`][it options].
+
 ## `before([fn][, options])`
 
 <!-- YAML
@@ -3539,6 +3587,9 @@ Emitted when code coverage is enabled and all tests have completed.
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+  * `retryCount` {number|undefined} The number of retries performed for a test
+    marked `flaky` (`0` if it passed on the first attempt). `undefined` for
+    tests that are not flaky.
 
 Emitted when a test completes its execution.
 This event is not emitted in the same order as the tests are
@@ -3647,6 +3698,9 @@ Emitted when a test is enqueued for execution.
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+  * `retryCount` {number|undefined} The number of retries performed for a test
+    marked `flaky` (`0` if it passed on the first attempt). `undefined` for
+    tests that are not flaky.
 
 Emitted when a test fails.
 This event is guaranteed to be emitted in the same order as the tests are
@@ -3712,6 +3766,9 @@ since the parent runner only knows about file-level tests. When using
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+  * `retryCount` {number|undefined} The number of retries performed for a test
+    marked `flaky` (`0` if it passed on the first attempt). `undefined` for
+    tests that are not flaky.
 
 Emitted when a test passes.
 This event is guaranteed to be emitted in the same order as the tests are
@@ -4527,6 +4584,14 @@ changes:
   * `todo` {boolean|string} If truthy, the test marked as `TODO`. If a string
     is provided, that string is displayed in the test results as the reason why
     the test is `TODO`. **Default:** `false`.
+  * `flaky` {boolean|number} If `true`, the test (or, for a suite, each of its
+    test-cases) is retried until it passes, up to a default of 20 retries. If a
+    positive integer is provided, it is retried up to that many times. A
+    non-positive or non-integer value throws. Each retry re-runs the test's
+    `beforeEach` and `afterEach` hooks. A test-case's own value overrides one
+    inherited from its suite; `flaky: false` opts out. Retries are intended for
+    tests that fail intermittently due to transient conditions; be aware that
+    non-idempotent state may leak between attempts. **Default:** `false`.
   * `timeout` {number} A number of milliseconds the test will fail after.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `Infinity`.

lib/internal/test_runner/harness.js

@@ -63,6 +63,7 @@ function createTestTree(rootTestOptions, globalOptions) {
         failed: 0,
         passed: 0,
         cancelled: 0,
+        flaky: 0,
         skipped: 0,
         todo: 0,
         topLevel: 0,
@@ -406,7 +407,7 @@ function runInParentContext(Factory) {
 
     return run(name, options, fn, overrides);
   };
-  ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => {
+  ArrayPrototypeForEach(['expectFailure', 'flaky', 'skip', 'todo', 'only'], (keyword) => {
     test[keyword] = (name, options, fn) => {
       const overrides = {
         __proto__: null,

lib/internal/test_runner/reporter/dot.js

@@ -12,7 +12,11 @@ module.exports = async function* dot(source) {
   const failedTests = [];
   for await (const { type, data } of source) {
     if (type === 'test:pass') {
-      yield `${colors.green}.${colors.reset}`;
+      // A flaky test that needed retries is shown in yellow to distinguish it
+      // from a clean pass, without using 'F' (which reads as a failure).
+      yield data.retryCount > 0 ?
+        `${colors.yellow}.${colors.reset}` :
+        `${colors.green}.${colors.reset}`;
     }
     if (type === 'test:fail') {
       yield `${colors.red}X${colors.reset}`;

lib/internal/test_runner/reporter/junit.js

@@ -131,6 +131,16 @@ module.exports = async function* junitReporter(source) {
               attrs: { __proto__: null, type: 'todo', message: event.data.todo },
             });
           }
+          if (event.data.retryCount > 0) {
+            ArrayPrototypePush(currentTest.children, {
+              __proto__: null, nesting: event.data.nesting + 1, tag: 'properties',
+              attrs: { __proto__: null },
+              children: [{
+                __proto__: null, nesting: event.data.nesting + 2, tag: 'property',
+                attrs: { __proto__: null, name: 'flaky', value: `${event.data.retryCount} retries` },
+              }],
+            });
+          }
           if (event.type === 'test:fail') {
             const error = event.data.details?.error;
             currentTest.children.push({

lib/internal/test_runner/reporter/tap.js

@@ -33,12 +33,14 @@ async function * tapReporter(source) {
   for await (const { type, data } of source) {
     switch (type) {
       case 'test:fail': {
-        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure);
+        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name,
+                         data.skip, data.todo, data.expectFailure, data.retryCount);
         const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
         yield reportDetails(data.nesting, data.details, location);
         break;
       } case 'test:pass':
-        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure);
+        yield reportTest(data.nesting, data.testNumber, 'ok', data.name,
+                         data.skip, data.todo, data.expectFailure, data.retryCount);
         yield reportDetails(data.nesting, data.details, null);
         break;
       case 'test:plan':
@@ -75,7 +77,7 @@ async function * tapReporter(source) {
   }
 }
 
-function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) {
+function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, retryCount) {
   let line = `${indent(nesting)}${status} ${testNumber}`;
 
   if (name) {
@@ -88,6 +90,11 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure
     line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
   } else if (expectFailure !== undefined) {
     line += ` # EXPECTED FAILURE${typeof expectFailure === 'string' ? ` ${tapEscape(expectFailure)}` : ''}`;
+  } else if (retryCount > 0) {
+    const word = retryCount === 1 ? 're-try' : 're-tries';
+    line += status === 'ok' ?
+      ` # FLAKY ${retryCount} ${word}` :
+      ` # FLAKY failed after ${retryCount} ${word}`;
   }
 
   line += '\n';

lib/internal/test_runner/reporter/utils.js

@@ -71,7 +71,7 @@ function formatError(error, indent) {
 function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') {
   let color = reporterColorMap[type] ?? colors.white;
   let symbol = reporterUnicodeSymbolMap[type] ?? ' ';
-  const { skip, todo, expectFailure } = data;
+  const { skip, todo, expectFailure, retryCount } = data;
   const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : '';
   const replayed = data.details?.passed_on_attempt !== undefined ?
     ` ${colors.gray}(passed on attempt ${data.details.passed_on_attempt})${colors.white}` :
@@ -90,6 +90,11 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde
     }
   } else if (expectFailure !== undefined) {
     title += ` # EXPECTED FAILURE`;
+  } else if (retryCount > 0) {
+    const word = retryCount === 1 ? 're-try' : 're-tries';
+    title += type === 'test:fail' ?
+      ` # FLAKY failed after ${retryCount} ${word}` :
+      ` # FLAKY ${retryCount} ${word}`;
   }
 
   const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : '';

lib/internal/test_runner/runner.js

@@ -313,7 +313,13 @@ class FileTest extends Test {
         skipped: item.data.skip !== undefined,
         isTodo: item.data.todo !== undefined,
         passed: item.type === 'test:pass',
-        cancelled: kCanceledTests.has(item.data.details?.error?.failureType),
+        // A retried timeout that exhausted (retryCount > 0) is a failure, not a
+        // cancellation only an un-retried timeout/abort stays cancelled.
+        cancelled: kCanceledTests.has(item.data.details?.error?.failureType) &&
+          !(item.data.retryCount > 0),
+        // retryCount is present (even 0) only for flaky-marked tests, so it lets
+        // the parent count flaky tests across the process-isolation IPC boundary.
+        flakyRetries: item.data.retryCount !== undefined ? 1 : 0,
         nesting: item.data.nesting,
         reportedType: item.data.details?.type,
       }, this.root.harness);