chore(test): add ci benchmark harness

benchmark/ci-test/README.md

@@ -0,0 +1,30 @@
+# CI Test Benchmark
+
+`ut run benchmark:ci-test` runs the reusable CI test benchmark harness and writes a Markdown report, a JSON summary, and the raw Vitest JSON reporter output.
+
+## Environment Setup
+
+- Use Node.js `>=22.18.0`.
+- Make sure the Utoo CLI is available as `ut`. The CI workflow uses `utooland/setup-utoo` before dependency installation.
+- Install workspace dependencies from the repository root with `ut install --from pnpm`, matching the CI workflow.
+- Run commands from the repository root so workspace paths and `vitest.config.ts` defaults can be detected.
+- Keep Redis and MySQL available when benchmarking suites that require them. The CI test job uses Redis 7 on the default Redis port and MySQL 8 with a `test` database; the benchmark harness mirrors the CI Vitest flags but does not start external services.
+- For CI-like metadata, set the relevant environment variables before running the command, for example `CI=1`, `GITHUB_SHA`, `RUNNER_OS`, or worker-related `VITEST_*` variables.
+
+## Usage
+
+```sh
+ut run benchmark:ci-test
+ut run benchmark:ci-test -- --coverage
+ut run benchmark:ci-test -- --output-dir .tmp/ci-benchmark -- ut execute vitest run packages/extend2/test/index.test.ts
+```
+
+The default output directory is `benchmark/ci-test/<timestamp>`. Use `--output-dir` for a deterministic path when collecting artifacts.
+
+## Outputs
+
+- `report.md`: human-readable benchmark report.
+- `report.json`: structured report containing environment, command, wall time, Vitest summary, long-tail file/project durations, and coverage/worker/isolate parameters.
+- `vitest-results.json`: raw Vitest JSON reporter output.
+
+The harness only writes reports for explicit benchmark runs. It does not change required checks or CI gate semantics. To collect reports in GitHub Actions, run the command in a manual or optional job and upload the output directory as an artifact.

package.json

@@ -23,6 +23,7 @@
     "pretest": "ut run clean-dist && ut run pretest --workspaces --if-present",
     "test": "vitest run --bail 1 --retry 2 --testTimeout 20000 --hookTimeout 20000",
     "test:cov": "ut run test -- --coverage",
+    "benchmark:ci-test": "node scripts/ci-test-benchmark.js",
     "preci": "ut run pretest --workspaces --if-present",
     "ci": "ut run test -- --coverage",
     "site:dev": "cd site && npm run dev",

scripts/ci-test-benchmark.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+import { main } from './ci-test-benchmark/index.js';
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

scripts/ci-test-benchmark/cli.js

@@ -0,0 +1,111 @@
+import { DEFAULT_TOP_LIMIT, VITEST_JSON_PLACEHOLDER } from './constants.js';
+
+export function printHelp() {
+  console.log(`
+Usage:
+  ut run benchmark:ci-test
+  ut run benchmark:ci-test -- --coverage
+  ut run benchmark:ci-test -- --output-dir .tmp/bench -- ut execute vitest run --maxWorkers=4
+
+Options:
+  --output-dir <dir>                 Directory for report.md, report.json, and raw Vitest JSON.
+  --name <label>                     Human-readable benchmark label.
+  --top <n>                          Number of long-tail files/projects to include. Default: ${DEFAULT_TOP_LIMIT}.
+  --coverage                         Append --coverage to the default Vitest command.
+  --no-append-vitest-json-reporter   Do not append --reporter=json/--outputFile to the command.
+  --dry-run                          Generate reports without executing the test command.
+  --help                             Show this help.
+
+Custom command:
+  Arguments after -- replace the default command. The script appends Vitest JSON reporter args by default.
+  Use ${VITEST_JSON_PLACEHOLDER} inside a custom command arg if the output path must be embedded manually.
+`);
+}
+
+export function parseArgs(argv) {
+  argv = normalizePackageManagerArgv(argv);
+  const options = {
+    appendVitestJsonReporter: true,
+    coverage: false,
+    dryRun: false,
+    name: 'CI test benchmark',
+    outputDir: '',
+    top: DEFAULT_TOP_LIMIT,
+  };
+  const command = [];
+
+  for (let index = 0; index < argv.length; index++) {
+    const arg = argv[index];
+    if (arg === '--') {
+      command.push(...argv.slice(index + 1));
+      break;
+    }
+    if (arg === '--help' || arg === '-h') {
+      options.help = true;
+      continue;
+    }
+    if (arg === '--coverage') {
+      options.coverage = true;
+      continue;
+    }
+    if (arg === '--dry-run') {
+      options.dryRun = true;
+      continue;
+    }
+    if (arg === '--no-append-vitest-json-reporter') {
+      options.appendVitestJsonReporter = false;
+      continue;
+    }
+    if (arg === '--output-dir') {
+      options.outputDir = readOptionValue(argv, ++index, arg);
+      continue;
+    }
+    if (arg.startsWith('--output-dir=')) {
+      options.outputDir = arg.slice('--output-dir='.length);
+      continue;
+    }
+    if (arg === '--name') {
+      options.name = readOptionValue(argv, ++index, arg);
+      continue;
+    }
+    if (arg.startsWith('--name=')) {
+      options.name = arg.slice('--name='.length);
+      continue;
+    }
+    if (arg === '--top') {
+      options.top = parsePositiveInteger(readOptionValue(argv, ++index, arg), arg);
+      continue;
+    }
+    if (arg.startsWith('--top=')) {
+      options.top = parsePositiveInteger(arg.slice('--top='.length), '--top');
+      continue;
+    }
+
+    throw new Error(`Unknown option: ${arg}`);
+  }
+
+  return { command, options };
+}
+
+function normalizePackageManagerArgv(argv) {
+  if (argv[0] === '--' && argv.length > 1 && argv[1].startsWith('--')) {
+    return argv.slice(1);
+  }
+  return argv;
+}
+
+function readOptionValue(argv, index, optionName) {
+  const value = argv[index];
+  if (!value || value.startsWith('--')) {
+    throw new Error(`${optionName} requires a value`);
+  }
+  return value;
+}
+
+function parsePositiveInteger(value, optionName) {
+  const parsed = Number.parseInt(value, 10);
+  if (!Number.isInteger(parsed) || parsed < 1) {
+    throw new Error(`${optionName} must be a positive integer`);
+  }
+  return parsed;
+}

scripts/ci-test-benchmark/command.js

@@ -0,0 +1,92 @@
+import { DEFAULT_COMMAND, VITEST_JSON_PLACEHOLDER } from './constants.js';
+
+const BOOLEAN_COMMAND_OPTIONS = new Set(['--isolate', '--no-isolate']);
+
+export function buildCommand(commandArgs, options, vitestJsonPath) {
+  const command = commandArgs.length > 0 ? [...commandArgs] : [...DEFAULT_COMMAND];
+  if (options.coverage && commandArgs.length === 0) {
+    command.push('--coverage');
+  }
+
+  const replaced = command.map((arg) => arg.replaceAll(VITEST_JSON_PLACEHOLDER, vitestJsonPath));
+  if (!options.appendVitestJsonReporter) {
+    return replaced;
+  }
+
+  const reporterArgs = [];
+  if (!hasJsonReporter(replaced)) {
+    reporterArgs.push('--reporter=json');
+  }
+  if (!hasVitestOutputFile(replaced)) {
+    reporterArgs.push(`--outputFile=${vitestJsonPath}`);
+  }
+  return reporterArgs.length === 0 ? replaced : [...replaced, ...reporterArgs];
+}
+
+export function extractCommandParameters(command) {
+  return {
+    coverage: command.some((arg) => arg === '--coverage' || arg.startsWith('--coverage=')),
+    hookTimeout: collectOptionValues(command, ['--hookTimeout']),
+    isolate: collectOptionValues(command, ['--isolate', '--no-isolate']),
+    pool: collectOptionValues(command, ['--pool']),
+    reporter: collectOptionValues(command, ['--reporter']),
+    retry: collectOptionValues(command, ['--retry']),
+    testTimeout: collectOptionValues(command, ['--testTimeout']),
+    workers: collectOptionValues(command, [
+      '--maxWorkers',
+      '--minWorkers',
+      '--poolOptions.threads.maxThreads',
+      '--poolOptions.threads.minThreads',
+      '--poolOptions.forks.maxForks',
+      '--poolOptions.forks.minForks',
+    ]),
+  };
+}
+
+export function shellJoin(command) {
+  return command.map((arg) => (arg.includes(' ') ? JSON.stringify(arg) : arg)).join(' ');
+}
+
+function hasJsonReporter(command) {
+  for (let index = 0; index < command.length; index++) {
+    const arg = command[index];
+    if (arg === '--reporter=json') {
+      return true;
+    }
+    if (arg === '--reporter' && command[index + 1] === 'json') {
+      return true;
+    }
+  }
+  return false;
+}
+
+function hasVitestOutputFile(command) {
+  return command.some(
+    (arg) => arg === '--outputFile' || arg.startsWith('--outputFile=') || arg.startsWith('--outputFile.'),
+  );
+}
+
+function collectOptionValues(command, names) {
+  const values = [];
+  for (let index = 0; index < command.length; index++) {
+    const arg = command[index];
+    for (const name of names) {
+      if (arg === name) {
+        values.push({ name, value: getSeparatedOptionValue(name, command[index + 1]) });
+      } else if (arg.startsWith(`${name}=`)) {
+        values.push({ name, value: arg.slice(name.length + 1) });
+      }
+    }
+  }
+  return values;
+}
+
+function getSeparatedOptionValue(name, next) {
+  if (name.startsWith('--no-')) {
+    return false;
+  }
+  if (BOOLEAN_COMMAND_OPTIONS.has(name)) {
+    return true;
+  }
+  return next && !next.startsWith('--') ? next : true;
+}

scripts/ci-test-benchmark/constants.js

@@ -0,0 +1,22 @@
+import path from 'node:path';
+
+export const DEFAULT_TOP_LIMIT = 20;
+export const DEFAULT_OUTPUT_ROOT = path.join('benchmark', 'ci-test');
+export const VITEST_JSON_FILENAME = 'vitest-results.json';
+export const REPORT_JSON_FILENAME = 'report.json';
+export const REPORT_MARKDOWN_FILENAME = 'report.md';
+export const VITEST_JSON_PLACEHOLDER = '{vitestJson}';
+export const DEFAULT_COMMAND = [
+  'ut',
+  'execute',
+  'vitest',
+  'run',
+  '--bail',
+  '1',
+  '--retry',
+  '2',
+  '--testTimeout',
+  '20000',
+  '--hookTimeout',
+  '20000',
+];

scripts/ci-test-benchmark/environment.js

@@ -0,0 +1,84 @@
+import os from 'node:os';
+import path from 'node:path';
+
+import { extractCommandParameters } from './command.js';
+import { readJsonIfExists, readTextIfExists } from './fs.js';
+
+export async function collectEnvironment(command) {
+  const [packageJson, vitestConfig] = await Promise.all([
+    readJsonIfExists(path.resolve(process.cwd(), 'package.json')),
+    readVitestConfigDefaults(path.resolve(process.cwd(), 'vitest.config.ts')),
+  ]);
+  const cpus = os.cpus();
+
+  return {
+    arch: os.arch(),
+    ci: Boolean(process.env.CI),
+    commandParameters: extractCommandParameters(command),
+    cpuCount: cpus.length,
+    cpuModel: cpus[0]?.model ?? 'unknown',
+    cwd: process.cwd(),
+    env: pickEnv([
+      'CI',
+      'GITHUB_ACTIONS',
+      'GITHUB_EVENT_NAME',
+      'GITHUB_REF',
+      'GITHUB_RUN_ATTEMPT',
+      'GITHUB_RUN_ID',
+      'GITHUB_SHA',
+      'NODE_ENV',
+      'RUNNER_ARCH',
+      'RUNNER_NAME',
+      'RUNNER_OS',
+      'VITEST_MAX_THREADS',
+      'VITEST_MIN_THREADS',
+      'VITEST_POOL_ID',
+    ]),
+    loadavg: os.loadavg(),
+    node: process.version,
+    packageManager: packageJson?.packageManager ?? null,
+    platform: os.platform(),
+    release: os.release(),
+    totalMemoryBytes: os.totalmem(),
+    vitestConfig,
+  };
+}
+
+function pickEnv(names) {
+  const values = {};
+  for (const name of names) {
+    if (process.env[name] !== undefined) {
+      values[name] = process.env[name];
+    }
+  }
+  return values;
+}
+
+async function readVitestConfigDefaults(configPath) {
+  const source = await readTextIfExists(configPath);
+  if (!source) {
+    return null;
+  }
+  return {
+    coverageProvider: matchStringProperty(source, 'provider'),
+    isolate: matchBooleanProperty(source, 'isolate'),
+    pool: matchStringProperty(source, 'pool'),
+  };
+}
+
+function matchStringProperty(source, property) {
+  const match = new RegExp(`^\\s*(?!//|/\\*)${escapeRegExp(property)}:\\s*['"]([^'"]+)['"]`, 'm').exec(source);
+  return match?.[1] ?? null;
+}
+
+function matchBooleanProperty(source, property) {
+  const match = new RegExp(`^\\s*(?!//|/\\*)${escapeRegExp(property)}:\\s*(true|false)`, 'm').exec(source);
+  if (!match) {
+    return null;
+  }
+  return match[1] === 'true';
+}
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}