diff --git a/benchmark/ci-test/README.md b/benchmark/ci-test/README.md new file mode 100644 index 0000000000..568421a2f0 --- /dev/null +++ b/benchmark/ci-test/README.md @@ -0,0 +1,30 @@ +# CI Test Benchmark + +`ut run benchmark:ci-test` runs the reusable CI test benchmark harness and writes a Markdown report, a JSON summary, and the raw Vitest JSON reporter output. + +## Environment Setup + +- Use Node.js `>=22.18.0`. +- Make sure the Utoo CLI is available as `ut`. The CI workflow uses `utooland/setup-utoo` before dependency installation. +- Install workspace dependencies from the repository root with `ut install --from pnpm`, matching the CI workflow. +- Run commands from the repository root so workspace paths and `vitest.config.ts` defaults can be detected. +- Keep Redis and MySQL available when benchmarking suites that require them. The CI test job uses Redis 7 on the default Redis port and MySQL 8 with a `test` database; the benchmark harness mirrors the CI Vitest flags but does not start external services. +- For CI-like metadata, set the relevant environment variables before running the command, for example `CI=1`, `GITHUB_SHA`, `RUNNER_OS`, or worker-related `VITEST_*` variables. + +## Usage + +```sh +ut run benchmark:ci-test +ut run benchmark:ci-test -- --coverage +ut run benchmark:ci-test -- --output-dir .tmp/ci-benchmark -- ut execute vitest run packages/extend2/test/index.test.ts +``` + +The default output directory is `benchmark/ci-test/`. Use `--output-dir` for a deterministic path when collecting artifacts. + +## Outputs + +- `report.md`: human-readable benchmark report. +- `report.json`: structured report containing environment, command, wall time, Vitest summary, long-tail file/project durations, and coverage/worker/isolate parameters. +- `vitest-results.json`: raw Vitest JSON reporter output. + +The harness only writes reports for explicit benchmark runs. It does not change required checks or CI gate semantics. To collect reports in GitHub Actions, run the command in a manual or optional job and upload the output directory as an artifact. diff --git a/package.json b/package.json index 7063b24ae8..1d8b94c162 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "pretest": "ut run clean-dist && ut run pretest --workspaces --if-present", "test": "vitest run --bail 1 --retry 2 --testTimeout 20000 --hookTimeout 20000", "test:cov": "ut run test -- --coverage", + "benchmark:ci-test": "node scripts/ci-test-benchmark.js", "preci": "ut run pretest --workspaces --if-present", "ci": "ut run test -- --coverage", "site:dev": "cd site && npm run dev", diff --git a/scripts/ci-test-benchmark.js b/scripts/ci-test-benchmark.js new file mode 100644 index 0000000000..b067468174 --- /dev/null +++ b/scripts/ci-test-benchmark.js @@ -0,0 +1,8 @@ +#!/usr/bin/env node + +import { main } from './ci-test-benchmark/index.js'; + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/ci-test-benchmark/cli.js b/scripts/ci-test-benchmark/cli.js new file mode 100644 index 0000000000..43701f2c5a --- /dev/null +++ b/scripts/ci-test-benchmark/cli.js @@ -0,0 +1,111 @@ +import { DEFAULT_TOP_LIMIT, VITEST_JSON_PLACEHOLDER } from './constants.js'; + +export function printHelp() { + console.log(` +Usage: + ut run benchmark:ci-test + ut run benchmark:ci-test -- --coverage + ut run benchmark:ci-test -- --output-dir .tmp/bench -- ut execute vitest run --maxWorkers=4 + +Options: + --output-dir Directory for report.md, report.json, and raw Vitest JSON. + --name