diff --git a/.github/actions/resolve-benchmark-versions/action.yml b/.github/actions/resolve-benchmark-versions/action.yml new file mode 100644 index 0000000000000..a776666cd612f --- /dev/null +++ b/.github/actions/resolve-benchmark-versions/action.yml @@ -0,0 +1,10 @@ +name: Resolve benchmark versions +description: Resolve stable and prerelease Compose and Kotlin versions from Maven Central, plus the latest dev Compose from JetBrains. +runs: + using: composite + steps: + - name: Resolve versions + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + run: python3 "$ACTION_PATH/resolve_versions.py" diff --git a/.github/actions/resolve-benchmark-versions/resolve_versions.py b/.github/actions/resolve-benchmark-versions/resolve_versions.py new file mode 100644 index 0000000000000..31cd1dd955763 --- /dev/null +++ b/.github/actions/resolve-benchmark-versions/resolve_versions.py @@ -0,0 +1,68 @@ +import json +import os +import re +import urllib.request +import xml.etree.ElementTree as ET +from pathlib import Path + +def latest_versions(group, artifact): + url = ( + "https://repo.maven.apache.org/maven2/" + f"{group.replace('.', '/')}/{artifact}/maven-metadata.xml" + ) + with urllib.request.urlopen(url, timeout=60) as response: + metadata = ET.parse(response) + stable, unstable = [], [] + for node in metadata.findall("./versioning/versions/version"): + version = node.text + match = re.fullmatch( + r"(\d+)\.(\d+)\.(\d+)(?:-(alpha|beta|rc)(\d*)?)?", + version, re.IGNORECASE, + ) + if not match: + continue + major, minor, patch, stage, number = match.groups() + key = (int(major), int(minor), int(patch)) + if stage is None: + stable.append((key, version)) + else: + rank = {"alpha": 0, "beta": 1, "rc": 2}[stage.lower()] + unstable.append((key + (rank, int(number or 0)), version)) + if not stable or not unstable: + raise RuntimeError(f"Missing stable or prerelease version for {artifact}") + return max(stable)[1], max(unstable)[1] + +def latest_dev_compose_version(): + url = ( + "https://packages.jetbrains.team/maven/p/cmp/dev/" + "org/jetbrains/compose/compose-gradle-plugin/maven-metadata.xml" + ) + with urllib.request.urlopen(url, timeout=60) as response: + metadata = ET.parse(response) + version = metadata.findtext("./versioning/latest") + available = {node.text for node in metadata.findall("./versioning/versions/version")} + if not version or version not in available: + raise RuntimeError("Missing latest Compose version in the dev repository metadata") + return version + + +compose = latest_versions("org.jetbrains.compose", "compose-gradle-plugin") +dev_compose = latest_dev_compose_version() +selected_compose = tuple(dict.fromkeys((*compose, dev_compose, "local"))) +kotlin = latest_versions("org.jetbrains.kotlin", "kotlin-gradle-plugin") +versions = { + "compose": {"stable": compose[0], "unstable": compose[1], "dev": dev_compose, "local": "local"}, + "kotlin": {"stable": kotlin[0], "unstable": kotlin[1]}, +} +results = Path(os.environ["RUNNER_TEMP"]) / "benchmark-results" +results.mkdir(parents=True, exist_ok=True) +(results / "versions.json").write_text(json.dumps(versions, indent=2) + "\n") +with open(os.environ["GITHUB_ENV"], "a") as env: + env.write(f"COMPOSE_VERSIONS={','.join(selected_compose)}\n") + env.write(f"KOTLIN_VERSIONS={','.join(kotlin)}\n") +with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary: + summary.write("## Benchmark versions\n\n") + summary.write(f"- Compose: {', '.join(selected_compose)}\n") + summary.write(f"- Kotlin: {', '.join(kotlin)}\n") + summary.write("- Local Compose uses the core checkout's own Kotlin version.\n") +print(json.dumps(versions, indent=2)) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000000..8e10f9a30a090 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,177 @@ +name: Benchmarks + +on: + workflow_dispatch: + push: + branches: + - benchmarks + +permissions: + contents: read + +jobs: + jvm: + name: JVM benchmarks + runs-on: macos-26-xlarge + timeout-minutes: 360 + + defaults: + run: + shell: bash + working-directory: compose/mpp/benchmarks + + steps: + - name: Checkout Repository + uses: actions/checkout@v5 + + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Resolve latest benchmark versions + uses: ./.github/actions/resolve-benchmark-versions + + - name: Build launcher + run: ./gradlew :benchmarks-launcher:installDist --console=plain + + - name: Run JVM benchmarks + run: | + mkdir -p "$RUNNER_TEMP/benchmark-results" + ./benchmarks-launcher/build/install/benchmarks-launcher/bin/benchmarks-launcher \ + --target jvm \ + --compose-version "$COMPOSE_VERSIONS" \ + --kotlin-version "$KOTLIN_VERSIONS" \ + --compose-core-path "$GITHUB_WORKSPACE" \ + --tests AnimatedVisibility LazyGrid MultipleComponents TextLayout CanvasDrawing \ + --repeat 3 \ + --output-dir "$RUNNER_TEMP/benchmark-results" + + # GitHub packages the whole session, including index.html, as one ZIP artifact. + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: jvm-benchmarks-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/benchmark-results/ + if-no-files-found: warn + retention-days: 30 + + web: + name: Web benchmarks (${{ matrix.target }}) + runs-on: macos-26-xlarge + timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + target: [js, wasmJs] + + defaults: + run: + shell: bash + working-directory: compose/mpp/benchmarks + + steps: + - name: Checkout Repository + uses: actions/checkout@v5 + + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Check Chrome + run: '"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --version' + + - name: Resolve latest benchmark versions + uses: ./.github/actions/resolve-benchmark-versions + + - name: Build launcher + run: ./gradlew :benchmarks-launcher:installDist --console=plain + + - name: Run benchmarks in Chrome + env: + BENCHMARK_TARGET: ${{ matrix.target }} + # Use the runner's GPU through ANGLE's Metal backend instead of SwiftShader. + BENCHMARK_CHROME_ARGUMENTS: '["--use-gl=angle", "--use-angle=metal"]' + run: | + ./benchmarks-launcher/build/install/benchmarks-launcher/bin/benchmarks-launcher \ + --target "$BENCHMARK_TARGET" \ + --compose-version "$COMPOSE_VERSIONS" \ + --kotlin-version "$KOTLIN_VERSIONS" \ + --compose-core-path "$GITHUB_WORKSPACE" \ + --browser "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \ + --tests AnimatedVisibility LazyGrid MultipleComponents TextLayout CanvasDrawing \ + --repeat 3 \ + --output-dir "$RUNNER_TEMP/benchmark-results" + + # GitHub packages the whole session, including index.html, as one ZIP artifact. + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.target }}-benchmarks-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/benchmark-results/ + if-no-files-found: warn + retention-days: 30 + + ios: + name: iOS benchmarks + runs-on: macos-26-xlarge + timeout-minutes: 360 + + defaults: + run: + shell: bash + working-directory: compose/mpp/benchmarks + + steps: + - name: Checkout Repository + uses: actions/checkout@v5 + + - name: Setup iOS benchmark environment + id: ios + uses: ./.github/actions/setup-ios-instrumented-test-environment + with: + xcode-path: /Applications/Xcode_26.5.app + xcode-cache-key-suffix: '26.5' + simulator-device: iPhone 17 + simulator-os: '26.5' + + - name: Resolve latest benchmark versions + uses: ./.github/actions/resolve-benchmark-versions + + - name: Build launcher + run: ./gradlew :benchmarks-launcher:installDist --console=plain + + - name: Run iOS benchmarks + env: + SIMULATOR_ID: ${{ steps.ios.outputs.simulator-id }} + run: | + ./benchmarks-launcher/build/install/benchmarks-launcher/bin/benchmarks-launcher \ + --target ios \ + --simulator "$SIMULATOR_ID" \ + --compose-version "$COMPOSE_VERSIONS" \ + --kotlin-version "$KOTLIN_VERSIONS" \ + --compose-core-path "$GITHUB_WORKSPACE" \ + --tests AnimatedVisibility LazyGrid MultipleComponents TextLayout CanvasDrawing \ + --repeat 3 \ + --output-dir "$RUNNER_TEMP/benchmark-results" + + # GitHub packages the whole session, including index.html, as one ZIP artifact. + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: ios-benchmarks-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/benchmark-results/ + if-no-files-found: warn + retention-days: 30 diff --git a/compose/mpp/benchmarks/.gitignore b/compose/mpp/benchmarks/.gitignore new file mode 100644 index 0000000000000..09cc89828e371 --- /dev/null +++ b/compose/mpp/benchmarks/.gitignore @@ -0,0 +1,2 @@ +/launcher/ +yarn.lock \ No newline at end of file diff --git a/compose/mpp/benchmarks/README.md b/compose/mpp/benchmarks/README.md new file mode 100644 index 0000000000000..7903c5ac5a272 --- /dev/null +++ b/compose/mpp/benchmarks/README.md @@ -0,0 +1,258 @@ +# Compose benchmarks + +A project with Compose Multiplatform benchmarks for measuring application startup and rendering +performance. This is a standalone Gradle build, separate from the core build. + +Run all commands below from `compose/mpp/benchmarks`. The launcher runs on macOS and Linux +and requires JDK 17 or newer. Apple targets require macOS with Xcode. + +## Modules + +- `benchmarks-app` — an application with Compose benchmark cases. +- `benchmarks-runtime` — runs measurements and sends results. +- `benchmarks-skiko-init-legacy` / `benchmarks-skiko-init-current` — the app selects one and directly calls its `configureBenchmarkSkiko()`. +- `benchmarks-data` — shared report models. +- `benchmarks-report` — a JVM library that generates a static HTML report from a session directory. +- `benchmarks-collector` — receives reports over HTTP and saves them as JSON files. +- `benchmarks-launcher` — a console application that builds and runs benchmarks sequentially. +- `benchmarks-ios-app` — an iOS host application for benchmarks; a separate Xcode project. + +## Building the launcher + +Build the launcher and copy the distribution: + +```bash +./gradlew :benchmarks-launcher:installDist +mkdir -p launcher +cp -R benchmarks-launcher/build/install/benchmarks-launcher/. ./launcher/ +``` + +See all available options: + +```bash +./launcher/bin/benchmarks-launcher --help +``` + +## Running benchmarks + +For each version pair, the launcher visits every selected target and, for JS/WasmJS, every selected +browser. For each configuration it builds the collector and the release application, starts the +collector, and runs tests sequentially. Each test run gets a separate UUID. Native targets use a +separate application process per run. Web targets use Selenium WebDriver: rendering runs reuse one +browser session and tab per configuration, navigating to a new URL for each run. After receiving a +report, the tab navigates to `about:blank` to stop the previous workload. All web rendering runs finish +before startup runs; the rendering browser is closed, and every startup run launches and closes a +fresh browser session. It then moves to the next version pair. Builds always run; Gradle can reuse +unchanged task outputs. + +The matrix continues after test errors, timeouts, or a failed configuration build. If any configuration +fails, the launcher returns a nonzero exit code. On completion or interruption, it stops the processes +it started and keeps collected results. + +Main options: + +- `--target` — required list of targets, separated by spaces or commas: `jvm`, `macos` (ARM64), + `js`, `wasmJs`, or `ios` (ARM64 Simulator). Duplicate entries are ignored. +- `--compose-version` — a list of published Compose versions and/or `local`; defaults to one catalog + version unless a core checkout is configured. +- `--kotlin-version` — a list of Kotlin and matching Compose Compiler versions; defaults to one catalog version. +- `--tests` — test names separated by spaces or commas; defaults to all tests. +- `--modes` — measurement modes; defaults to all four: + - `startup` — startup time to the first frames; + - `realFrames` — frame measurements in a real UI; + - `averageFrame` — average offscreen rendering measurements; + - `fps` — per-frame offscreen rendering measurements. +- `--repeat` — repetitions of each test/mode pair; default: `1`. +- `--timeout` — timeout per test in seconds, excluding build time; default: `300`. +- `--port` — a free port for the collector; default: `8090`. +- `--output-dir` — the base directory for results; it can already exist. Each invocation creates a new + timestamped subdirectory for the whole version matrix. + +Browser targets require `--browser` with a list of executable paths, commands on `PATH`, or macOS +`.app` bundles, separated by spaces or commas. Quote paths containing spaces. Duplicate entries are +ignored. Each selected browser runs on each selected web target; non-web targets ignore this list. +Chromium, Firefox, and Safari are detected from the executable name. ChromeDriver/GeckoDriver launch +the selected Chromium/Firefox executable with a temporary profile for each session. Rendering runs +share the profile and its caches; there is no separate preload run. JS/Wasm and other static resources +are cacheable, while HTML is loaded on every navigation. Every startup run gets a fresh profile in +Chromium/Firefox. Safari uses an automation session managed by `safaridriver`; its caches are not +explicitly cleared. Safari must be closed before the series, and the launcher also quits the Safari +application after ending a session to ensure startup runs restart the browser. + +Selenium Manager resolves ChromeDriver/GeckoDriver automatically, downloading them when necessary. +The first run may require network access; cache `~/.cache/selenium` on CI. Drivers can also be +provisioned explicitly using Selenium's `SE_CHROMEDRIVER` / `SE_GECKODRIVER` environment variables. +Safari uses the system `safaridriver` (or the Technology Preview driver for Safari Technology Preview). +Enable **Allow remote automation** in Safari's developer settings before running Safari benchmarks; +the launcher does not change this setting. Browser runs require a graphical desktop/session, as before. +WebDriver does not wait for the page's `load` event; the collector report determines completion. +After an error or timeout, the current run is failed without a retry, and the next run starts a new +browser session. + +For iOS, select a simulator with `--simulator UUID`. Without this option, exactly one available +simulator must already be booted. + +### Example: JVM + +Run two tests in startup and average rendering modes, with three repetitions: + +```bash +./launcher/bin/benchmarks-launcher --target jvm \ + --compose-version 1.13.0-beta02 --kotlin-version 2.4.21 \ + --tests LazyGrid TextLayout --modes startup averageFrame --repeat 3 +``` + +### Example: version matrix + +Compare two Kotlin versions with published and local Compose (four configurations): + +```bash +./launcher/bin/benchmarks-launcher --target jvm \ + --kotlin-version 2.4.20 2.4.21 --compose-version 1.13.0-beta02,local \ + --tests LazyGrid --modes startup --output-dir ./results +``` + +### Example: Firefox + +Measure offscreen rendering in a WasmJS application: + +```bash +./launcher/bin/benchmarks-launcher --target wasmJs \ + --browser "/Applications/Firefox.app" \ + --tests LazyGrid --modes averageFrame +``` + +### Example: local Compose + +Build core sources and use Kotlin 2.4.20 for the benchmark project: + +```bash +./launcher/bin/benchmarks-launcher --target jvm \ + --compose-version local --kotlin-version 2.4.20 \ + --tests LazyGrid --modes startup --output-dir ./results +``` + +## Compose and Skiko compatibility + +`benchmarks-runtime` owns the scene and exposes configuration hooks without depending on any init +module. Both init modules provide the same `org.jetbrains.compose.benchmarks.skiko.configureBenchmarkSkiko()` function. +The app calls it at startup to configure `onBenchmarkComposeSceneInit` and, on JVM, +`getJvmGraphicsContext`. No reflection or service loading is needed. + +The app declares `benchmarks-skiko-init-current` in `commonMain`. When the Compose version is +exactly `1.12.2`, Gradle substitutes `benchmarks-skiko-init-legacy` on all targets. The version comes +from `benchmarkComposeVersion` or the catalog default. Other versions and local core builds keep +current initialization. Skiko and its native libraries come from Compose and are not overridden. + +Current initialization supports Metal/OpenGL/DirectX offscreen contexts on JVM. Legacy initialization +for Compose `1.12.2` uses DirectX on Windows and null-rendering on macOS/Linux, as in the original +multiplatform benchmarks, and leaves the scene initialization hook as a no-op. Offscreen GPU results +from different backends are not directly comparable. Other old Compose versions are not supported. + +## Launcher results + +By default, results go under `out/benchmark-reports`. `--output-dir` replaces this base directory. +Each invocation creates a UTC timestamp directory in `yyyy-MM-dd-HH-mm-ss-SSS` format, then groups +results by versions and target: + +```text +-