diff --git a/CLAUDE.md b/CLAUDE.md index 3e03fa7f9..0ce5a5d7d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -273,10 +273,21 @@ pnpm test-high-risk-ios # --grep '@ios @high-risk' pnpm test-ios-parallel --tier standard # tiered, 6 sims — fastest full-suite local run pnpm test-ios-parallel --list-tiers # tiers, their cost and their passes + +pnpm test-android-parallel --tier full # tiered, 8 emulators (the pool must already be up) +pnpm test-android-parallel --list-tiers ``` -`--tier` provisions throwaway simulators and runs one pass per device class; a `--grep` alongside it -narrows every pass rather than replacing the device-class filter. See **Parallelism** above. +`--tier` runs one pass per device class; a `--grep` alongside it narrows every pass rather than +replacing the device-class filter. See **Parallelism** above. + +The two runners differ in one way that matters: the iOS one **provisions** throwaway simulators and +deletes them afterwards, while the Android one provisions nothing, because Appium will not boot an +emulator. Bring the pool up with `pnpm create-emulators ` first — the runner checks the udids it +needs are attached and refuses up front, since nothing else validates `workers × devices` against the +Android pool (`global-setup` does that arithmetic for iOS only). Its worker counts are the arithmetic +that fills the pool, **not measurements**: nothing above one worker has been timed on Android, and an +emulator is a QEMU VM costing 5-7 GB, so the iOS numbers do not transfer. ### How tags work diff --git a/package.json b/package.json index 2d39280a5..378732850 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "test-mobile": "npx playwright test --project mobile --grep ", "test-ios": "_TESTING=1 npx playwright test --project mobile --grep '@ios'", "test-ios-parallel": "npx ts-node scripts/run_ios_parallel.ts", + "test-android-parallel": "npx ts-node scripts/run_android_parallel.ts", "test-android": "_TESTING=1 npx playwright test --project mobile --grep '@android'", "test-high-risk-android": "_TESTING=1 npx playwright test --project mobile --grep '@android @high-risk'", "test-high-risk-ios": "_TESTING=1 npx playwright test --project mobile --grep '@ios @high-risk'", diff --git a/run/constants/parallelism.ts b/run/constants/parallelism.ts index aa158c89f..c5173db60 100644 --- a/run/constants/parallelism.ts +++ b/run/constants/parallelism.ts @@ -102,12 +102,79 @@ export const PARALLEL_TIERS = { }, } as const satisfies Record; +/** + * Emulator-parallelism tiers, the Android counterpart of the table above. + * + * The mechanics are identical - `openAndroidApp` offsets each worker's pool by `TEST_PARALLEL_INDEX` + * exactly as `openiOSApp` does, and `DEVICES_PER_TEST_COUNT` is the same single global per + * invocation - so parallelism is expressed the same way, as one pass per device class. + * + * **The worker counts here are UNMEASURED.** They are the arithmetic that fills the pool + * (`devices * workers <= emulators`), not findings. The iOS numbers above do not transfer: a booted + * emulator is a QEMU virtual machine costing gigabytes of RAM, where a simulator is a process tree, + * so the two saturate a host for entirely different reasons. Treat a first run on any of these as a + * measurement, and lower the workers before concluding a flake is the app's fault. + * + * RAM is the binding constraint, and it is not subtle. Measured on the 64 GB Linux runner with four + * emulators up: 5.4-7.4 GB PSS each, 24 GB total, essentially all private, so there is no shared-page + * inflation to discount. Eight lands near 50 GB. Check `free -g` against the tier before running one. + * + * Unlike the iOS runner, nothing here provisions anything: Appium will not boot an emulator, so the + * pool has to be up before the run starts (`pnpm create-emulators `). + */ +export const ANDROID_PARALLEL_TIERS = { + /** 4 emulators — the pool the suite shipped with. 3- and 4-device specs stay serial. */ + conservative: { + summary: '4 emulators — safest; 3- and 4-device specs stay serial', + passes: [ + { devices: 1, workers: 4 }, + { devices: 2, workers: 2 }, + { devices: 3, workers: 1 }, + { devices: 4, workers: 1 }, + ], + }, + + /** 6 emulators — every pass but `@4-devices` gets at least two workers. ~37 GB of emulator RAM. */ + standard: { + summary: '6 emulators — every pass but @4-devices parallelises', + passes: [ + { devices: 1, workers: 6 }, + { devices: 2, workers: 3 }, + { devices: 3, workers: 2 }, + { devices: 4, workers: 1 }, + ], + }, + + /** + * 8 emulators, the `MAX_EMULATORS` cap. The only tier in which `@4-devices` parallelises at all. + * + * Near 50 GB of emulator RAM before the tests do anything, so this wants a 64 GB host and nothing + * else running on it - on the Linux runner that means both GitHub Actions runner services stopped, + * not just one. + */ + full: { + summary: '8 emulators — fills the pool; needs a 64 GB host to itself', + passes: [ + { devices: 1, workers: 8 }, + { devices: 2, workers: 4 }, + { devices: 3, workers: 2 }, + { devices: 4, workers: 2 }, + ], + }, +} as const satisfies Record; + +export type AndroidParallelTierName = keyof typeof ANDROID_PARALLEL_TIERS; + +export const ANDROID_PARALLEL_TIER_NAMES = Object.keys( + ANDROID_PARALLEL_TIERS +) as AndroidParallelTierName[]; + export type ParallelTierName = keyof typeof PARALLEL_TIERS; export const PARALLEL_TIER_NAMES = Object.keys(PARALLEL_TIERS) as ParallelTierName[]; -/** Simulators a tier needs: the largest single pass, since passes run one after another. */ -export function simulatorsRequired(tier: ParallelTier): number { +/** Devices a tier needs: the largest single pass, since passes run one after another. */ +export function devicesRequired(tier: ParallelTier): number { return Math.max(...tier.passes.map(p => p.devices * p.workers)); } diff --git a/run/desktop/DesktopWrapper.ts b/run/desktop/DesktopWrapper.ts index 2a6b8cc82..cb6fdf484 100644 --- a/run/desktop/DesktopWrapper.ts +++ b/run/desktop/DesktopWrapper.ts @@ -1114,10 +1114,9 @@ export class DesktopWrapper implements IBaseDeviceWrapper { /** Returns immediately if the loader never shows — see the helper for why that is a pass. */ public async waitForLoadingAnimationToFinish( loader: DataTestId, - appearWithinMs?: number, - finishWithinMs?: number + options?: { appearWithinMs?: number; finishWithinMs?: number; windowMayClose?: boolean } ): Promise { - await waitForLoadingAnimationToFinish(this.page, loader, appearWithinMs, finishWithinMs); + await waitForLoadingAnimationToFinish(this.page, loader, options); } public async clickOnTextMessage( diff --git a/run/desktop/utils.ts b/run/desktop/utils.ts index eaeb38762..2b4e982dc 100644 --- a/run/desktop/utils.ts +++ b/run/desktop/utils.ts @@ -212,12 +212,18 @@ export async function waitForMatchingPlaceholder( * `network_page.spec.ts` grew its own `networkDataLoaded` rather than use this: a loader that never * cleared used to hang until the test timed out with nothing to go on. Now it fails against this * selector, which says what was still spinning. + * + * `windowMayClose` is for the work that restarts the app (clearing all data): the loader goes away + * with the window, which Playwright reports as a target-closed error rather than a hidden element. */ export async function waitForLoadingAnimationToFinish( window: Page, loader: DataTestId, - appearWithinMs = 2_000, - finishWithinMs = 60_000 + { + appearWithinMs = 2_000, + finishWithinMs = 60_000, + windowMayClose = false, + }: { appearWithinMs?: number; finishWithinMs?: number; windowMayClose?: boolean } = {} ) { const selector = buildSelectorEscapeText({ strategy: 'data-testid', selector: loader }); @@ -234,10 +240,23 @@ export async function waitForLoadingAnimationToFinish( } console.info(`${loader} was found, waiting for it to be gone`); - await window.waitForSelector(selector, { timeout: finishWithinMs, state: 'hidden' }); + try { + await window.waitForSelector(selector, { timeout: finishWithinMs, state: 'hidden' }); + } catch (e) { + if (!windowMayClose || !windowWentAway(window, e as Error)) { + throw e; + } + console.info(`${loader} went away with its window — the app restarted`); + return; + } console.info('Loading animation has finished'); } +/** `isClosed()` does not always flip before the pending wait rejects, hence the message check too. */ +function windowWentAway(window: Page, e: Error): boolean { + return window.isClosed() || /closed/i.test(e.message); +} + /** * The absence counterpart to `waitForElement`. `hidden` covers never-attached, so it also suits an * element that should never have rendered. diff --git a/run/test/specs/desktop/delete_account.spec.ts b/run/test/specs/desktop/delete_account.spec.ts index 628f156c2..3607e0f12 100644 --- a/run/test/specs/desktop/delete_account.spec.ts +++ b/run/test/specs/desktop/delete_account.spec.ts @@ -34,11 +34,12 @@ sessionTestTwoWindows('Delete account from swarm', async ([windowA, windowB]) => // Confirm deletion by clicking Clear, twice await windowA.clickOnMatchingText(tStripped('clear')); await windowA.clickOnMatchingText(tStripped('clear')); - await windowA.waitForLoadingAnimationToFinish(Global.loadingSpinner.selector); - // await sleepFor(7500); - // Wait for window to close and reopen - - // await windowA.close(); + // The delete restarts the app, so the spinner is optional at both ends: on a fast network it + // can be gone before we look, and when we do catch it, it leaves with the window. + await windowA.waitForLoadingAnimationToFinish(Global.loadingSpinner.selector, { + appearWithinMs: 1_000, + windowMayClose: true, + }); restoringWindows = await openAppsAndWaitWindows(1); // not using sessionTest here as we need to close and reopen one of the window const [restoringWindowPage] = restoringWindows; const restoringWindow = new DesktopWrapper(restoringWindowPage, 'alice-restoring'); diff --git a/run/test/utils/capabilities_android.ts b/run/test/utils/capabilities_android.ts index 92e9aa129..0c4297806 100644 --- a/run/test/utils/capabilities_android.ts +++ b/run/test/utils/capabilities_android.ts @@ -4,6 +4,7 @@ import { isString } from 'lodash'; import type { ProContext } from './pro_context'; +import { BASE_PORT, MAX_EMULATORS } from '../../../scripts/android_config'; import { getAndroidApk } from './binaries'; import { buildAndroidLaunchExtras } from './devnet_android'; dotenv.config({ quiet: true }); @@ -49,7 +50,13 @@ const sharedCapabilities: W3CUiautomator2DriverCaps['alwaysMatch'] = { 'appium:enforceAppInstall': true, }; -const udids = ['emulator-5554', 'emulator-5556', 'emulator-5558', 'emulator-5560']; +// Derived rather than listed: the pool size and the port step already live in `android_config`, and +// spelling the udids out here meant growing the pool was a two-file change with nothing to catch a +// half-done one. +const udids = Array.from( + { length: MAX_EMULATORS }, + (_, i) => `emulator-${BASE_PORT + i * 2}` as const +); const emulatorCapabilities: W3CUiautomator2DriverCaps['alwaysMatch'][] = udids.map(udid => ({ ...sharedCapabilities, diff --git a/run/test/utils/copy_file_to_simulator.ts b/run/test/utils/copy_file_to_simulator.ts index c7618b9fe..ccf47b9eb 100644 --- a/run/test/utils/copy_file_to_simulator.ts +++ b/run/test/utils/copy_file_to_simulator.ts @@ -2,10 +2,10 @@ import { execSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; +import { mediaFolder } from '../../constants/testfiles'; import { DeviceWrapper } from '../../types/DeviceWrapper'; const TARGET_GROUP_ID = 'group.com.apple.FileProvider.LocalStorage'; -const MEDIA_ROOT = path.join('run', 'test', 'media'); /** * Utility for copying a file from the local 'media' directory to the current iOS simulator's @@ -44,7 +44,7 @@ function getSimulatorDownloadsPath( * Copies a file from the 'media' directory to the simulator's "Downloads" folder on disk if not already present. */ export function copyFileToSimulator(device: DeviceWrapper, fileName: string): void { - const sourcePath = path.join(MEDIA_ROOT, fileName); + const sourcePath = path.join(mediaFolder, fileName); const groupContainerPath = getFilesAppGroupContainerPath(device.udid); const { downloadsPath, destinationPath } = getSimulatorDownloadsPath( diff --git a/scripts/android_config.ts b/scripts/android_config.ts index fbe5303cb..6b39d30fb 100644 --- a/scripts/android_config.ts +++ b/scripts/android_config.ts @@ -36,8 +36,15 @@ export const AVD_RAM_MB = 4192; /** The suite's first udid; each subsequent emulator is +2 (adb's console/adb port pairing). */ export const BASE_PORT = 5554; -/** `capabilities_android.ts` declares exactly this many udids. */ -export const MAX_EMULATORS = 4; +/** + * Size of the emulator pool, and the count `capabilities_android.ts` builds its udid list from. + * + * Bounded by host RAM, not by anything in the suite. Measured on the 64 GB Linux runner: four + * emulators sit at 5.4-7.4 GB PSS each (24 GB total, essentially all private), leaving eight at + * roughly 50 GB and ~11 GB of headroom. Raise this only after measuring on the host in question - an + * over-subscribed host fails with timeouts indistinguishable from product bugs. + */ +export const MAX_EMULATORS = 8; /** The command-line tools bundle, used only when provisioning a CI machine from scratch. */ export const CMDLINE_TOOLS_ZIP = 'commandlinetools-linux-11076708_latest.zip'; diff --git a/scripts/parallel_shared.ts b/scripts/parallel_shared.ts new file mode 100644 index 000000000..d73f5bd9d --- /dev/null +++ b/scripts/parallel_shared.ts @@ -0,0 +1,408 @@ +import { spawn } from 'child_process'; + +import type { ClientPlatform, ServiceNetwork } from '../run/types/target'; + +import { + devicesRequired, + type ParallelPass, + type ParallelTier, + passGrep, +} from '../run/constants/parallelism'; +import { ALLOWED_NETWORKS } from '../run/test/utils/network_target'; + +/** + * The whole of a tiered parallel run, for both platforms. + * + * `run_ios_parallel.ts` and `run_android_parallel.ts` differ in exactly one thing: what they have to + * arrange before a run. The iOS one creates and deletes a simulator pool; the Android one can only + * check that an already-booted one is there, because Appium will not boot an emulator. That is the + * `prepareDevices` hook, and everything else — argument parsing, validation, the child environment, + * the pass loop, cleanup and the exit status — is `runParallelSuite` here. + * + * Nothing in this module knows which platform it is serving; the caller passes the tier table, the + * noun for its devices and the env var its worker count lives in. + */ + +/** Flags both runners accept. A platform's own flags extend this. */ +export type ParallelArgsBase = { + workers: number; + devicesPerWorker: number; + grep: string; + tier?: string; + listTiers: boolean; + /** Set when the caller passed --workers/--devices-per-worker, so --tier can reject the combination. */ + explicitPools: boolean; + network?: string; + passthrough: string[]; +}; + +/** Platform-only flags, declared rather than parsed by hand, so the shared loop can own the rest. */ +export type ExtraFlags = { + /** `--keep` — present or absent, no value. */ + boolean?: Record void>; + /** `--runtime 26.1` and `--runtime=26.1` both reach the handler with just the value. */ + value?: Record void>; +}; + +/** + * Accepts both `--flag value` and `--flag=value`; the boolean says whether `next` was consumed. + * + * A missing value is refused rather than read as empty: `--network` and `--grep` would otherwise + * silently fall back to their defaults, and an unnoticed `--network` default provisions the pool and + * runs the suite against the wrong network. + */ +function readValue(current: string, next: string | undefined): [string, boolean] { + const eq = current.indexOf('='); + const [value, consumedNext] = eq !== -1 ? [current.slice(eq + 1), false] : [next ?? '', true]; + + if (!value || value.startsWith('--')) { + console.error(`Missing value for ${eq === -1 ? current : current.slice(0, eq)}.`); + process.exit(1); + } + + return [value, consumedNext]; +} + +function parseParallelArgs({ + argv, + defaults, + tierNames, + extra, +}: { + argv: string[]; + defaults: T; + tierNames: readonly string[]; + extra?: ExtraFlags; +}): T { + const args = { ...defaults }; + + // Everything after a lone `--` is forwarded verbatim to Playwright. + const sepIndex = argv.indexOf('--'); + const ownArgs = sepIndex === -1 ? argv : argv.slice(0, sepIndex); + if (sepIndex !== -1) { + args.passthrough = argv.slice(sepIndex + 1); + } + + for (let i = 0; i < ownArgs.length; i++) { + const arg = ownArgs[i]; + const booleanHandler = extra?.boolean?.[arg]; + if (booleanHandler) { + booleanHandler(args); + continue; + } + + const extraValueName = Object.keys(extra?.value ?? {}).find(name => arg.startsWith(name)); + if (extraValueName) { + const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); + extra?.value?.[extraValueName]?.(args, value); + if (consumedNext) i++; + continue; + } + + if (arg === '--list-tiers') { + args.listTiers = true; + } else if (arg.startsWith('--tier')) { + const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); + if (!tierNames.includes(value)) { + console.error(`Invalid --tier "${value}". Use ${tierNames.join(' | ')}.`); + process.exit(1); + } + args.tier = value; + if (consumedNext) i++; + } else if (arg.startsWith('--workers')) { + const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); + args.workers = parseInt(value); + args.explicitPools = true; + if (consumedNext) i++; + } else if (arg.startsWith('--devices-per-worker')) { + const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); + args.devicesPerWorker = parseInt(value); + args.explicitPools = true; + if (consumedNext) i++; + } else if (arg.startsWith('--grep')) { + const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); + args.grep = value; + if (consumedNext) i++; + } else if (arg.startsWith('--network')) { + const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); + args.network = value; + if (consumedNext) i++; + } else { + console.error(`Unknown argument: "${arg}". Forward Playwright args after a "--" separator.`); + process.exit(1); + } + } + + return args; +} + +/** + * What one platform's runner has to say about itself. Everything else is shared. + */ +export type ParallelSuite = { + platform: ClientPlatform & ('android' | 'ios'); + /** Singular, for the messages: "simulator" / "emulator". */ + deviceNoun: string; + /** The platform filter a run falls back to when no --grep is given. */ + defaultGrep: string; + workersEnvVar: 'PLAYWRIGHT_WORKERS_COUNT_ANDROID' | 'PLAYWRIGHT_WORKERS_COUNT_IOS'; + maxDevices: number; + tiers: Record; + tierNames: readonly string[]; + tiersPreamble: string; + defaults: T; + /** Flags only this platform takes, e.g. --keep and --runtime on iOS. */ + extraFlags?: ExtraFlags; + /** + * The one thing the platforms do not share: iOS creates the pool it just sized, Android can only + * check that one is attached. Returning env merges it into the child's (iOS passes its new UDIDs + * that way); returning a cleanup runs it once, whether the run finished or threw. + */ + prepareDevices: (args: T, deviceCount: number) => DevicePrep | void; +}; + +export type DevicePrep = { env?: NodeJS.ProcessEnv; cleanup?: () => void }; + +/** + * Both suites live in the `mobile` project. Passing it keeps a desktop or cross-platform title that + * happens to contain `@ios`/`@android` out of the run — no title does today, so this changes nothing + * yet; it is what stops one appearing from silently widening a mobile run. + */ +const MOBILE_PROJECT_ARGS = ['--project', 'mobile']; + +/** + * How many devices the run will draw, refusing anything the pool cannot serve. + * + * Checked before either runner touches a device: on iOS an invalid `--network` would otherwise create + * the whole simulator pool before failing downstream, and on Android an over-subscribed run would + * reach the tests and fail each one individually. + */ +function validateParallelArgs( + suite: ParallelSuite, + args: T +): number { + const { tiers, maxDevices, deviceNoun } = suite; + + if (args.network && !ALLOWED_NETWORKS.includes(args.network as ServiceNetwork)) { + console.error(`Invalid --network "${args.network}". Use ${ALLOWED_NETWORKS.join(' | ')}.`); + process.exit(1); + } + + const refuse = (message: string): never => { + console.error(message); + process.exit(1); + }; + + if (args.tier) { + if (args.explicitPools) { + refuse( + `--tier sets devices-per-worker and workers per pass, so it cannot be combined with ` + + `--workers / --devices-per-worker. Drop one or the other.` + ); + } + const needed = devicesRequired(tiers[args.tier]); + if (needed > maxDevices) { + refuse( + `Tier "${args.tier}" needs ${needed} ${deviceNoun}s, but the maximum is ${maxDevices}.` + ); + } + return needed; + } + + if (isNaN(args.workers) || args.workers < 1) { + refuse(`Invalid --workers value: ${args.workers}`); + } + if (isNaN(args.devicesPerWorker) || args.devicesPerWorker < 1) { + refuse(`Invalid --devices-per-worker value: ${args.devicesPerWorker}`); + } + + const total = args.workers * args.devicesPerWorker; + if (total > maxDevices) { + refuse( + `Requested ${args.workers} workers x ${args.devicesPerWorker} devices-per-worker = ` + + `${total} ${deviceNoun}s, but the maximum is ${maxDevices}. ` + + `Lower --workers or --devices-per-worker.` + ); + } + + return total; +} + +function printTiers({ + tiers, + tierNames, + deviceNoun, + tiersPreamble, +}: ParallelSuite): void { + console.log(`\n${tiersPreamble}\n`); + for (const name of tierNames) { + const tier = tiers[name]; + console.log(` ${name} — ${tier.summary}`); + console.log(` ${deviceNoun}s needed: ${devicesRequired(tier)}`); + for (const pass of tier.passes) { + console.log( + ` @${pass.devices}-devices x${pass.workers} worker(s) ` + + `(${pass.devices * pass.workers} ${deviceNoun}s)` + ); + } + console.log(''); + } +} + +/** Runs one Playwright invocation to completion and resolves with its exit status. */ +function runPlaywright(playwrightArgs: string[], env: NodeJS.ProcessEnv): Promise { + return new Promise((resolve, reject) => { + console.log(`\nRunning: npx ${playwrightArgs.join(' ')}\n`); + const child = spawn('npx', playwrightArgs, { stdio: 'inherit', env }); + + // Attached per invocation and detached on exit. A tiered run spawns one child per pass, so + // leaving these registered would leak listeners and signal already-dead children. + const forward = (signal: NodeJS.Signals) => () => child.kill(signal); + const onInt = forward('SIGINT'); + const onTerm = forward('SIGTERM'); + process.on('SIGINT', onInt); + process.on('SIGTERM', onTerm); + const detach = () => { + process.off('SIGINT', onInt); + process.off('SIGTERM', onTerm); + }; + + child.on('error', err => { + detach(); + reject(err); + }); + child.on('exit', (code, signal) => { + detach(); + resolve(code ?? (signal ? 1 : 0)); + }); + }); +} + +type PassResult = { pass: ParallelPass; code: number }; + +function printTierSummary(name: string, deviceNoun: string, results: PassResult[]): void { + console.log(`\n=== tier "${name}" summary ===`); + for (const { pass, code } of results) { + const status = code === 0 ? 'pass' : `FAILED (exit ${code})`; + console.log( + ` @${pass.devices}-devices x${pass.workers} worker(s) (${deviceNoun}s): ${status}` + ); + } + console.log(''); +} + +/** + * One Playwright invocation per device class, in sequence, resolving with the run's exit status. + * + * A failing pass does NOT stop the ones after it: a regression run is worth completing so you see + * every device class rather than everything up to the first breakage. + */ +async function runTier( + suite: ParallelSuite, + args: T, + tierName: string, + baseEnv: NodeJS.ProcessEnv +): Promise { + const { platform, deviceNoun, defaultGrep, workersEnvVar } = suite; + const results: PassResult[] = []; + + for (const pass of suite.tiers[tierName].passes) { + // The pass owns the platform and device-count filter; a caller-supplied --grep is ANDed on top as + // a further lookahead rather than replacing it, so `--grep '@high-risk'` narrows each pass instead + // of selecting the wrong device class. + const passFilter = passGrep(pass, platform); + const combined = args.grep === defaultGrep ? passFilter : `${passFilter}(?=.*${args.grep})`; + + console.log( + `\n=== tier "${tierName}": @${pass.devices}-devices, ${pass.workers} worker(s), ` + + `${pass.devices * pass.workers} ${deviceNoun}(s) ===` + ); + + const code = await runPlaywright( + [ + 'playwright', + 'test', + ...MOBILE_PROJECT_ARGS, + '--grep', + combined, + // An empty pass is not a failure: an extra --grep can legitimately clear one device class + // while the others still have work to do. + '--pass-with-no-tests', + ...args.passthrough, + ], + { + ...baseEnv, + DEVICES_PER_TEST_COUNT: String(pass.devices), + [workersEnvVar]: String(pass.workers), + } + ); + + results.push({ pass, code }); + } + + printTierSummary(tierName, deviceNoun, results); + + return results.some(r => r.code !== 0) ? 1 : 0; +} + +/** Parses, validates, provisions, runs, cleans up, and exits with the run's status. */ +export async function runParallelSuite( + suite: ParallelSuite, + argv: string[] +): Promise { + const args = parseParallelArgs({ + argv, + defaults: suite.defaults, + tierNames: suite.tierNames, + extra: suite.extraFlags, + }); + + if (args.listTiers) { + printTiers(suite); + process.exit(0); + } + + const deviceCount = validateParallelArgs(suite, args); + const prep = suite.prepareDevices(args, deviceCount) ?? {}; + + const childEnv: NodeJS.ProcessEnv = { + ...process.env, + ...prep.env, + PLATFORM: suite.platform, + [suite.workersEnvVar]: String(args.workers), + DEVICES_PER_TEST_COUNT: String(args.devicesPerWorker), + }; + // Silences the driver's per-command logging; a tiered run is long enough that the noise buries + // the reporter's own output. + childEnv._TESTING = childEnv._TESTING ?? '1'; + // Left unset otherwise, so .env's NETWORK_TARGET is respected. Devnet also needs DEVNET_SEED_URL + // in .env — everything else about it is discovered (see run/test/utils/network_target.ts). + if (args.network) { + childEnv.NETWORK_TARGET = args.network; + } + + let cleanedUp = false; + const cleanup = () => { + if (cleanedUp) { + return; + } + cleanedUp = true; + prep.cleanup?.(); + }; + + try { + const code = args.tier + ? await runTier(suite, args, args.tier, childEnv) + : await runPlaywright( + ['playwright', 'test', ...MOBILE_PROJECT_ARGS, '--grep', args.grep, ...args.passthrough], + childEnv + ); + cleanup(); + // Preserve the child's exit status so CI/other callers see the real result. + process.exit(code); + } catch (err) { + console.error('Failed to start Playwright:', err); + cleanup(); + process.exit(1); + } +} diff --git a/scripts/print_tier.ts b/scripts/print_tier.ts index 2daec85c7..68596c52e 100644 --- a/scripts/print_tier.ts +++ b/scripts/print_tier.ts @@ -10,10 +10,10 @@ * import, and anything on stdout here would be parsed as data by the caller. */ import { + devicesRequired, PARALLEL_TIER_NAMES, PARALLEL_TIERS, type ParallelTierName, - simulatorsRequired, } from '../run/constants/parallelism'; function main(): void { @@ -33,7 +33,7 @@ function main(): void { const tier = PARALLEL_TIERS[name as ParallelTierName]; if (wantSims) { - console.log(String(simulatorsRequired(tier))); + console.log(String(devicesRequired(tier))); return; } diff --git a/scripts/run_android_parallel.ts b/scripts/run_android_parallel.ts new file mode 100644 index 000000000..22af160c1 --- /dev/null +++ b/scripts/run_android_parallel.ts @@ -0,0 +1,130 @@ +import { spawnSync } from 'child_process'; +import dotenv from 'dotenv'; + +import { + ANDROID_PARALLEL_TIER_NAMES, + ANDROID_PARALLEL_TIERS, + type AndroidParallelTierName, +} from '../run/constants/parallelism'; +import { getAdbFullPath } from '../run/test/utils/binaries'; +import { BASE_PORT, MAX_EMULATORS } from './android_config'; +import { type ParallelArgsBase, runParallelSuite } from './parallel_shared'; + +/** + * Parallel Android test runner — the counterpart of `run_ios_parallel.ts`. + * + * Runs the Android suite across multiple Playwright workers, one invocation per device class. + * `DEVICES_PER_TEST_COUNT` is a single global per invocation, so a single run has to size every + * worker's pool for the largest spec: at `D=4` a `@1-devices` spec occupies a worker holding four + * emulators and using one. Passes run in sequence, so the emulator draw is the largest pass rather + * than the sum. + * + * **It provisions nothing, and that is the difference from the iOS runner.** Appium will not boot an + * emulator, so the pool has to be up before the run starts (`pnpm create-emulators `). This checks + * the pool it needs is actually attached and refuses up front rather than letting each test fail with + * `Invalid actual capability given` — nothing else validates `workers × devices` against the pool. + * + * Usage: + * pnpm test-android-parallel --tier standard # tiered: one pass per device class + * pnpm test-android-parallel --list-tiers # tiers and what they cost + * pnpm test-android-parallel --tier full --grep '@high-risk' # narrows every pass + * pnpm test-android-parallel --workers 2 --devices-per-worker 2 + * pnpm test-android-parallel --tier full -- --repeat-each 2 # after `--` goes to Playwright + * + * Notes: + * - The tier worker counts are UNMEASURED on Android; see `run/constants/parallelism.ts`. Treat a + * first run as a measurement. + * - RAM, not CPU, is the ceiling: an emulator costs 5-7 GB and an over-subscribed host fails with + * timeouts indistinguishable from product bugs. + * - A `--grep` alongside `--tier` is ANDed in as a further lookahead, so it narrows each pass + * rather than replacing the pass's own device-class filter. + * - A failing pass does not stop the rest; the exit status is non-zero if any pass failed. + */ + +dotenv.config({ quiet: true }); + +const DEFAULT_GREP = '@android'; + +type ParsedArgs = ParallelArgsBase & { tier?: AndroidParallelTierName }; + +/** The udids a pool of `count` emulators occupies, in the order the suite allocates them. */ +function poolUdids(count: number): string[] { + return Array.from({ length: count }, (_, i) => `emulator-${BASE_PORT + i * 2}`); +} + +/** Emulators currently attached and past boot, by udid. */ +function attachedEmulators(): Set { + const adb = getAdbFullPath(); + const result = spawnSync(adb, ['devices'], { encoding: 'utf8' }); + if (result.status !== 0) { + console.error(`\`${adb} devices\` failed:\n${result.stderr || result.stdout}`); + process.exit(1); + } + + return new Set( + result.stdout + .split('\n') + .slice(1) + .map(line => line.trim().split(/\s+/)) + // Only `device`; an emulator still in `offline` cannot take a session and would fail mid-run. + .filter(([udid, state]) => udid?.startsWith('emulator-') && state === 'device') + .map(([udid]) => udid) + ); +} + +/** + * All this runner provisions: nothing. Appium will not boot an emulator, so refuse before Playwright + * starts if the pool the run needs is not up. + * + * Worth doing here because nothing downstream does: `global-setup` only checks this arithmetic for + * iOS, so an over-subscribed Android run reaches the tests and fails each one individually with + * `Invalid actual capability given: N`, which reads as a suite bug rather than a missing emulator. + */ +function requirePool(needed: number): void { + if (!process.env.ANDROID_APK) { + console.error('ANDROID_APK is not set — point it at a QA/AQA build first.'); + process.exit(1); + } + + // `needed <= MAX_EMULATORS` already, from the shared validator. + const wanted = poolUdids(needed); + const attached = attachedEmulators(); + const missing = wanted.filter(udid => !attached.has(udid)); + if (missing.length) { + console.error( + `\nThis run needs ${needed} emulator(s) on ${wanted.join(', ')}, but ` + + `${missing.join(', ')} ${missing.length === 1 ? 'is' : 'are'} not attached.\n\n` + + `Appium does not boot emulators — start them first:\n` + + ` pnpm create-emulators ${needed}\n` + ); + process.exit(1); + } + + console.log(`✓ ${needed} emulator(s) attached: ${wanted.join(', ')}`); +} + +void runParallelSuite( + { + platform: 'android', + deviceNoun: 'emulator', + defaultGrep: DEFAULT_GREP, + workersEnvVar: 'PLAYWRIGHT_WORKERS_COUNT_ANDROID', + maxDevices: MAX_EMULATORS, + tiers: ANDROID_PARALLEL_TIERS, + tierNames: ANDROID_PARALLEL_TIER_NAMES, + tiersPreamble: + 'Available tiers (worker counts are unmeasured — see run/constants/parallelism.ts):', + defaults: { + // Defaults to the whole suite on one worker: the pool has to be booted already, so guessing at + // a wider one would fail the pool check rather than run anything. + workers: 1, + devicesPerWorker: 4, + grep: DEFAULT_GREP, + listTiers: false, + explicitPools: false, + passthrough: [], + }, + prepareDevices: (_args, deviceCount) => requirePool(deviceCount), + }, + process.argv.slice(2) +); diff --git a/scripts/run_ios_parallel.ts b/scripts/run_ios_parallel.ts index 173d0c75b..2fbe9f972 100644 --- a/scripts/run_ios_parallel.ts +++ b/scripts/run_ios_parallel.ts @@ -1,21 +1,15 @@ -import { spawn } from 'child_process'; import dotenv from 'dotenv'; -import type { ServiceNetwork } from '../run/types/target'; - import { PARALLEL_TIER_NAMES, PARALLEL_TIERS, - type ParallelPass, type ParallelTierName, - passGrep, - simulatorsRequired, } from '../run/constants/parallelism'; import { type Simulator } from '../run/test/utils/capabilities_ios'; -import { ALLOWED_NETWORKS } from '../run/test/utils/network_target'; import { ensureWdaBuilt } from './build_wda'; import { createIOSSimulators, resolveDeviceConfig } from './create_ios_simulators'; import { deleteSimulators } from './ios_shared'; +import { type ParallelArgsBase, runParallelSuite } from './parallel_shared'; /** * Self-contained parallel iOS test runner. @@ -81,140 +75,12 @@ const MAX_SIMULATORS = 12; const DEFAULT_GREP = '@ios'; -type ParsedArgs = { - workers: number; - devicesPerWorker: number; - grep: string; +type ParsedArgs = ParallelArgsBase & { keep: boolean; tier?: ParallelTierName; - listTiers: boolean; - /** Set when the caller passed --workers/--devices-per-worker, so --tier can reject the combination. */ - explicitPools: boolean; runtime?: string; - network?: string; - passthrough: string[]; }; -function parseArgs(argv: string[]): ParsedArgs { - const args: ParsedArgs = { - workers: 2, - devicesPerWorker: 2, - grep: DEFAULT_GREP, - keep: false, - listTiers: false, - explicitPools: false, - passthrough: [], - }; - - // Everything after a lone `--` is forwarded verbatim to Playwright. - const sepIndex = argv.indexOf('--'); - const ownArgs = sepIndex === -1 ? argv : argv.slice(0, sepIndex); - if (sepIndex !== -1) { - args.passthrough = argv.slice(sepIndex + 1); - } - - // Accepts both `--flag value` and `--flag=value`. - const readValue = (current: string, next: string | undefined): [string, boolean] => { - const eq = current.indexOf('='); - if (eq !== -1) { - return [current.slice(eq + 1), false]; - } - return [next ?? '', true]; - }; - - for (let i = 0; i < ownArgs.length; i++) { - const arg = ownArgs[i]; - if (arg === '--keep') { - args.keep = true; - } else if (arg === '--list-tiers') { - args.listTiers = true; - } else if (arg.startsWith('--tier')) { - const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); - if (!PARALLEL_TIER_NAMES.includes(value as ParallelTierName)) { - console.error(`Invalid --tier "${value}". Use ${PARALLEL_TIER_NAMES.join(' | ')}.`); - process.exit(1); - } - args.tier = value as ParallelTierName; - if (consumedNext) i++; - } else if (arg.startsWith('--workers')) { - const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); - args.workers = parseInt(value); - args.explicitPools = true; - if (consumedNext) i++; - } else if (arg.startsWith('--devices-per-worker')) { - const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); - args.devicesPerWorker = parseInt(value); - args.explicitPools = true; - if (consumedNext) i++; - } else if (arg.startsWith('--grep')) { - const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); - args.grep = value; - if (consumedNext) i++; - } else if (arg.startsWith('--runtime')) { - const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); - args.runtime = value; - if (consumedNext) i++; - } else if (arg.startsWith('--network')) { - const [value, consumedNext] = readValue(arg, ownArgs[i + 1]); - args.network = value; - if (consumedNext) i++; - } else { - console.error(`Unknown argument: "${arg}". Forward Playwright args after a "--" separator.`); - process.exit(1); - } - } - - return args; -} - -function validate(args: ParsedArgs): number { - if (!process.env.IOS_APP_PATH_PREFIX) { - console.error('IOS_APP_PATH_PREFIX is not set — point it at a simulator Session.app first.'); - process.exit(1); - } - // Validate --network before provisioning: an unknown value (e.g. a "devent" typo) would - // otherwise create the whole simulator pool and spawn Playwright before failing downstream. - if (args.network && !ALLOWED_NETWORKS.includes(args.network as ServiceNetwork)) { - console.error(`Invalid --network "${args.network}". Use ${ALLOWED_NETWORKS.join(' | ')}.`); - process.exit(1); - } - if (args.tier) { - if (args.explicitPools) { - console.error( - `--tier sets devices-per-worker and workers per pass, so it cannot be combined with ` + - `--workers / --devices-per-worker. Drop one or the other.` - ); - process.exit(1); - } - const needed = simulatorsRequired(PARALLEL_TIERS[args.tier]); - if (needed > MAX_SIMULATORS) { - console.error( - `Tier "${args.tier}" needs ${needed} simulators, but the maximum is ${MAX_SIMULATORS}.` - ); - process.exit(1); - } - return needed; - } - if (isNaN(args.workers) || args.workers < 1) { - console.error(`Invalid --workers value: ${args.workers}`); - process.exit(1); - } - if (isNaN(args.devicesPerWorker) || args.devicesPerWorker < 1) { - console.error(`Invalid --devices-per-worker value: ${args.devicesPerWorker}`); - process.exit(1); - } - const totalSimulators = args.workers * args.devicesPerWorker; - if (totalSimulators > MAX_SIMULATORS) { - console.error( - `Requested ${args.workers} workers x ${args.devicesPerWorker} devices-per-worker = ` + - `${totalSimulators} simulators, but the maximum is ${MAX_SIMULATORS}. ` + - `Lower --workers or --devices-per-worker.` - ); - process.exit(1); - } - return totalSimulators; -} - function printKeepInfo(simulators: Simulator[]): void { console.log(`\nLeaving ${simulators.length} simulator(s) in place (--keep).`); console.log('To reuse them with `pnpm test-ios`, put these lines in your .env:\n'); @@ -224,67 +90,18 @@ function printKeepInfo(simulators: Simulator[]): void { console.log('`xcrun simctl delete `.\n'); } -function printTiers(): void { - console.log('\nAvailable tiers (see run/constants/parallelism.ts for the measurements):\n'); - for (const name of PARALLEL_TIER_NAMES) { - const tier = PARALLEL_TIERS[name]; - console.log(` ${name} — ${tier.summary}`); - console.log(` simulators needed: ${simulatorsRequired(tier)}`); - for (const pass of tier.passes) { - console.log( - ` @${pass.devices}-devices x${pass.workers} worker(s) ` + - `(${pass.devices * pass.workers} sims)` - ); - } - console.log(''); - } -} - -/** Runs one Playwright invocation to completion and resolves with its exit status. */ -function runPlaywright(playwrightArgs: string[], env: NodeJS.ProcessEnv): Promise { - return new Promise((resolve, reject) => { - console.log(`\nRunning: npx ${playwrightArgs.join(' ')}\n`); - const child = spawn('npx', playwrightArgs, { stdio: 'inherit', env }); - - // Attached per invocation and detached on exit. A tiered run spawns one child per pass, so - // leaving these registered would leak listeners and signal already-dead children. - const forward = (signal: NodeJS.Signals) => () => child.kill(signal); - const onInt = forward('SIGINT'); - const onTerm = forward('SIGTERM'); - process.on('SIGINT', onInt); - process.on('SIGTERM', onTerm); - const detach = () => { - process.off('SIGINT', onInt); - process.off('SIGTERM', onTerm); - }; - - child.on('error', err => { - detach(); - reject(err); - }); - child.on('exit', (code, signal) => { - detach(); - resolve(code ?? (signal ? 1 : 0)); - }); - }); -} - -function printTierSummary(name: string, results: { pass: ParallelPass; code: number }[]): void { - console.log(`\n=== tier "${name}" summary ===`); - for (const { pass, code } of results) { - const status = code === 0 ? 'pass' : `FAILED (exit ${code})`; - console.log(` @${pass.devices}-devices x${pass.workers} worker(s): ${status}`); - } - console.log(''); -} - -async function main(): Promise { - const args = parseArgs(process.argv.slice(2)); - if (args.listTiers) { - printTiers(); - return; +/** + * The throwaway pool, which is all this runner does that the Android one cannot. + * + * The UDIDs go into the child's environment only: `capabilities_ios` reads IOS_N_SIMULATOR from + * process.env and its own `dotenv.config()` does not override an already-set var, so these win over + * any .env entries and the developer's .env is left untouched. + */ +function createPool(args: ParsedArgs, totalSimulators: number) { + if (!process.env.IOS_APP_PATH_PREFIX) { + console.error('IOS_APP_PATH_PREFIX is not set — point it at a simulator Session.app first.'); + process.exit(1); } - const totalSimulators = validate(args); // Build the WebDriverAgent runner once up front so the driver reuses it across every simulator // instead of building/launching WDA per session (the slowest, flakiest part of a cold-sim @@ -299,95 +116,49 @@ async function main(): Promise { const deviceConfig = resolveDeviceConfig({ runtime: args.runtime }); const simulators = createIOSSimulators({ ...deviceConfig, totalSimulators }); - // Inject the freshly-created UDIDs into the child's environment only. capabilities_ios reads - // IOS_N_SIMULATOR from process.env; dotenv.config() there does NOT override already-set vars, - // so these win over any .env entries and the developer's .env is left untouched. - const childEnv: NodeJS.ProcessEnv = { ...process.env }; + const env: NodeJS.ProcessEnv = {}; simulators.forEach((sim, i) => { - childEnv[`IOS_${i + 1}_SIMULATOR`] = sim.udid; + env[`IOS_${i + 1}_SIMULATOR`] = sim.udid; }); - childEnv.PLATFORM = 'ios'; - childEnv.PLAYWRIGHT_WORKERS_COUNT_IOS = String(args.workers); - childEnv.DEVICES_PER_TEST_COUNT = String(args.devicesPerWorker); - childEnv._TESTING = childEnv._TESTING ?? '1'; - // Service network selection. Devnet also needs DEVNET_SEED_URL in .env — the pubkey and storage - // ports are discovered from that seed node (see run/test/utils/network_target.ts), so nothing else - // is required. Left unset here so .env's NETWORK_TARGET is respected. - if (args.network) { - childEnv.NETWORK_TARGET = args.network; - } - - let cleanedUp = false; - const cleanup = () => { - if (cleanedUp) { - return; - } - cleanedUp = true; - if (args.keep) { - printKeepInfo(simulators); - return; - } - console.log('\nDeleting temporary simulators...'); - const deleted = deleteSimulators(simulators.map(s => s.udid)); - console.log(`✓ Deleted ${deleted} simulator(s)`); - }; - - try { - if (args.tier) { - const tier = PARALLEL_TIERS[args.tier]; - const results: { pass: ParallelPass; code: number }[] = []; - - for (const pass of tier.passes) { - // The pass owns the platform and device-count filter; a caller-supplied --grep is ANDed on - // top as a further lookahead rather than replacing it, so `--grep '@ios @high-risk'` narrows - // each pass instead of selecting the wrong device class. - const grep = - args.grep === DEFAULT_GREP ? passGrep(pass) : `${passGrep(pass)}(?=.*${args.grep})`; - console.log( - `\n=== tier "${args.tier}": @${pass.devices}-devices, ${pass.workers} worker(s), ` + - `${pass.devices * pass.workers} simulator(s) ===` - ); - - const code = await runPlaywright( - [ - 'playwright', - 'test', - '--grep', - grep, - // An empty pass is not a failure: an extra --grep can legitimately clear one device - // class while the others still have work to do. - '--pass-with-no-tests', - ...args.passthrough, - ], - { - ...childEnv, - DEVICES_PER_TEST_COUNT: String(pass.devices), - PLAYWRIGHT_WORKERS_COUNT_IOS: String(pass.workers), - } - ); - // Deliberately not bailing on the first failure — a regression run is worth completing so - // you see every device class, not just up to the first one that broke. - results.push({ pass, code }); + return { + env, + cleanup: () => { + if (args.keep) { + printKeepInfo(simulators); + return; } - - printTierSummary(args.tier, results); - cleanup(); - process.exit(results.some(r => r.code !== 0) ? 1 : 0); - } - - const code = await runPlaywright( - ['playwright', 'test', '--grep', args.grep, ...args.passthrough], - childEnv - ); - cleanup(); - // Preserve the child's exit status so CI/other callers see the real result. - process.exit(code); - } catch (err) { - console.error('Failed to start Playwright:', err); - cleanup(); - process.exit(1); - } + console.log('\nDeleting temporary simulators...'); + const deleted = deleteSimulators(simulators.map(s => s.udid)); + console.log(`✓ Deleted ${deleted} simulator(s)`); + }, + }; } -void main(); +void runParallelSuite( + { + platform: 'ios', + deviceNoun: 'simulator', + defaultGrep: DEFAULT_GREP, + workersEnvVar: 'PLAYWRIGHT_WORKERS_COUNT_IOS', + maxDevices: MAX_SIMULATORS, + tiers: PARALLEL_TIERS, + tierNames: PARALLEL_TIER_NAMES, + tiersPreamble: 'Available tiers (see run/constants/parallelism.ts for the measurements):', + defaults: { + workers: 2, + devicesPerWorker: 2, + grep: DEFAULT_GREP, + keep: false, + listTiers: false, + explicitPools: false, + passthrough: [], + }, + extraFlags: { + boolean: { '--keep': args => void (args.keep = true) }, + value: { '--runtime': (args, value) => void (args.runtime = value) }, + }, + prepareDevices: createPool, + }, + process.argv.slice(2) +);