From fbdc7d56976d1b924090a862b46924d553c09bd5 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 11 May 2026 10:29:38 +0200 Subject: [PATCH 01/15] feat: scaffold macOS native capture pipeline --- .../macos-native-recorder-roadmap.md | 208 ++++++++++++++++++ electron/native/README.md | 27 +++ package.json | 1 + .../build-macos-screencapturekit-helper.mjs | 13 ++ src/lib/nativeMacRecording.test.ts | 26 +++ src/lib/nativeMacRecording.ts | 82 +++++++ 6 files changed, 357 insertions(+) create mode 100644 docs/engineering/macos-native-recorder-roadmap.md create mode 100644 scripts/build-macos-screencapturekit-helper.mjs create mode 100644 src/lib/nativeMacRecording.test.ts create mode 100644 src/lib/nativeMacRecording.ts diff --git a/docs/engineering/macos-native-recorder-roadmap.md b/docs/engineering/macos-native-recorder-roadmap.md new file mode 100644 index 000000000..469401bb1 --- /dev/null +++ b/docs/engineering/macos-native-recorder-roadmap.md @@ -0,0 +1,208 @@ +# macOS Native Recorder Roadmap + +OpenScreen's macOS recorder should follow the same architecture boundaries as the Windows native recorder: Electron owns session orchestration and persistence, while a platform-native helper owns capture, timing, encoding, and platform-specific permissions. + +This work is intentionally scoped as a macOS-only port. Windows native capture remains owned by the WGC helper, and Linux remains on the existing Electron path. + +## Goals + +- Capture displays and windows through ScreenCaptureKit. +- Exclude the real system cursor during capture when using the editable OpenScreen cursor overlay. +- Preserve the current high-quality cursor overlay path in preview and export. +- Capture macOS system audio through ScreenCaptureKit on supported macOS versions. +- Capture microphone audio through the same native timing domain where the OS supports it, or through an explicit companion path until it can be moved into the helper. +- Mix system audio and microphone audio into the primary MP4 without renderer-side track assembly. +- Capture webcam video natively and compose it into the helper-owned MP4 during the native-recording migration. +- Keep screen video, audio, webcam, and cursor aligned to one native timing origin. +- Package per-architecture helper binaries with macOS builds. + +## Non-Goals + +- Replacing the editor/export pipeline. +- Changing Windows native capture behavior. +- Adding Linux native capture. +- Shipping a silent fallback from native macOS capture to Electron capture when the user explicitly requested a native-only feature. + +## Architecture + +The renderer keeps the existing recording controls. On macOS, `useScreenRecorder` should eventually send a complete recording request to Electron instead of assembling display, audio, microphone, webcam, and cursor streams in the browser. + +Electron owns the native recording session: + +- resolves the selected display/window source; +- resolves output paths; +- starts cursor telemetry capture when editable cursor mode is selected; +- starts the ScreenCaptureKit helper process; +- sends pause/resume/stop/cancel commands; +- writes `RecordingSession` manifests; +- reports explicit errors when a macOS-native capability is unavailable. + +The helper owns macOS media capture: + +- ScreenCaptureKit display/window frames; +- ScreenCaptureKit system audio where supported; +- microphone capture or helper-owned companion audio capture; +- webcam capture and initial picture-in-picture composition; +- AVFoundation/VideoToolbox encoding and muxing; +- stream timestamp normalization. + +## Helper Contract V1 + +The helper receives a single JSON argument: + +```json +{ + "schemaVersion": 1, + "recordingId": 1234567890, + "source": { + "type": "display", + "sourceId": "screen:0:0", + "displayId": 1, + "windowId": null, + "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 } + }, + "video": { + "fps": 60, + "width": 1920, + "height": 1080, + "bitrate": 18000000, + "hideSystemCursor": true + }, + "audio": { + "system": { "enabled": true }, + "microphone": { + "enabled": true, + "deviceId": "default", + "deviceName": "MacBook Pro Microphone", + "gain": 1.4 + } + }, + "webcam": { + "enabled": true, + "deviceId": "default", + "deviceName": "FaceTime HD Camera", + "width": 1280, + "height": 720, + "fps": 30 + }, + "cursor": { + "mode": "editable-overlay" + }, + "outputs": { + "screenPath": "/Users/me/Library/Application Support/openscreen/recordings/recording-123.mp4", + "manifestPath": "/Users/me/Library/Application Support/openscreen/recordings/recording-123.session.json" + } +} +``` + +The helper emits newline-delimited JSON events to stdout: + +```json +{ "event": "ready", "schemaVersion": 1 } +{ "event": "recording-started", "timestampMs": 1234567890 } +{ "event": "warning", "code": "microphone-unavailable", "message": "..." } +{ "event": "recording-stopped", "screenPath": "..." } +{ "event": "error", "code": "screen-permission-denied", "message": "..." } +``` + +## Implementation Phases + +### 1. Native Session Boundary + +- Add a structured macOS native recording request type. +- Add a macOS helper resolver and build script placeholders. +- Keep the helper contract process-based, matching the Windows helper boundary. +- Do not route production macOS recording through this helper until the helper is available and validated. + +Acceptance: + +- TypeScript build passes. +- The macOS helper path and request contract are documented and testable without affecting Windows/Linux behavior. + +### 2. ScreenCaptureKit Display Capture + +- Implement a Swift helper using ScreenCaptureKit. +- Select display captures by `displayId`. +- Encode H.264 MP4 through AVFoundation/VideoToolbox. +- Set `showsCursor = false` when editable cursor overlay mode is selected. + +Acceptance: + +- Display-only recording produces a valid MP4. +- The real cursor is not baked into editable-cursor recordings. + +### 3. ScreenCaptureKit Window Capture + +- Resolve Electron `window:*` selections to ScreenCaptureKit window ids. +- Capture `SCContentFilter(desktopIndependentWindow:)`. +- Handle closed/minimized/protected windows with explicit errors. +- Keep window selection and capture source resolution in Electron/main, not the renderer. + +Acceptance: + +- Capturing a normal app window works with cursor/audio/webcam disabled. +- Unsupported windows return clear native errors. + +### 4. System Audio + +- Enable ScreenCaptureKit system audio on supported macOS versions. +- Keep audio format and timing owned by the helper. +- Encode or mux AAC audio into the primary MP4. + +Acceptance: + +- System-audio-only recordings produce a valid AAC track. +- Unsupported macOS versions return an explicit capability error. + +### 5. Microphone + +- Resolve the selected microphone device from the renderer-provided browser `deviceId` and user-visible label. +- Capture microphone audio in the helper timing domain. +- Apply OpenScreen microphone gain policy. +- Mix system and microphone audio before final AAC output. + +Acceptance: + +- Mic-only and mic-plus-system recordings produce a valid, balanced AAC track. +- Device selection honors the selected microphone, not only the default device. + +### 6. Webcam Composition + +- Capture the selected camera natively through AVFoundation. +- Match browser device id first where possible, then user-visible label. +- Compose an initial picture-in-picture overlay into the primary MP4. +- Hide webcam output until the first usable frame to avoid black startup flashes. + +Acceptance: + +- Native display/window recordings can include webcam without returning to Electron capture. +- Selected camera is honored. + +### 7. Runtime Controls + +- Add pause/resume commands to the helper. +- Add cancel command that removes partial outputs. +- Keep restart as stop-discard-start until the helper exposes a native restart operation. + +Acceptance: + +- Pause/resume keeps output duration coherent. +- Cancel leaves no stale media/session files. + +### 8. Test Pipeline + +- `npm run build:native:mac`: builds Swift helper binaries on macOS. +- `npm run test:sck-helper:mac`: display-only helper smoke test. +- `npm run test:sck-window:mac`: window capture smoke test. +- `npm run test:sck-audio:mac`: system audio smoke test when supported. +- `npm run test:sck-mic:mac`: microphone smoke test. +- `npm run test:sck-webcam:mac`: webcam smoke test when a webcam is available. +- Packaging check: confirms helpers are available under `electron/native/bin/darwin-${arch}` in packaged builds. + +## SSOT Rules + +- `src/lib/nativeMacRecording.ts` is the renderer/main TypeScript request contract. +- This document is the feature-level contract and phase checklist. +- The Swift helper owns ScreenCaptureKit/AVFoundation media timing. +- Electron owns output paths, session manifests, and selected source/device resolution. +- Renderer code must use existing hooks/client APIs and should not bind directly to helper process details. diff --git a/electron/native/README.md b/electron/native/README.md index 659829cd9..8d6ea4653 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -1,5 +1,32 @@ # Native capture helpers +## macOS + +macOS native recording will use a ScreenCaptureKit helper with the same process boundary as the Windows WGC helper: + +1. Electron resolves the selected source, output paths, and user-selected devices. +2. The helper receives one structured JSON request. +3. The helper owns ScreenCaptureKit/AVFoundation capture, timing, encoding, and muxing. +4. Electron persists the resulting media/session manifest and reports helper errors explicitly. + +Expected development helper locations: + +1. `OPENSCREEN_SCK_CAPTURE_EXE`, for local development and diagnostics. +2. `electron/native/screencapturekit/build/openscreen-screencapturekit-helper`, for locally built Swift output. +3. `electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper` or `electron/native/bin/darwin-x64/openscreen-screencapturekit-helper`, for packaged prebuilt helpers. + +The current macOS helper script is a placeholder: + +```bash +npm run build:native:mac +``` + +On non-macOS hosts this command exits successfully and does not affect Windows/Linux development. On macOS it fails until the Swift ScreenCaptureKit helper lands. + +See `docs/engineering/macos-native-recorder-roadmap.md` for the contract, rollout phases, and SSOT rules. + +## Windows + Windows native recording is resolved from one of these locations: 1. `OPENSCREEN_WGC_CAPTURE_EXE`, for local development and diagnostics. diff --git a/package.json b/package.json index 9388bcd31..12d02d717 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "format": "biome format --write .", "i18n:check": "node scripts/i18n-check.mjs", "preview": "vite preview", + "build:native:mac": "node scripts/build-macos-screencapturekit-helper.mjs", "build:mac": "tsc && vite build && electron-builder --mac", "build:native:win": "node scripts/build-windows-wgc-helper.mjs", "build:win": "npm run build:native:win && tsc && vite build && electron-builder --win --config.npmRebuild=false", diff --git a/scripts/build-macos-screencapturekit-helper.mjs b/scripts/build-macos-screencapturekit-helper.mjs new file mode 100644 index 000000000..5e9292f67 --- /dev/null +++ b/scripts/build-macos-screencapturekit-helper.mjs @@ -0,0 +1,13 @@ +#!/usr/bin/env node + +import process from "node:process"; + +if (process.platform !== "darwin") { + console.log("Skipping macOS ScreenCaptureKit helper build: host platform is not macOS."); + process.exit(0); +} + +console.error( + "macOS ScreenCaptureKit helper sources are not implemented yet. See docs/engineering/macos-native-recorder-roadmap.md.", +); +process.exit(1); diff --git a/src/lib/nativeMacRecording.test.ts b/src/lib/nativeMacRecording.test.ts new file mode 100644 index 000000000..fce88f6a7 --- /dev/null +++ b/src/lib/nativeMacRecording.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { parseMacDisplayIdFromSourceId, parseMacWindowIdFromSourceId } from "./nativeMacRecording"; + +describe("nativeMacRecording source parsing", () => { + it("parses Electron window source ids into ScreenCaptureKit window ids", () => { + expect(parseMacWindowIdFromSourceId("window:12345:0")).toBe(12345); + expect(parseMacWindowIdFromSourceId("window:987")).toBe(987); + }); + + it("rejects non-window source ids for window parsing", () => { + expect(parseMacWindowIdFromSourceId("screen:1:0")).toBeNull(); + expect(parseMacWindowIdFromSourceId("window:not-a-number:0")).toBeNull(); + expect(parseMacWindowIdFromSourceId(null)).toBeNull(); + }); + + it("parses Electron display source ids into ScreenCaptureKit display ids", () => { + expect(parseMacDisplayIdFromSourceId("screen:1:0")).toBe(1); + expect(parseMacDisplayIdFromSourceId("screen:69733248")).toBe(69733248); + }); + + it("rejects non-display source ids for display parsing", () => { + expect(parseMacDisplayIdFromSourceId("window:123:0")).toBeNull(); + expect(parseMacDisplayIdFromSourceId("screen:not-a-number:0")).toBeNull(); + expect(parseMacDisplayIdFromSourceId(undefined)).toBeNull(); + }); +}); diff --git a/src/lib/nativeMacRecording.ts b/src/lib/nativeMacRecording.ts new file mode 100644 index 000000000..0e596c6dd --- /dev/null +++ b/src/lib/nativeMacRecording.ts @@ -0,0 +1,82 @@ +import type { Rectangle } from "electron"; +import type { CursorCaptureMode } from "./recordingSession"; + +export type NativeMacSourceType = "display" | "window"; + +export type NativeMacRecordingRequest = { + recordingId?: number; + source: { + type: NativeMacSourceType; + sourceId: string; + displayId?: number; + windowId?: number; + bounds?: Rectangle; + }; + video: { + fps: number; + width: number; + height: number; + bitrate?: number; + hideSystemCursor: boolean; + }; + audio: { + system: { + enabled: boolean; + }; + microphone: { + enabled: boolean; + deviceId?: string; + deviceName?: string; + gain: number; + }; + }; + webcam: { + enabled: boolean; + deviceId?: string; + deviceName?: string; + width: number; + height: number; + fps: number; + }; + cursor: { + mode: CursorCaptureMode; + }; + outputs: { + screenPath: string; + manifestPath?: string; + }; +}; + +export type NativeMacRecordingStartResult = { + success: boolean; + recordingId?: number; + path?: string; + helperPath?: string; + error?: string; +}; + +export function parseMacWindowIdFromSourceId(sourceId?: string | null) { + if (!sourceId?.startsWith("window:")) { + return null; + } + + const windowIdPart = sourceId.split(":")[1]; + if (!windowIdPart || !/^\d+$/.test(windowIdPart)) { + return null; + } + + return Number(windowIdPart); +} + +export function parseMacDisplayIdFromSourceId(sourceId?: string | null) { + if (!sourceId?.startsWith("screen:")) { + return null; + } + + const displayIdPart = sourceId.split(":")[1]; + if (!displayIdPart || !/^\d+$/.test(displayIdPart)) { + return null; + } + + return Number(displayIdPart); +} From 7102110de5391878bd4a27b7fa688a32159735e0 Mon Sep 17 00:00:00 2001 From: Etienne Date: Tue, 12 May 2026 08:14:54 +0200 Subject: [PATCH 02/15] chore: ignore macos native build outputs --- .gitignore | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 84a5a1cea..0861e392d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,8 +16,16 @@ dist-ssr # Native helper build outputs /electron/native/wgc-capture/build/ +/electron/native/screencapturekit/build/ +/electron/native/screencapturekit/.build/ +/electron/native/screencapturekit/.swiftpm/ /electron/native/bin/ +# Native macOS generated files +DerivedData/ +*.xcuserstate +xcuserdata/ + # Editor directories and files .vscode/* .zed/ @@ -49,4 +57,4 @@ result-* .direnv/ #kilocode -.kilo/ \ No newline at end of file +.kilo/ From b9e213474993e23237235bcc6cf6caab828543b2 Mon Sep 17 00:00:00 2001 From: Etienne Date: Tue, 12 May 2026 08:33:18 +0200 Subject: [PATCH 03/15] feat: add macos screencapturekit helper --- electron/electron-env.d.ts | 18 + electron/ipc/handlers.ts | 365 ++++++++++++++++++ electron/native/README.md | 10 +- .../native/screencapturekit/Package.swift | 22 ++ .../main.swift | 355 +++++++++++++++++ electron/preload.ts | 10 + .../build-macos-screencapturekit-helper.mjs | 68 +++- src/components/launch/LaunchWindow.tsx | 8 +- src/hooks/useScreenRecorder.ts | 179 ++++++++- src/lib/nativeMacRecording.ts | 35 ++ vite.config.ts | 5 + 11 files changed, 1064 insertions(+), 11 deletions(-) create mode 100644 electron/native/screencapturekit/Package.swift create mode 100644 electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index e92ce19fc..7ff73bcc9 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -78,6 +78,13 @@ interface Window { reason?: string; error?: string; }>; + isNativeMacCaptureAvailable: () => Promise<{ + success: boolean; + available: boolean; + helperPath?: string; + reason?: "unsupported-platform" | "missing-helper" | string; + error?: string; + }>; startNativeWindowsRecording: ( request: import("../src/lib/nativeWindowsRecording").NativeWindowsRecordingRequest, ) => Promise; @@ -89,6 +96,17 @@ interface Window { discarded?: boolean; error?: string; }>; + startNativeMacRecording: ( + request: import("../src/lib/nativeMacRecording").NativeMacRecordingRequest, + ) => Promise; + stopNativeMacRecording: (discard?: boolean) => Promise<{ + success: boolean; + path?: string; + session?: import("../src/lib/recordingSession").RecordingSession; + message?: string; + discarded?: boolean; + error?: string; + }>; discardCursorTelemetry: (recordingId: number) => Promise; getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 9797c95c0..68d607760 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -15,6 +15,7 @@ import { shell, systemPreferences, } from "electron"; +import type { NativeMacRecordingRequest } from "../../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording"; import { type CursorCaptureMode, @@ -276,6 +277,12 @@ let nativeWindowsCaptureRecordingId: number | null = null; let nativeWindowsCursorOffsetMs = 0; let nativeWindowsCursorCaptureMode: CursorCaptureMode = "editable-overlay"; const NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS = 15_000; +let nativeMacCaptureProcess: ChildProcessWithoutNullStreams | null = null; +let nativeMacCaptureOutput = ""; +let nativeMacCaptureTargetPath: string | null = null; +let nativeMacCaptureRecordingId: number | null = null; +let nativeMacCursorOffsetMs = 0; +let nativeMacCursorCaptureMode: CursorCaptureMode = "editable-overlay"; function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { if (!sample || typeof sample !== "object") { @@ -499,6 +506,35 @@ async function findNativeWindowsCaptureHelperPath() { return null; } +function getNativeMacCaptureHelperCandidates() { + const envPath = process.env.OPENSCREEN_SCK_CAPTURE_EXE?.trim(); + const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; + const helperName = "openscreen-screencapturekit-helper"; + return [ + envPath, + resolveUnpackedAppPath("electron", "native", "screencapturekit", "build", helperName), + resolveUnpackedAppPath("electron", "native", "bin", archTag, helperName), + resolvePackagedResourcePath("electron", "native", "bin", archTag, helperName), + ].filter((candidate): candidate is string => Boolean(candidate)); +} + +async function findNativeMacCaptureHelperPath() { + if (process.platform !== "darwin") { + return null; + } + + for (const candidate of getNativeMacCaptureHelperCandidates()) { + try { + await fs.access(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next configured helper location. + } + } + + return null; +} + function isWindowsGraphicsCaptureOsSupported() { if (process.platform !== "win32") { return false; @@ -785,6 +821,134 @@ function readNativeWindowsWebcamFormat(output: string) { } } +function tryParseNativeHelperEvent(line: string) { + try { + const parsed = JSON.parse(line); + return parsed && typeof parsed === "object" ? parsed : null; + } catch { + return null; + } +} + +function waitForNativeMacCaptureStart(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(new Error("Timed out waiting for native macOS capture to start")); + }, 10_000); + + const inspect = (chunk: Buffer) => { + nativeMacCaptureOutput += chunk.toString(); + for (const line of nativeMacCaptureOutput.split(/\r?\n/)) { + const event = tryParseNativeHelperEvent(line.trim()); + if (!event) continue; + if (event.event === "recording-started") { + cleanup(); + resolve(); + return; + } + if (event.event === "error") { + cleanup(); + reject(new Error(event.message ?? event.code ?? "Native macOS capture failed")); + return; + } + } + }; + + const onOutput = (chunk: Buffer) => inspect(chunk); + const onClose = (code: number | null) => { + cleanup(); + reject( + new Error( + nativeMacCaptureOutput.trim() || + `Native macOS capture exited before recording started (code=${code ?? "unknown"})`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const cleanup = () => { + clearTimeout(timer); + proc.stdout.off("data", onOutput); + proc.stderr.off("data", onOutput); + proc.off("close", onClose); + proc.off("error", onError); + }; + + proc.stdout.on("data", onOutput); + proc.stderr.on("data", onOutput); + proc.once("close", onClose); + proc.once("error", onError); + }); +} + +function waitForNativeMacCaptureStop(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject( + new Error( + `Timed out waiting for native macOS capture to stop. Output path: ${ + nativeMacCaptureTargetPath ?? "unknown" + }. Output: ${nativeMacCaptureOutput.trim()}`, + ), + ); + }, 30_000); + + const inspect = (chunk: Buffer) => { + nativeMacCaptureOutput += chunk.toString(); + for (const line of nativeMacCaptureOutput.split(/\r?\n/)) { + const event = tryParseNativeHelperEvent(line.trim()); + if (!event) continue; + if (event.event === "recording-stopped") { + cleanup(); + resolve(event.screenPath ?? nativeMacCaptureTargetPath ?? ""); + return; + } + if (event.event === "error") { + cleanup(); + reject(new Error(event.message ?? event.code ?? "Native macOS capture failed")); + return; + } + } + }; + + const onOutput = (chunk: Buffer) => inspect(chunk); + const onClose = (code: number | null) => { + if (code === 0 && nativeMacCaptureTargetPath) { + cleanup(); + resolve(nativeMacCaptureTargetPath); + return; + } + cleanup(); + reject( + new Error( + nativeMacCaptureOutput.trim() || + `Native macOS capture exited with code=${code ?? "unknown"}`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const cleanup = () => { + clearTimeout(timer); + proc.stdout.off("data", onOutput); + proc.stderr.off("data", onOutput); + proc.off("close", onClose); + proc.off("error", onError); + }; + + proc.stdout.on("data", onOutput); + proc.stderr.on("data", onOutput); + proc.once("close", onClose); + proc.once("error", onError); + }); +} + function setCurrentRecordingSessionState(session: RecordingSession | null) { currentRecordingSession = session; currentVideoPath = session?.screenVideoPath ?? null; @@ -1041,6 +1205,17 @@ export function registerIpcHandlers( : { success: true, available: false, reason: "missing-helper" }; }); + ipcMain.handle("is-native-mac-capture-available", async () => { + if (process.platform !== "darwin") { + return { success: true, available: false, reason: "unsupported-platform" }; + } + + const helperPath = await findNativeMacCaptureHelperPath(); + return helperPath + ? { success: true, available: true, helperPath } + : { success: true, available: false, reason: "missing-helper" }; + }); + ipcMain.handle( "start-native-windows-recording", async (_, request: NativeWindowsRecordingRequest) => { @@ -1217,6 +1392,121 @@ export function registerIpcHandlers( }, ); + ipcMain.handle("start-native-mac-recording", async (_, request: NativeMacRecordingRequest) => { + try { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS capture requires macOS." }; + } + if (nativeMacCaptureProcess) { + return { success: false, error: "Native macOS capture is already running." }; + } + + const helperPath = await findNativeMacCaptureHelperPath(); + if (!helperPath) { + return { success: false, error: "Native macOS capture helper is not available." }; + } + + if (!request?.source?.sourceId) { + return { success: false, error: "Native macOS capture request is missing a source." }; + } + + const recordingId = + typeof request.recordingId === "number" && Number.isFinite(request.recordingId) + ? request.recordingId + : Date.now(); + const outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); + const cursorCaptureMode = + normalizeCursorCaptureMode(request.cursor?.mode) ?? "editable-overlay"; + const sourceDisplay = + request.source.type === "display" && typeof request.source.displayId === "number" + ? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ?? + null) + : getSelectedDisplay(); + const bounds = request.source.bounds ?? sourceDisplay?.bounds ?? getSelectedSourceBounds(); + const config: NativeMacRecordingRequest = { + ...request, + schemaVersion: 1, + recordingId, + source: { + ...request.source, + bounds, + }, + video: { + ...request.video, + hideSystemCursor: cursorCaptureMode === "editable-overlay", + }, + cursor: { + mode: cursorCaptureMode, + }, + outputs: { + screenPath: outputPath, + manifestPath: path.join( + RECORDINGS_DIR, + `${RECORDING_FILE_PREFIX}${recordingId}${RECORDING_SESSION_SUFFIX}`, + ), + }, + }; + + console.info("[native-sck] starting macOS capture", { + helperPath, + source: config.source, + audio: config.audio, + webcam: config.webcam, + cursor: config.cursor, + outputPath, + }); + + await fs.mkdir(RECORDINGS_DIR, { recursive: true }); + nativeMacCaptureOutput = ""; + nativeMacCaptureTargetPath = outputPath; + nativeMacCaptureRecordingId = recordingId; + nativeMacCursorOffsetMs = 0; + nativeMacCursorCaptureMode = cursorCaptureMode; + + const cursorStartTimeMs = Date.now(); + if (cursorCaptureMode === "editable-overlay") { + await startCursorRecording(cursorStartTimeMs); + } else { + pendingCursorRecordingData = null; + } + + const proc = spawn(helperPath, [JSON.stringify(config)], { + cwd: RECORDINGS_DIR, + stdio: ["pipe", "pipe", "pipe"], + }); + nativeMacCaptureProcess = proc; + + await waitForNativeMacCaptureStart(proc); + const captureStartedAtMs = Date.now(); + nativeMacCursorOffsetMs = + cursorCaptureMode === "editable-overlay" + ? Math.max(0, captureStartedAtMs - cursorStartTimeMs) + : 0; + + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(true, source.name); + } + + return { + success: true, + recordingId, + path: outputPath, + helperPath, + }; + } catch (error) { + console.error("Failed to start native macOS recording:", error); + nativeMacCaptureProcess?.kill(); + nativeMacCaptureProcess = null; + nativeMacCaptureTargetPath = null; + nativeMacCaptureRecordingId = null; + nativeMacCursorOffsetMs = 0; + nativeMacCursorCaptureMode = "editable-overlay"; + await stopCursorRecording(); + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } + }); + ipcMain.handle("stop-native-windows-recording", async (_, discard?: boolean) => { const proc = nativeWindowsCaptureProcess; const preferredPath = nativeWindowsCaptureTargetPath; @@ -1301,6 +1591,81 @@ export function registerIpcHandlers( } }); + ipcMain.handle("stop-native-mac-recording", async (_, discard?: boolean) => { + const proc = nativeMacCaptureProcess; + const preferredPath = nativeMacCaptureTargetPath; + const recordingId = nativeMacCaptureRecordingId ?? Date.now(); + const cursorCaptureMode = nativeMacCursorCaptureMode; + + if (!proc) { + return { success: false, error: "Native macOS capture is not running." }; + } + + try { + const stoppedPathPromise = waitForNativeMacCaptureStop(proc); + proc.stdin.write("stop\n"); + const stoppedPath = await stoppedPathPromise; + const screenVideoPath = stoppedPath || preferredPath; + if (!screenVideoPath) { + throw new Error("Native macOS capture did not return an output path."); + } + + if (cursorCaptureMode === "editable-overlay") { + await stopCursorRecording(); + } else { + pendingCursorRecordingData = null; + } + if (discard) { + pendingCursorRecordingData = null; + await Promise.all([ + fs.rm(screenVideoPath, { force: true }), + fs.rm(`${screenVideoPath}.cursor.json`, { force: true }), + ]); + return { success: true, discarded: true }; + } + + if (cursorCaptureMode === "editable-overlay") { + shiftPendingCursorTelemetry(nativeMacCursorOffsetMs); + await writePendingCursorTelemetry(screenVideoPath); + } + + const session: RecordingSession = { + screenVideoPath, + createdAt: recordingId, + cursorCaptureMode, + }; + setCurrentRecordingSessionState(session); + currentProjectPath = null; + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(screenVideoPath).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + + return { + success: true, + path: screenVideoPath, + session, + message: "Native macOS recording session stored successfully", + }; + } catch (error) { + console.error("Failed to stop native macOS recording:", error); + await stopCursorRecording(); + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } finally { + nativeMacCaptureProcess = null; + nativeMacCaptureTargetPath = null; + nativeMacCaptureRecordingId = null; + nativeMacCursorOffsetMs = 0; + nativeMacCursorCaptureMode = "editable-overlay"; + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(false, source.name); + } + } + }); + ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => { try { return await storeRecordedSessionFiles(payload); diff --git a/electron/native/README.md b/electron/native/README.md index 8d6ea4653..2810f6b86 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -9,19 +9,23 @@ macOS native recording will use a ScreenCaptureKit helper with the same process 3. The helper owns ScreenCaptureKit/AVFoundation capture, timing, encoding, and muxing. 4. Electron persists the resulting media/session manifest and reports helper errors explicitly. -Expected development helper locations: +Helper locations: 1. `OPENSCREEN_SCK_CAPTURE_EXE`, for local development and diagnostics. 2. `electron/native/screencapturekit/build/openscreen-screencapturekit-helper`, for locally built Swift output. 3. `electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper` or `electron/native/bin/darwin-x64/openscreen-screencapturekit-helper`, for packaged prebuilt helpers. -The current macOS helper script is a placeholder: +Build the macOS helper with: ```bash npm run build:native:mac ``` -On non-macOS hosts this command exits successfully and does not affect Windows/Linux development. On macOS it fails until the Swift ScreenCaptureKit helper lands. +On non-macOS hosts this command exits successfully and does not affect Windows/Linux development. On macOS it builds the Swift package at `electron/native/screencapturekit`, writes the development binary to `electron/native/screencapturekit/build/openscreen-screencapturekit-helper`, and copies the redistributable binary to `electron/native/bin/darwin-${arch}/openscreen-screencapturekit-helper`. + +The current helper implementation supports the first native media slice: display/window ScreenCaptureKit video capture, cursor exclusion through `SCStreamConfiguration.showsCursor`, H.264 encoding, and MP4 muxing. System audio, microphone capture, webcam composition, and runtime controls are intentionally left as explicit roadmap phases. + +Electron exposes `is-native-mac-capture-available` for capability probing. It resolves the same helper locations listed above and reports `missing-helper` until a Swift helper binary is present; production recording is not routed through the macOS helper yet. See `docs/engineering/macos-native-recorder-roadmap.md` for the contract, rollout phases, and SSOT rules. diff --git a/electron/native/screencapturekit/Package.swift b/electron/native/screencapturekit/Package.swift new file mode 100644 index 000000000..f040dd25f --- /dev/null +++ b/electron/native/screencapturekit/Package.swift @@ -0,0 +1,22 @@ +// swift-tools-version: 5.9 + +import PackageDescription + +let package = Package( + name: "OpenScreenScreenCaptureKitHelper", + platforms: [ + .macOS(.v13) + ], + products: [ + .executable( + name: "openscreen-screencapturekit-helper", + targets: ["OpenScreenScreenCaptureKitHelper"] + ) + ], + targets: [ + .executableTarget( + name: "OpenScreenScreenCaptureKitHelper", + path: "Sources/OpenScreenScreenCaptureKitHelper" + ) + ] +) diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift new file mode 100644 index 000000000..cf83eec54 --- /dev/null +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift @@ -0,0 +1,355 @@ +import AVFoundation +import CoreMedia +import Foundation +import ScreenCaptureKit + +struct Rectangle: Decodable { + let x: Double + let y: Double + let width: Double + let height: Double +} + +struct RecordingRequest: Decodable { + struct Source: Decodable { + let type: String + let sourceId: String + let displayId: UInt32? + let windowId: UInt32? + let bounds: Rectangle? + } + + struct Video: Decodable { + let fps: Int + let width: Int + let height: Int + let bitrate: Int? + let hideSystemCursor: Bool + } + + struct Audio: Decodable { + struct SystemAudio: Decodable { + let enabled: Bool + } + + struct Microphone: Decodable { + let enabled: Bool + let deviceId: String? + let deviceName: String? + let gain: Double + } + + let system: SystemAudio + let microphone: Microphone + } + + struct Webcam: Decodable { + let enabled: Bool + let deviceId: String? + let deviceName: String? + let width: Int + let height: Int + let fps: Int + } + + struct Cursor: Decodable { + let mode: String + } + + struct Outputs: Decodable { + let screenPath: String + let manifestPath: String? + } + + let schemaVersion: Int? + let recordingId: Int? + let source: Source + let video: Video + let audio: Audio + let webcam: Webcam + let cursor: Cursor + let outputs: Outputs +} + +enum HelperError: Error, CustomStringConvertible { + case invalidArguments + case unsupportedMacOS + case unsupportedFeature(String) + case sourceNotFound(String) + case invalidSourceType(String) + case writerSetupFailed(String) + + var description: String { + switch self { + case .invalidArguments: + return "Expected one JSON recording request argument." + case .unsupportedMacOS: + return "ScreenCaptureKit recording requires macOS 13 or newer." + case .unsupportedFeature(let message): + return message + case .sourceNotFound(let message): + return message + case .invalidSourceType(let sourceType): + return "Unsupported source type: \(sourceType)." + case .writerSetupFailed(let message): + return message + } + } +} + +func emit(_ fields: [String: Any]) { + if let data = try? JSONSerialization.data(withJSONObject: fields, options: []), + let line = String(data: data, encoding: .utf8) + { + print(line) + fflush(stdout) + } +} + +func emitError(code: String, message: String) { + emit([ + "event": "error", + "code": code, + "message": message, + ]) +} + +@available(macOS 13.0, *) +final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { + private let request: RecordingRequest + private let sampleQueue = DispatchQueue(label: "app.openscreen.sck-helper.samples") + private let stateQueue = DispatchQueue(label: "app.openscreen.sck-helper.state") + private var stream: SCStream? + private var writer: AVAssetWriter? + private var videoInput: AVAssetWriterInput? + private var didStartWriting = false + private var isStopping = false + + init(request: RecordingRequest) { + self.request = request + } + + func start() async throws { + try rejectUnsupportedPhaseFeatures() + + let content = try await SCShareableContent.excludingDesktopWindows( + false, + onScreenWindowsOnly: true + ) + let filter = try makeContentFilter(from: content) + let configuration = makeStreamConfiguration() + let stream = SCStream(filter: filter, configuration: configuration, delegate: self) + + try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: sampleQueue) + try setupWriter() + + self.stream = stream + emit(["event": "ready", "schemaVersion": 1]) + try await stream.startCapture() + emit([ + "event": "recording-started", + "timestampMs": Int(Date().timeIntervalSince1970 * 1000), + ]) + } + + func stop() async { + let shouldStop = stateQueue.sync { + if isStopping { + return false + } + isStopping = true + return true + } + if !shouldStop { + return + } + + do { + try await stream?.stopCapture() + } catch { + emit([ + "event": "warning", + "code": "stop-capture-failed", + "message": "\(error)", + ]) + } + + await finishWriter() + } + + func stream(_ stream: SCStream, didStopWithError error: Error) { + emitError(code: "capture-stopped-with-error", message: "\(error)") + Task { + await stop() + } + } + + func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + guard type == .screen else { + return + } + guard CMSampleBufferDataIsReady(sampleBuffer) else { + return + } + guard let videoInput, let writer else { + return + } + + let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + if !didStartWriting { + writer.startWriting() + writer.startSession(atSourceTime: presentationTime) + didStartWriting = true + } + + if videoInput.isReadyForMoreMediaData { + videoInput.append(sampleBuffer) + } + } + + private func rejectUnsupportedPhaseFeatures() throws { + if request.audio.system.enabled { + throw HelperError.unsupportedFeature( + "System audio capture is planned for the roadmap system-audio phase." + ) + } + if request.audio.microphone.enabled { + throw HelperError.unsupportedFeature( + "Microphone capture is planned for the roadmap microphone phase." + ) + } + if request.webcam.enabled { + throw HelperError.unsupportedFeature( + "Webcam composition is planned for the roadmap webcam phase." + ) + } + } + + private func makeContentFilter(from content: SCShareableContent) throws -> SCContentFilter { + switch request.source.type { + case "display": + guard let displayId = request.source.displayId else { + throw HelperError.sourceNotFound("Display capture requires source.displayId.") + } + guard let display = content.displays.first(where: { $0.displayID == displayId }) else { + throw HelperError.sourceNotFound("No ScreenCaptureKit display found for id \(displayId).") + } + return SCContentFilter(display: display, excludingWindows: []) + case "window": + guard let windowId = request.source.windowId else { + throw HelperError.sourceNotFound("Window capture requires source.windowId.") + } + guard let window = content.windows.first(where: { $0.windowID == windowId }) else { + throw HelperError.sourceNotFound("No ScreenCaptureKit window found for id \(windowId).") + } + return SCContentFilter(desktopIndependentWindow: window) + default: + throw HelperError.invalidSourceType(request.source.type) + } + } + + private func makeStreamConfiguration() -> SCStreamConfiguration { + let configuration = SCStreamConfiguration() + configuration.width = request.video.width + configuration.height = request.video.height + configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(max(1, request.video.fps))) + configuration.queueDepth = 6 + configuration.showsCursor = !request.video.hideSystemCursor + configuration.pixelFormat = kCVPixelFormatType_32BGRA + return configuration + } + + private func setupWriter() throws { + let outputUrl = URL(fileURLWithPath: request.outputs.screenPath) + try? FileManager.default.removeItem(at: outputUrl) + try FileManager.default.createDirectory( + at: outputUrl.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + + let writer = try AVAssetWriter(outputURL: outputUrl, fileType: .mp4) + let settings: [String: Any] = [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: request.video.width, + AVVideoHeightKey: request.video.height, + AVVideoCompressionPropertiesKey: [ + AVVideoAverageBitRateKey: request.video.bitrate ?? 18_000_000, + AVVideoExpectedSourceFrameRateKey: request.video.fps, + ], + ] + let input = AVAssetWriterInput(mediaType: .video, outputSettings: settings) + input.expectsMediaDataInRealTime = true + + guard writer.canAdd(input) else { + throw HelperError.writerSetupFailed("Unable to add H.264 video input to AVAssetWriter.") + } + + writer.add(input) + self.writer = writer + self.videoInput = input + } + + private func finishWriter() async { + guard let writer else { + return + } + + videoInput?.markAsFinished() + + await withCheckedContinuation { continuation in + writer.finishWriting { + continuation.resume() + } + } + + if writer.status == .completed { + emit([ + "event": "recording-stopped", + "screenPath": request.outputs.screenPath, + ]) + } else { + emitError( + code: "writer-failed", + message: writer.error.map { "\($0)" } ?? "AVAssetWriter failed with status \(writer.status.rawValue)." + ) + } + } +} + +@main +struct OpenScreenScreenCaptureKitHelper { + static func main() async { + do { + guard CommandLine.arguments.count == 2 else { + throw HelperError.invalidArguments + } + + guard #available(macOS 13.0, *) else { + throw HelperError.unsupportedMacOS + } + + let requestData = Data(CommandLine.arguments[1].utf8) + let decoder = JSONDecoder() + let request = try decoder.decode(RecordingRequest.self, from: requestData) + let recorder = ScreenCaptureRecorder(request: request) + let stopTask = Task.detached { + while let line = readLine() { + let command = line.trimmingCharacters(in: .whitespacesAndNewlines) + if command == "stop" { + await recorder.stop() + exit(0) + } + } + } + + try await recorder.start() + await stopTask.value + } catch let error as HelperError { + emitError(code: "helper-error", message: error.description) + exit(1) + } catch { + emitError(code: "helper-error", message: "\(error)") + exit(1) + } + } +} diff --git a/electron/preload.ts b/electron/preload.ts index 8302b959f..e50e8b3ba 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,4 +1,5 @@ import { contextBridge, ipcRenderer } from "electron"; +import type { NativeMacRecordingRequest } from "../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording"; import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession"; import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts"; @@ -68,12 +69,21 @@ contextBridge.exposeInMainWorld("electronAPI", { isNativeWindowsCaptureAvailable: () => { return ipcRenderer.invoke("is-native-windows-capture-available"); }, + isNativeMacCaptureAvailable: () => { + return ipcRenderer.invoke("is-native-mac-capture-available"); + }, startNativeWindowsRecording: (request: NativeWindowsRecordingRequest) => { return ipcRenderer.invoke("start-native-windows-recording", request); }, stopNativeWindowsRecording: (discard?: boolean) => { return ipcRenderer.invoke("stop-native-windows-recording", discard); }, + startNativeMacRecording: (request: NativeMacRecordingRequest) => { + return ipcRenderer.invoke("start-native-mac-recording", request); + }, + stopNativeMacRecording: (discard?: boolean) => { + return ipcRenderer.invoke("stop-native-mac-recording", discard); + }, getCursorTelemetry: (videoPath?: string) => { return ipcRenderer.invoke("get-cursor-telemetry", videoPath); }, diff --git a/scripts/build-macos-screencapturekit-helper.mjs b/scripts/build-macos-screencapturekit-helper.mjs index 5e9292f67..f5f0c82b0 100644 --- a/scripts/build-macos-screencapturekit-helper.mjs +++ b/scripts/build-macos-screencapturekit-helper.mjs @@ -1,13 +1,75 @@ #!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; import process from "node:process"; +import { fileURLToPath } from "node:url"; if (process.platform !== "darwin") { console.log("Skipping macOS ScreenCaptureKit helper build: host platform is not macOS."); process.exit(0); } -console.error( - "macOS ScreenCaptureKit helper sources are not implemented yet. See docs/engineering/macos-native-recorder-roadmap.md.", +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const root = path.resolve(__dirname, ".."); +const helperName = "openscreen-screencapturekit-helper"; +const packageDir = path.join(root, "electron", "native", "screencapturekit"); +const buildDir = path.join(packageDir, "build"); +const swiftBuildDir = path.join(buildDir, "swiftpm"); +const builtHelperPath = path.join(swiftBuildDir, "release", helperName); +const localHelperPath = path.join(buildDir, helperName); +const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; +const distributableDir = path.join(root, "electron", "native", "bin", archTag); +const distributablePath = path.join(distributableDir, helperName); + +const xcodebuildVersion = spawnSync("xcodebuild", ["-version"], { + cwd: root, + encoding: "utf8", +}); + +if (xcodebuildVersion.status !== 0) { + const message = `${xcodebuildVersion.stderr ?? ""}${xcodebuildVersion.stdout ?? ""}`.trim(); + console.error( + [ + "Unable to build the macOS ScreenCaptureKit helper because full Xcode is not active.", + "", + message, + "", + "Install Xcode from the App Store or Apple Developer downloads, then run:", + " sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer", + " sudo xcodebuild -license accept", + "", + "Command Line Tools alone may not include the Swift SDK/platform metadata required by SwiftPM.", + ].join("\n"), + ); + process.exit(1); +} + +const result = spawnSync( + "swift", + ["build", "-c", "release", "--package-path", packageDir, "--build-path", swiftBuildDir], + { + cwd: root, + stdio: "inherit", + }, ); -process.exit(1); + +if (result.error) { + console.error(`Failed to start Swift build: ${result.error.message}`); + process.exit(1); +} + +if (result.status !== 0) { + process.exit(result.status ?? 1); +} + +fs.mkdirSync(buildDir, { recursive: true }); +fs.mkdirSync(distributableDir, { recursive: true }); +fs.copyFileSync(builtHelperPath, localHelperPath); +fs.copyFileSync(builtHelperPath, distributablePath); +fs.chmodSync(localHelperPath, 0o755); +fs.chmodSync(distributablePath, 0o755); + +console.log(`Built macOS ScreenCaptureKit helper: ${localHelperPath}`); +console.log(`Copied redistributable helper: ${distributablePath}`); diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 056a4fd83..d4fa92897 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -127,7 +127,7 @@ export function LaunchWindow() { const [isWebcamFocused, setIsWebcamFocused] = useState(false); const webcamExpanded = isWebcamHovered || isWebcamFocused; const [isLanguageMenuOpen, setIsLanguageMenuOpen] = useState(false); - const [isWindows, setIsWindows] = useState(false); + const [supportsCursorModeToggle, setSupportsCursorModeToggle] = useState(false); const languageTriggerRef = useRef(null); const languageMenuPanelRef = useRef(null); const [languageMenuStyle, setLanguageMenuStyle] = useState<{ @@ -192,12 +192,12 @@ export function LaunchWindow() { .getPlatform() .then((platform) => { if (!cancelled) { - setIsWindows(platform === "win32"); + setSupportsCursorModeToggle(platform === "win32" || platform === "darwin"); } }) .catch(() => { if (!cancelled) { - setIsWindows(false); + setSupportsCursorModeToggle(false); } }); @@ -609,7 +609,7 @@ export function LaunchWindow() { ? getIcon("webcamOn", "text-green-400") : getIcon("webcamOff", "text-white/40")} - {isWindows && ( + {supportsCursorModeToggle && ( - + {canPauseRecording && ( + + + + )}