diff --git a/.gitignore b/.gitignore index 84a5a1cea..0861e392d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,8 +16,16 @@ dist-ssr # Native helper build outputs /electron/native/wgc-capture/build/ +/electron/native/screencapturekit/build/ +/electron/native/screencapturekit/.build/ +/electron/native/screencapturekit/.swiftpm/ /electron/native/bin/ +# Native macOS generated files +DerivedData/ +*.xcuserstate +xcuserdata/ + # Editor directories and files .vscode/* .zed/ @@ -49,4 +57,4 @@ result-* .direnv/ #kilocode -.kilo/ \ No newline at end of file +.kilo/ diff --git a/docs/engineering/macos-native-recorder-roadmap.md b/docs/engineering/macos-native-recorder-roadmap.md new file mode 100644 index 000000000..63afc423b --- /dev/null +++ b/docs/engineering/macos-native-recorder-roadmap.md @@ -0,0 +1,210 @@ +# macOS Native Recorder Roadmap + +OpenScreen's macOS recorder should follow the same architecture boundaries as the Windows native recorder: Electron owns session orchestration and persistence, while a platform-native helper owns capture, timing, encoding, and platform-specific permissions. + +This work is intentionally scoped as a macOS-only port. Windows native capture remains owned by the WGC helper, and Linux remains on the existing Electron path. + +## Goals + +- Capture displays and windows through ScreenCaptureKit. +- Exclude the real system cursor during capture when using the editable OpenScreen cursor overlay. +- Preserve the current high-quality cursor overlay path in preview and export. +- Capture macOS system audio through ScreenCaptureKit on supported macOS versions. +- Capture microphone audio through the same native timing domain where the OS supports it, or through an explicit companion path until it can be moved into the helper. +- Mix system audio and microphone audio into the primary MP4 without renderer-side track assembly. +- Capture webcam video natively and compose it into the helper-owned MP4 during the native-recording migration. +- Keep screen video, audio, webcam, and cursor aligned to one native timing origin. +- Package per-architecture helper binaries with macOS builds. + +## Non-Goals + +- Replacing the editor/export pipeline. +- Changing Windows native capture behavior. +- Adding Linux native capture. +- Shipping a silent fallback from native macOS capture to Electron capture when the user explicitly requested a native-only feature. + +## Architecture + +The renderer keeps the existing recording controls. On macOS, `useScreenRecorder` should eventually send a complete recording request to Electron instead of assembling display, audio, microphone, webcam, and cursor streams in the browser. + +Electron owns the native recording session: + +- resolves the selected display/window source; +- resolves output paths; +- starts cursor telemetry capture when editable cursor mode is selected; +- starts the ScreenCaptureKit helper process; +- sends pause/resume/stop/cancel commands; +- writes `RecordingSession` manifests; +- reports explicit errors when a macOS-native capability is unavailable. + +The helper owns macOS media capture: + +- ScreenCaptureKit display/window frames; +- ScreenCaptureKit system audio where supported; +- microphone capture or helper-owned companion audio capture; +- webcam capture and initial picture-in-picture composition; +- AVFoundation/VideoToolbox encoding and muxing; +- stream timestamp normalization. + +## Helper Contract V1 + +The helper receives a single JSON argument: + +```json +{ + "schemaVersion": 1, + "recordingId": 1234567890, + "source": { + "type": "display", + "sourceId": "screen:0:0", + "displayId": 1, + "windowId": null, + "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 } + }, + "video": { + "fps": 60, + "width": 1920, + "height": 1080, + "bitrate": 18000000, + "hideSystemCursor": true + }, + "audio": { + "system": { "enabled": true }, + "microphone": { + "enabled": true, + "deviceId": "default", + "deviceName": "MacBook Pro Microphone", + "gain": 1.4 + } + }, + "webcam": { + "enabled": true, + "deviceId": "default", + "deviceName": "FaceTime HD Camera", + "width": 1280, + "height": 720, + "fps": 30 + }, + "cursor": { + "mode": "editable-overlay" + }, + "outputs": { + "screenPath": "/Users/me/Library/Application Support/openscreen/recordings/recording-123.mp4", + "manifestPath": "/Users/me/Library/Application Support/openscreen/recordings/recording-123.session.json" + } +} +``` + +The helper emits newline-delimited JSON events to stdout: + +```json +{ "event": "ready", "schemaVersion": 1 } +{ "event": "recording-started", "timestampMs": 1234567890 } +{ "event": "warning", "code": "microphone-unavailable", "message": "..." } +{ "event": "recording-stopped", "screenPath": "..." } +{ "event": "error", "code": "screen-permission-denied", "message": "..." } +``` + +## Implementation Phases + +Current PR status: macOS screen/window capture routes through the ScreenCaptureKit helper when it is available so editable-cursor recordings can hide the system cursor. The helper now writes ScreenCaptureKit system audio into the primary MP4 and attempts runtime-gated native microphone capture on macOS versions that expose ScreenCaptureKit microphone output. Webcam capture is currently an Electron-recorded sidecar attached to the same recording session; native AVFoundation webcam composition remains the target end state. + +### 1. Native Session Boundary + +- Add a structured macOS native recording request type. +- Add a macOS helper resolver and build script placeholders. +- Keep the helper contract process-based, matching the Windows helper boundary. +- Do not route production macOS recording through this helper until the helper is available and validated. + +Acceptance: + +- TypeScript build passes. +- The macOS helper path and request contract are documented and testable without affecting Windows/Linux behavior. + +### 2. ScreenCaptureKit Display Capture + +- Implement a Swift helper using ScreenCaptureKit. +- Select display captures by `displayId`. +- Encode H.264 MP4 through AVFoundation/VideoToolbox. +- Set `showsCursor = false` when editable cursor overlay mode is selected. + +Acceptance: + +- Display-only recording produces a valid MP4. +- The real cursor is not baked into editable-cursor recordings. + +### 3. ScreenCaptureKit Window Capture + +- Resolve Electron `window:*` selections to ScreenCaptureKit window ids. +- Capture `SCContentFilter(desktopIndependentWindow:)`. +- Handle closed/minimized/protected windows with explicit errors. +- Keep window selection and capture source resolution in Electron/main, not the renderer. + +Acceptance: + +- Capturing a normal app window works with cursor/audio/webcam disabled. +- Unsupported windows return clear native errors. + +### 4. System Audio + +- Enable ScreenCaptureKit system audio on supported macOS versions. +- Keep audio format and timing owned by the helper. +- Encode or mux AAC audio into the primary MP4. + +Acceptance: + +- System-audio-only recordings produce a valid AAC track. +- Unsupported macOS versions return an explicit capability error. + +### 5. Microphone + +- Resolve the selected microphone device from the renderer-provided browser `deviceId` and user-visible label. +- Capture microphone audio in the helper timing domain. +- Apply OpenScreen microphone gain policy. +- Mix system and microphone audio before final AAC output. + +Acceptance: + +- Mic-only and mic-plus-system recordings produce a valid, balanced AAC track. +- Device selection honors the selected microphone, not only the default device. + +### 6. Webcam Composition + +- Capture the selected camera natively through AVFoundation. +- Match browser device id first where possible, then user-visible label. +- Compose an initial picture-in-picture overlay into the primary MP4. +- Hide webcam output until the first usable frame to avoid black startup flashes. + +Acceptance: + +- Native display/window recordings can include webcam without returning to Electron capture. +- Selected camera is honored. + +### 7. Runtime Controls + +- Add pause/resume commands to the helper. +- Add cancel command that removes partial outputs. +- Keep restart as stop-discard-start until the helper exposes a native restart operation. + +Acceptance: + +- Pause/resume keeps output duration coherent. +- Cancel leaves no stale media/session files. + +### 8. Test Pipeline + +- `npm run build:native:mac`: builds Swift helper binaries on macOS. +- `npm run test:sck-helper:mac`: display-only helper smoke test. +- `npm run test:sck-window:mac`: window capture smoke test. +- `npm run test:sck-audio:mac`: system audio smoke test when supported. +- `npm run test:sck-mic:mac`: microphone smoke test. +- `npm run test:sck-webcam:mac`: webcam smoke test when a webcam is available. +- Packaging check: confirms helpers are available under `electron/native/bin/darwin-${arch}` in packaged builds. + +## SSOT Rules + +- `src/lib/nativeMacRecording.ts` is the renderer/main TypeScript request contract. +- This document is the feature-level contract and phase checklist. +- The Swift helper owns ScreenCaptureKit/AVFoundation media timing. +- Electron owns output paths, session manifests, and selected source/device resolution. +- Renderer code must use existing hooks/client APIs and should not bind directly to helper process details. diff --git a/electron-builder.json5 b/electron-builder.json5 index dc6687d97..8ad4a80eb 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -46,6 +46,13 @@ ], "icon": "icons/icons/mac/icon.icns", "artifactName": "${productName}-Mac-${arch}-${version}-Installer.${ext}", + "extraResources": [ + { + "from": "electron/native/bin", + "to": "electron/native/bin", + "filter": ["darwin-*/*"] + } + ], "extendInfo": { "NSAudioCaptureUsageDescription": "OpenScreen needs audio capture permission to record system audio.", "NSMicrophoneUsageDescription": "OpenScreen needs microphone access to record voice audio.", @@ -73,7 +80,7 @@ { "from": "electron/native/bin", "to": "electron/native/bin", - "filter": ["**/*"] + "filter": ["win32-*/*"] } ] }, diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index e92ce19fc..abb688d16 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -31,7 +31,16 @@ interface Window { switchToEditor: () => Promise; switchToHud: () => Promise; startNewRecording: () => Promise<{ success: boolean; error?: string }>; - openSourceSelector: () => Promise; + openSourceSelector: () => Promise<{ + opened: boolean; + reason?: string; + access?: { + success: boolean; + granted: boolean; + status: string; + error?: string; + }; + }>; selectSource: (source: ProcessedDesktopSource) => Promise; getSelectedSource: () => Promise; requestCameraAccess: () => Promise<{ @@ -40,6 +49,18 @@ interface Window { status: string; error?: string; }>; + requestScreenAccess: () => Promise<{ + success: boolean; + granted: boolean; + status: string; + error?: string; + }>; + requestNativeMacCursorAccess: () => Promise<{ + success: boolean; + granted: boolean; + status: string; + error?: string; + }>; assetBaseUrl: string; storeRecordedVideo: ( videoData: ArrayBuffer, @@ -78,6 +99,13 @@ interface Window { reason?: string; error?: string; }>; + isNativeMacCaptureAvailable: () => Promise<{ + success: boolean; + available: boolean; + helperPath?: string; + reason?: "unsupported-platform" | "missing-helper" | string; + error?: string; + }>; startNativeWindowsRecording: ( request: import("../src/lib/nativeWindowsRecording").NativeWindowsRecordingRequest, ) => Promise; @@ -89,6 +117,37 @@ interface Window { discarded?: boolean; error?: string; }>; + startNativeMacRecording: ( + request: import("../src/lib/nativeMacRecording").NativeMacRecordingRequest, + ) => Promise; + pauseNativeMacRecording: () => Promise<{ + success: boolean; + error?: string; + }>; + resumeNativeMacRecording: () => Promise<{ + success: boolean; + error?: string; + }>; + stopNativeMacRecording: (discard?: boolean) => Promise<{ + success: boolean; + path?: string; + session?: import("../src/lib/recordingSession").RecordingSession; + message?: string; + discarded?: boolean; + error?: string; + }>; + attachNativeMacWebcamRecording: (payload: { + screenVideoPath: string; + recordingId: number; + webcam: import("../src/lib/recordingSession").RecordedVideoAssetInput; + cursorCaptureMode?: import("../src/lib/recordingSession").CursorCaptureMode; + }) => Promise<{ + success: boolean; + path?: string; + session?: import("../src/lib/recordingSession").RecordingSession; + message?: string; + error?: string; + }>; discardCursorTelemetry: (recordingId: number) => Promise; getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; @@ -138,6 +197,12 @@ interface Window { message?: string; error?: string; }>; + preparePreviewAudioTrack: (filePath: string) => Promise<{ + success: boolean; + path?: string | null; + message?: string; + error?: string; + }>; clearCurrentVideoPath: () => Promise<{ success: boolean }>; saveProjectFile: ( projectData: unknown, @@ -178,6 +243,7 @@ interface Window { hudOverlayHide: () => void; hudOverlayClose: () => void; setHudOverlayIgnoreMouseEvents: (ignore: boolean) => void; + moveHudOverlayBy: (deltaX: number, deltaY: number) => void; showCountdownOverlay: (value: number, runId: number) => Promise; setCountdownOverlayValue: (value: number, runId: number) => Promise; hideCountdownOverlay: (runId: number) => Promise; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 9797c95c0..009ade60a 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1,4 +1,5 @@ import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { EventEmitter } from "node:events"; import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import os from "node:os"; @@ -15,6 +16,7 @@ import { shell, systemPreferences, } from "electron"; +import type { NativeMacRecordingRequest } from "../../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording"; import { type CursorCaptureMode, @@ -22,6 +24,7 @@ import { normalizeProjectMedia, normalizeRecordingSession, type ProjectMedia, + type RecordedVideoAssetInput, type RecordingSession, type StoreRecordedSessionInput, } from "../../src/lib/recordingSession"; @@ -35,6 +38,7 @@ import type { import { mainT } from "../i18n"; import { RECORDINGS_DIR } from "../main"; import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; +import { requestMacCursorAccessibilityAccess } from "../native-bridge/cursor/recording/macNativeCursorRecordingSession"; import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; import { registerNativeBridgeHandlers } from "./nativeBridge"; @@ -43,6 +47,8 @@ const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json"); const RECORDING_FILE_PREFIX = "recording-"; const RECORDING_SESSION_SUFFIX = ".session.json"; const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([".webm", ".mp4", ".mov", ".avi", ".mkv"]); +const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio"); +const nativeMacCaptureEvents = new EventEmitter(); /** * Paths explicitly approved by the user via file picker dialogs or project loads. @@ -102,6 +108,102 @@ function hasAllowedImportVideoExtension(filePath: string): boolean { return ALLOWED_IMPORT_VIDEO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); } +function runProcess( + command: string, + args: string[], +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("error", reject); + child.on("close", (code) => resolve({ code, stdout, stderr })); + }); +} + +function parseAfinfoAudioTrackBitrates(output: string): number[] { + const bitrates: number[] = []; + const trackSections = output.split(/\n----\n/g).slice(1); + for (const section of trackSections) { + const match = section.match(/\bbit rate:\s*([0-9]+)\s*bits per second/i); + bitrates.push(match ? Number(match[1]) : 0); + } + return bitrates; +} + +async function prepareSupplementalPreviewAudioTrack(videoPath: string) { + const normalizedPath = await approveReadableVideoPath(videoPath); + if (!normalizedPath) { + return { + success: false, + message: "File path is not approved or is not a supported video file", + }; + } + + if (process.platform !== "darwin" || path.extname(normalizedPath).toLowerCase() !== ".mp4") { + return { success: true, path: null }; + } + + const afinfo = await runProcess("/usr/bin/afinfo", [normalizedPath]); + if (afinfo.code !== 0) { + return { success: true, path: null }; + } + + const bitrates = parseAfinfoAudioTrackBitrates(`${afinfo.stdout}\n${afinfo.stderr}`); + if (bitrates.length <= 1) { + return { success: true, path: null }; + } + + let supplementalTrackIndex = 1; + for (let index = 2; index < bitrates.length; index += 1) { + if (bitrates[index] > bitrates[supplementalTrackIndex]) { + supplementalTrackIndex = index; + } + } + + await fs.mkdir(PREVIEW_AUDIO_DIR, { recursive: true }); + const sourceStat = await fs.stat(normalizedPath); + const parsedPath = path.parse(normalizedPath); + const outputPath = path.join( + PREVIEW_AUDIO_DIR, + `${parsedPath.name}.track-${supplementalTrackIndex}.${Math.round(sourceStat.mtimeMs)}.m4a`, + ); + + try { + const outputStat = await fs.stat(outputPath); + if (outputStat.mtimeMs >= sourceStat.mtimeMs) { + return { success: true, path: pathToFileURL(outputPath).toString() }; + } + } catch { + // Generate below. + } + + const conversion = await runProcess("/usr/bin/afconvert", [ + "--read-track", + String(supplementalTrackIndex), + "-f", + "m4af", + "-d", + "aac", + normalizedPath, + outputPath, + ]); + if (conversion.code !== 0) { + return { + success: false, + message: conversion.stderr || conversion.stdout || "Failed to prepare preview audio", + }; + } + + return { success: true, path: pathToFileURL(outputPath).toString() }; +} + async function approveReadableVideoPath( filePath?: string | null, trustedDirs?: string[], @@ -215,6 +317,13 @@ type SelectedSource = { [key: string]: unknown; }; +type AttachNativeMacWebcamRecordingInput = { + screenVideoPath?: string; + recordingId?: number; + webcam?: RecordedVideoAssetInput; + cursorCaptureMode?: CursorCaptureMode; +}; + let selectedSource: SelectedSource | null = null; let selectedDesktopSource: DesktopCapturerSource | null = null; let lastEnumeratedSources = new Map(); @@ -276,6 +385,16 @@ let nativeWindowsCaptureRecordingId: number | null = null; let nativeWindowsCursorOffsetMs = 0; let nativeWindowsCursorCaptureMode: CursorCaptureMode = "editable-overlay"; const NATIVE_WINDOWS_CAPTURE_STOP_TIMEOUT_MS = 15_000; +let nativeMacCaptureProcess: ChildProcessWithoutNullStreams | null = null; +let nativeMacCaptureOutput = ""; +let nativeMacCaptureTargetPath: string | null = null; +let nativeMacCaptureRecordingId: number | null = null; +let nativeMacCursorOffsetMs = 0; +let nativeMacCursorCaptureMode: CursorCaptureMode = "editable-overlay"; +let nativeMacCursorRecordingStartMs = 0; +let nativeMacPauseStartedAtMs: number | null = null; +let nativeMacPauseRanges: Array<{ startMs: number; endMs: number }> = []; +let nativeMacIsPaused = false; function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { if (!sample || typeof sample !== "object") { @@ -499,6 +618,35 @@ async function findNativeWindowsCaptureHelperPath() { return null; } +function getNativeMacCaptureHelperCandidates() { + const envPath = process.env.OPENSCREEN_SCK_CAPTURE_EXE?.trim(); + const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; + const helperName = "openscreen-screencapturekit-helper"; + return [ + envPath, + resolveUnpackedAppPath("electron", "native", "screencapturekit", "build", helperName), + resolveUnpackedAppPath("electron", "native", "bin", archTag, helperName), + resolvePackagedResourcePath("electron", "native", "bin", archTag, helperName), + ].filter((candidate): candidate is string => Boolean(candidate)); +} + +async function findNativeMacCaptureHelperPath() { + if (process.platform !== "darwin") { + return null; + } + + for (const candidate of getNativeMacCaptureHelperCandidates()) { + try { + await fs.access(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next configured helper location. + } + } + + return null; +} + function isWindowsGraphicsCaptureOsSupported() { if (process.platform !== "win32") { return false; @@ -669,6 +817,62 @@ function shiftPendingCursorTelemetry(offsetMs: number) { }; } +function compactPendingCursorTelemetryPauseRanges( + ranges: Array<{ startMs: number; endMs: number }>, +) { + if (!pendingCursorRecordingData || ranges.length === 0) { + return; + } + + const normalizedRanges = ranges + .map((range) => ({ + startMs: Math.max(0, Math.min(range.startMs, range.endMs)), + endMs: Math.max(0, Math.max(range.startMs, range.endMs)), + })) + .filter((range) => Number.isFinite(range.startMs) && Number.isFinite(range.endMs)) + .filter((range) => range.endMs > range.startMs) + .sort((a, b) => a.startMs - b.startMs); + + if (normalizedRanges.length === 0) { + return; + } + + pendingCursorRecordingData = { + ...pendingCursorRecordingData, + samples: pendingCursorRecordingData.samples + .map((sample) => { + let pausedBeforeSampleMs = 0; + for (const range of normalizedRanges) { + if (sample.timeMs >= range.startMs && sample.timeMs <= range.endMs) { + return null; + } + if (sample.timeMs > range.endMs) { + pausedBeforeSampleMs += range.endMs - range.startMs; + } + } + + return { + ...sample, + timeMs: Math.max(0, sample.timeMs - pausedBeforeSampleMs), + }; + }) + .filter((sample): sample is CursorRecordingSample => Boolean(sample)) + .sort((a, b) => a.timeMs - b.timeMs), + }; +} + +function completeNativeMacCursorPauseRange(endMs = Date.now()) { + if (nativeMacPauseStartedAtMs === null || nativeMacCursorRecordingStartMs <= 0) { + return; + } + + nativeMacPauseRanges.push({ + startMs: Math.max(0, nativeMacPauseStartedAtMs - nativeMacCursorRecordingStartMs), + endMs: Math.max(0, endMs - nativeMacCursorRecordingStartMs), + }); + nativeMacPauseStartedAtMs = null; +} + function waitForNativeWindowsCaptureStart(proc: ChildProcessWithoutNullStreams) { return new Promise((resolve, reject) => { const timer = setTimeout(() => { @@ -785,6 +989,157 @@ function readNativeWindowsWebcamFormat(output: string) { } } +function tryParseNativeHelperEvent(line: string) { + try { + const parsed = JSON.parse(line); + return parsed && typeof parsed === "object" ? parsed : null; + } catch { + return null; + } +} + +function inspectNativeMacCaptureOutput() { + for (const line of nativeMacCaptureOutput.split(/\r?\n/)) { + const event = tryParseNativeHelperEvent(line.trim()); + if (event) { + nativeMacCaptureEvents.emit("helper-event", event); + } + } +} + +function attachNativeMacCaptureOutputDrain(proc: ChildProcessWithoutNullStreams) { + let lineBuffer = ""; + const drain = (chunk: Buffer) => { + const text = chunk.toString(); + nativeMacCaptureOutput += text; + lineBuffer += text; + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() ?? ""; + for (const line of lines) { + const event = tryParseNativeHelperEvent(line.trim()); + if (event) { + nativeMacCaptureEvents.emit("helper-event", event); + } + } + }; + const cleanup = () => { + proc.stdout.off("data", drain); + proc.stderr.off("data", drain); + proc.off("close", cleanup); + proc.off("error", cleanup); + }; + + proc.stdout.on("data", drain); + proc.stderr.on("data", drain); + proc.once("close", cleanup); + proc.once("error", cleanup); +} + +function waitForNativeMacCaptureStart(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(new Error("Timed out waiting for native macOS capture to start")); + }, 10_000); + + const inspect = (event: Record) => { + if (event.event === "recording-started") { + cleanup(); + resolve(); + return; + } + if (event.event === "error") { + cleanup(); + reject(new Error(String(event.message ?? event.code ?? "Native macOS capture failed"))); + } + }; + + const onOutput = (event: Record) => inspect(event); + const onClose = (code: number | null) => { + cleanup(); + reject( + new Error( + nativeMacCaptureOutput.trim() || + `Native macOS capture exited before recording started (code=${code ?? "unknown"})`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const cleanup = () => { + clearTimeout(timer); + nativeMacCaptureEvents.off("helper-event", onOutput); + proc.off("close", onClose); + proc.off("error", onError); + }; + + nativeMacCaptureEvents.on("helper-event", onOutput); + proc.once("close", onClose); + proc.once("error", onError); + inspectNativeMacCaptureOutput(); + }); +} + +function waitForNativeMacCaptureStop(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject( + new Error( + `Timed out waiting for native macOS capture to stop. Output path: ${ + nativeMacCaptureTargetPath ?? "unknown" + }. Output: ${nativeMacCaptureOutput.trim()}`, + ), + ); + }, 30_000); + + const inspect = (event: Record) => { + if (event.event === "recording-stopped") { + cleanup(); + resolve(String(event.screenPath ?? nativeMacCaptureTargetPath ?? "")); + return; + } + if (event.event === "error") { + cleanup(); + reject(new Error(String(event.message ?? event.code ?? "Native macOS capture failed"))); + } + }; + + const onOutput = (event: Record) => inspect(event); + const onClose = (code: number | null) => { + if (code === 0 && nativeMacCaptureTargetPath) { + cleanup(); + resolve(nativeMacCaptureTargetPath); + return; + } + cleanup(); + reject( + new Error( + nativeMacCaptureOutput.trim() || + `Native macOS capture exited with code=${code ?? "unknown"}`, + ), + ); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const cleanup = () => { + clearTimeout(timer); + nativeMacCaptureEvents.off("helper-event", onOutput); + proc.off("close", onClose); + proc.off("error", onError); + }; + + nativeMacCaptureEvents.on("helper-event", onOutput); + proc.once("close", onClose); + proc.once("error", onError); + inspectNativeMacCaptureOutput(); + }); +} + function setCurrentRecordingSessionState(session: RecordingSession | null) { currentRecordingSession = session; currentVideoPath = session?.screenVideoPath ?? null; @@ -872,6 +1227,43 @@ export function registerIpcHandlers( onRecordingStateChange?: (recording: boolean, sourceName: string) => void, _switchToHud?: () => void, ) { + async function requestScreenAccess() { + if (process.platform !== "darwin") { + return { success: true, granted: true, status: "granted" }; + } + + try { + const status = systemPreferences.getMediaAccessStatus("screen"); + if (status === "granted") { + return { success: true, granted: true, status }; + } + + // Screen recording has no askForMediaAccess equivalent. Trigger the + // TCC prompt without opening OpenScreen's source selector above it. + if (status === "not-determined") { + const mainWin = getMainWindow(); + if (mainWin && !mainWin.isDestroyed()) { + if (!mainWin.isVisible()) { + mainWin.show(); + } + mainWin.focus(); + } + app.focus({ steal: true }); + desktopCapturer + .getSources({ types: ["screen"], thumbnailSize: { width: 1, height: 1 } }) + .catch(() => { + // Permission probing failure is reported by the explicit status check below. + }); + return { success: true, granted: false, status: "not-determined" }; + } + + return { success: true, granted: false, status }; + } catch (error) { + console.error("Failed to request screen access:", error); + return { success: false, granted: false, status: "unknown", error: String(error) }; + } + } + ipcMain.handle("get-sources", async (_, opts) => { const sources = await desktopCapturer.getSources(opts); lastEnumeratedSources = new Map(sources.map((source) => [source.id, source])); @@ -948,40 +1340,51 @@ export function registerIpcHandlers( }); ipcMain.handle("request-screen-access", async () => { - if (process.platform !== "darwin") { - return { success: true, granted: true, status: "granted" }; - } + return requestScreenAccess(); + }); - try { - const status = systemPreferences.getMediaAccessStatus("screen"); - if (status === "granted") { - return { success: true, granted: true, status }; - } + ipcMain.handle("request-native-mac-cursor-access", async () => { + return requestMacCursorAccessibilityAccess(); + }); - // Screen recording has no askForMediaAccess equivalent — the TCC prompt - // is triggered by desktopCapturer.getSources(). Fire it and return so - // the renderer can re-check status after the user responds. - if (status === "not-determined") { - desktopCapturer.getSources({ types: ["screen"] }).catch(() => { - // Permission probing failure is reported by the explicit status check below. - }); - return { success: true, granted: false, status: "not-determined" }; + ipcMain.handle("open-source-selector", async () => { + const access = await requestScreenAccess(); + if (!access.granted) { + if (process.platform === "darwin" && access.status !== "not-determined") { + const mainWin = getMainWindow(); + const messageOptions = { + type: "warning", + buttons: ["Open System Settings", "Cancel"], + defaultId: 0, + cancelId: 1, + message: "Screen Recording permission is required", + detail: + "Allow OpenScreen in macOS System Settings, then come back and choose a screen or window.", + } satisfies Electron.MessageBoxOptions; + const result = + mainWin && !mainWin.isDestroyed() + ? await dialog.showMessageBox(mainWin, messageOptions) + : await dialog.showMessageBox(messageOptions); + if (result.response === 0) { + await shell.openExternal( + "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture", + ); + } } - - return { success: true, granted: false, status }; - } catch (error) { - console.error("Failed to request screen access:", error); - return { success: false, granted: false, status: "unknown", error: String(error) }; + return { + opened: false, + reason: "screen-access-required", + access, + }; } - }); - ipcMain.handle("open-source-selector", () => { const sourceSelectorWin = getSourceSelectorWindow(); if (sourceSelectorWin) { sourceSelectorWin.focus(); - return; + return { opened: true }; } createSourceSelectorWindow(); + return { opened: true }; }); ipcMain.handle("switch-to-editor", () => { @@ -992,6 +1395,16 @@ export function registerIpcHandlers( createEditorWindow(); }); + ipcMain.handle("switch-to-hud", () => { + _switchToHud?.(); + return { success: true }; + }); + + ipcMain.handle("start-new-recording", () => { + _switchToHud?.(); + return { success: true }; + }); + ipcMain.handle("countdown-overlay-show", async (_, value: number, runId: number) => { const overlayWindow = getCountdownOverlayWindow?.() ?? createCountdownOverlayWindow(); if (overlayWindow.isDestroyed()) { @@ -1041,6 +1454,17 @@ export function registerIpcHandlers( : { success: true, available: false, reason: "missing-helper" }; }); + ipcMain.handle("is-native-mac-capture-available", async () => { + if (process.platform !== "darwin") { + return { success: true, available: false, reason: "unsupported-platform" }; + } + + const helperPath = await findNativeMacCaptureHelperPath(); + return helperPath + ? { success: true, available: true, helperPath } + : { success: true, available: false, reason: "missing-helper" }; + }); + ipcMain.handle( "start-native-windows-recording", async (_, request: NativeWindowsRecordingRequest) => { @@ -1217,6 +1641,201 @@ export function registerIpcHandlers( }, ); + ipcMain.handle("start-native-mac-recording", async (_, request: NativeMacRecordingRequest) => { + try { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS capture requires macOS." }; + } + if (nativeMacCaptureProcess) { + return { success: false, error: "Native macOS capture is already running." }; + } + + const helperPath = await findNativeMacCaptureHelperPath(); + if (!helperPath) { + return { success: false, error: "Native macOS capture helper is not available." }; + } + + if (!request?.source?.sourceId) { + return { success: false, error: "Native macOS capture request is missing a source." }; + } + + const recordingId = + typeof request.recordingId === "number" && Number.isFinite(request.recordingId) + ? request.recordingId + : Date.now(); + const outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); + const cursorCaptureMode = + normalizeCursorCaptureMode(request.cursor?.mode) ?? "editable-overlay"; + try { + await desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: 1, height: 1 }, + }); + } catch { + // The helper reports the final ScreenCaptureKit permission status. + } + if (request.audio?.microphone?.enabled) { + const micStatus = systemPreferences.getMediaAccessStatus("microphone"); + if (micStatus !== "granted") { + await systemPreferences.askForMediaAccess("microphone"); + } + } + const sourceDisplay = + request.source.type === "display" && typeof request.source.displayId === "number" + ? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ?? + null) + : getSelectedDisplay(); + const bounds = request.source.bounds ?? sourceDisplay?.bounds ?? getSelectedSourceBounds(); + const config: NativeMacRecordingRequest = { + ...request, + schemaVersion: 1, + recordingId, + source: { + ...request.source, + bounds, + }, + video: { + ...request.video, + hideSystemCursor: cursorCaptureMode === "editable-overlay", + }, + webcam: { + ...request.webcam, + enabled: false, + }, + cursor: { + mode: cursorCaptureMode, + }, + outputs: { + screenPath: outputPath, + manifestPath: path.join( + RECORDINGS_DIR, + `${RECORDING_FILE_PREFIX}${recordingId}${RECORDING_SESSION_SUFFIX}`, + ), + }, + }; + + console.info("[native-sck] starting macOS capture", { + helperPath, + source: config.source, + audio: config.audio, + webcam: config.webcam, + cursor: config.cursor, + outputPath, + }); + + await fs.mkdir(RECORDINGS_DIR, { recursive: true }); + nativeMacCaptureOutput = ""; + nativeMacCaptureTargetPath = outputPath; + nativeMacCaptureRecordingId = recordingId; + nativeMacCursorOffsetMs = 0; + nativeMacCursorCaptureMode = cursorCaptureMode; + nativeMacCursorRecordingStartMs = 0; + nativeMacPauseStartedAtMs = null; + nativeMacPauseRanges = []; + nativeMacIsPaused = false; + + const cursorStartTimeMs = Date.now(); + if (cursorCaptureMode === "editable-overlay") { + nativeMacCursorRecordingStartMs = cursorStartTimeMs; + await startCursorRecording(cursorStartTimeMs); + } else { + pendingCursorRecordingData = null; + } + + const proc = spawn(helperPath, [JSON.stringify(config)], { + cwd: RECORDINGS_DIR, + stdio: ["pipe", "pipe", "pipe"], + }); + nativeMacCaptureProcess = proc; + attachNativeMacCaptureOutputDrain(proc); + + await waitForNativeMacCaptureStart(proc); + const captureStartedAtMs = Date.now(); + nativeMacCursorOffsetMs = + cursorCaptureMode === "editable-overlay" + ? Math.max(0, captureStartedAtMs - cursorStartTimeMs) + : 0; + + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(true, source.name); + } + + return { + success: true, + recordingId, + path: outputPath, + helperPath, + }; + } catch (error) { + console.error("Failed to start native macOS recording:", error); + nativeMacCaptureProcess?.kill(); + nativeMacCaptureProcess = null; + nativeMacCaptureTargetPath = null; + nativeMacCaptureRecordingId = null; + nativeMacCursorOffsetMs = 0; + nativeMacCursorCaptureMode = "editable-overlay"; + nativeMacCursorRecordingStartMs = 0; + nativeMacPauseStartedAtMs = null; + nativeMacPauseRanges = []; + nativeMacIsPaused = false; + await stopCursorRecording(); + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } + }); + + ipcMain.handle("pause-native-mac-recording", async () => { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS capture requires macOS." }; + } + + const proc = nativeMacCaptureProcess; + if (!proc) { + return { success: false, error: "Native macOS capture is not running." }; + } + if (nativeMacIsPaused) { + return { success: true }; + } + if (!proc.stdin.writable) { + return { success: false, error: "Native macOS capture command channel is closed." }; + } + + try { + proc.stdin.write("pause\n"); + nativeMacIsPaused = true; + nativeMacPauseStartedAtMs = Date.now(); + return { success: true }; + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } + }); + + ipcMain.handle("resume-native-mac-recording", async () => { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS capture requires macOS." }; + } + + const proc = nativeMacCaptureProcess; + if (!proc) { + return { success: false, error: "Native macOS capture is not running." }; + } + if (!nativeMacIsPaused) { + return { success: true }; + } + if (!proc.stdin.writable) { + return { success: false, error: "Native macOS capture command channel is closed." }; + } + + try { + proc.stdin.write("resume\n"); + completeNativeMacCursorPauseRange(); + nativeMacIsPaused = false; + return { success: true }; + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } + }); + ipcMain.handle("stop-native-windows-recording", async (_, discard?: boolean) => { const proc = nativeWindowsCaptureProcess; const preferredPath = nativeWindowsCaptureTargetPath; @@ -1301,6 +1920,152 @@ export function registerIpcHandlers( } }); + ipcMain.handle("stop-native-mac-recording", async (_, discard?: boolean) => { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS capture requires macOS." }; + } + + const proc = nativeMacCaptureProcess; + const preferredPath = nativeMacCaptureTargetPath; + const recordingId = nativeMacCaptureRecordingId ?? Date.now(); + const cursorCaptureMode = nativeMacCursorCaptureMode; + + if (!proc) { + return { success: false, error: "Native macOS capture is not running." }; + } + + try { + completeNativeMacCursorPauseRange(); + const stoppedPathPromise = waitForNativeMacCaptureStop(proc); + proc.stdin.write("stop\n"); + const stoppedPath = await stoppedPathPromise; + const screenVideoPath = stoppedPath || preferredPath; + if (!screenVideoPath) { + throw new Error("Native macOS capture did not return an output path."); + } + + if (cursorCaptureMode === "editable-overlay") { + await stopCursorRecording(); + } else { + pendingCursorRecordingData = null; + } + if (discard) { + pendingCursorRecordingData = null; + await Promise.all([ + fs.rm(screenVideoPath, { force: true }), + fs.rm(`${screenVideoPath}.cursor.json`, { force: true }), + ]); + return { success: true, discarded: true }; + } + + if (cursorCaptureMode === "editable-overlay") { + compactPendingCursorTelemetryPauseRanges(nativeMacPauseRanges); + shiftPendingCursorTelemetry(nativeMacCursorOffsetMs); + await writePendingCursorTelemetry(screenVideoPath); + } + + const session: RecordingSession = { + screenVideoPath, + createdAt: recordingId, + cursorCaptureMode, + }; + setCurrentRecordingSessionState(session); + currentProjectPath = null; + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(screenVideoPath).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + + return { + success: true, + path: screenVideoPath, + session, + message: "Native macOS recording session stored successfully", + }; + } catch (error) { + console.error("Failed to stop native macOS recording:", error); + await stopCursorRecording(); + return { success: false, error: error instanceof Error ? error.message : String(error) }; + } finally { + nativeMacCaptureProcess = null; + nativeMacCaptureTargetPath = null; + nativeMacCaptureRecordingId = null; + nativeMacCursorOffsetMs = 0; + nativeMacCursorCaptureMode = "editable-overlay"; + nativeMacCursorRecordingStartMs = 0; + nativeMacPauseStartedAtMs = null; + nativeMacPauseRanges = []; + nativeMacIsPaused = false; + const source = selectedSource || { name: "Screen" }; + if (onRecordingStateChange) { + onRecordingStateChange(false, source.name); + } + } + }); + + ipcMain.handle( + "attach-native-mac-webcam-recording", + async (_, payload: AttachNativeMacWebcamRecordingInput) => { + try { + if (process.platform !== "darwin") { + return { success: false, error: "Native macOS webcam attachment requires macOS." }; + } + + const screenVideoPath = normalizeVideoSourcePath(payload.screenVideoPath); + if (!screenVideoPath || !isPathWithinDir(screenVideoPath, RECORDINGS_DIR)) { + return { + success: false, + error: "Native macOS webcam attachment requires a recording output path.", + }; + } + + await fs.access(screenVideoPath, fsConstants.R_OK); + + if (!payload.webcam?.fileName || !payload.webcam.videoData) { + return { success: false, error: "Native macOS webcam attachment is missing video data." }; + } + + const webcamVideoPath = resolveRecordingOutputPath(payload.webcam.fileName); + await fs.writeFile(webcamVideoPath, Buffer.from(payload.webcam.videoData)); + + const createdAt = + typeof payload.recordingId === "number" && Number.isFinite(payload.recordingId) + ? payload.recordingId + : Date.now(); + const cursorCaptureMode = normalizeCursorCaptureMode(payload.cursorCaptureMode); + const session: RecordingSession = { + screenVideoPath, + webcamVideoPath, + createdAt, + ...(cursorCaptureMode ? { cursorCaptureMode } : {}), + }; + setCurrentRecordingSessionState(session); + currentProjectPath = null; + + const sessionManifestPath = path.join( + RECORDINGS_DIR, + `${path.parse(screenVideoPath).name}${RECORDING_SESSION_SUFFIX}`, + ); + await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); + + return { + success: true, + path: screenVideoPath, + session, + message: "Native macOS webcam recording attached successfully", + }; + } catch (error) { + console.error("Failed to attach native macOS webcam recording:", error); + return { + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } + }, + ); + ipcMain.handle("store-recorded-session", async (_, payload: StoreRecordedSessionInput) => { try { return await storeRecordedSessionFiles(payload); @@ -1620,6 +2385,19 @@ export function registerIpcHandlers( } }); + ipcMain.handle("prepare-preview-audio-track", async (_, filePath: string) => { + try { + return await prepareSupplementalPreviewAudioTrack(filePath); + } catch (error) { + console.error("Failed to prepare preview audio track:", error); + return { + success: false, + message: "Failed to prepare preview audio track", + error: String(error), + }; + } + }); + ipcMain.handle( "save-project-file", async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => { diff --git a/electron/main.ts b/electron/main.ts index 716d03be7..3e2258f8f 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -4,7 +4,6 @@ import { fileURLToPath } from "node:url"; import { app, BrowserWindow, - desktopCapturer, ipcMain, Menu, nativeImage, @@ -493,23 +492,14 @@ app.whenReady().then(async () => { { useSystemPicker: false }, ); - // Request microphone and screen recording permissions from macOS + // Request microphone permission from macOS. Screen Recording is requested + // lazily from the source-picker action so the system prompt is not hidden + // behind OpenScreen's source selector window. if (process.platform === "darwin") { const micStatus = systemPreferences.getMediaAccessStatus("microphone"); if (micStatus !== "granted") { await systemPreferences.askForMediaAccess("microphone"); } - - // Screen recording has no askForMediaAccess equivalent — the TCC prompt is - // triggered by the first desktopCapturer.getSources() call. Firing it here - // at startup settles the permission state early and prevents repeated prompts - // driven by later getSources() calls (fixes repeated permission dialog). - const screenStatus = systemPreferences.getMediaAccessStatus("screen"); - if (screenStatus === "not-determined") { - desktopCapturer.getSources({ types: ["screen"] }).catch(() => { - // This only triggers the system prompt; permission state is read separately. - }); - } } // Listen for HUD overlay quit event (macOS only) diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index e072b75d9..0ba307788 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -1,4 +1,5 @@ import type { Rectangle } from "electron"; +import { MacNativeCursorRecordingSession } from "./macNativeCursorRecordingSession"; import type { CursorRecordingSession } from "./session"; import { TelemetryRecordingSession } from "./telemetryRecordingSession"; import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession"; @@ -25,9 +26,17 @@ export function createCursorRecordingSession( }); } - // macOS / Linux: capture cursor positions via Electron's `screen` API on an - // interval. No cursor sprites/assets and no clicks — just position telemetry, - // which is what auto-zoom and other features consume. + if (options.platform === "darwin") { + return new MacNativeCursorRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + startTimeMs: options.startTimeMs, + }); + } + + // Linux: capture cursor positions via Electron's `screen` API on an interval. + // No cursor sprites/assets and no clicks — just position telemetry. return new TelemetryRecordingSession({ getDisplayBounds: options.getDisplayBounds, maxSamples: options.maxSamples, diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts new file mode 100644 index 000000000..95ed10ced --- /dev/null +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts @@ -0,0 +1,411 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import { accessSync, constants as fsConstants } from "node:fs"; +import path from "node:path"; +import type { Readable } from "node:stream"; +import { type Rectangle, screen, systemPreferences } from "electron"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorType, +} from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; + +interface MacNativeCursorRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; + startTimeMs?: number; +} + +type MacCursorEvent = + | { + type: "ready"; + timestampMs: number; + accessibilityTrusted?: boolean; + mouseTapReady?: boolean; + } + | { + type: "sample"; + timestampMs: number; + cursorType?: NativeCursorType | null; + leftButtonDown?: boolean; + leftButtonPressed?: boolean; + leftButtonReleased?: boolean; + }; + +const HELPER_NAME = "openscreen-macos-cursor-helper"; +const READY_TIMEOUT_MS = 5_000; + +function helperCandidates() { + const envPath = process.env.OPENSCREEN_MAC_CURSOR_HELPER_EXE?.trim(); + const appRoot = process.env.APP_ROOT ? path.resolve(process.env.APP_ROOT) : process.cwd(); + const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; + const resourceRoot = + typeof process.resourcesPath === "string" + ? process.resourcesPath + : path.join(appRoot, "resources"); + + return [ + envPath, + path.join(appRoot, "electron", "native", "screencapturekit", "build", HELPER_NAME), + path.join(appRoot, "electron", "native", "bin", archTag, HELPER_NAME), + path.join(resourceRoot, "electron", "native", "bin", archTag, HELPER_NAME), + ].filter((candidate): candidate is string => Boolean(candidate)); +} + +export function findMacCursorHelperPath() { + for (const candidate of helperCandidates()) { + try { + accessSync(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Try the next helper location. + } + } + + return null; +} + +export async function requestMacCursorAccessibilityAccess() { + if (process.platform !== "darwin") { + return { success: true, granted: true, status: "granted" }; + } + + try { + systemPreferences.isTrustedAccessibilityClient(true); + } catch { + // Continue with helper probing; it can trigger the same macOS prompt. + } + + const helperPath = findMacCursorHelperPath(); + if (!helperPath) { + return { success: true, granted: false, status: "missing-helper" }; + } + + return new Promise<{ success: boolean; granted: boolean; status: string; error?: string }>( + (resolve) => { + const child = spawn(helperPath, [JSON.stringify({ sampleIntervalMs: 250 })], { + stdio: ["ignore", "pipe", "pipe"], + }); + let settled = false; + let lineBuffer = ""; + const finish = (result: { + success: boolean; + granted: boolean; + status: string; + error?: string; + }) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + if (!child.killed) { + child.kill("SIGTERM"); + } + resolve(result); + }; + const timer = setTimeout(() => { + finish({ + success: false, + granted: false, + status: "timeout", + error: "Timed out waiting for macOS cursor helper", + }); + }, READY_TIMEOUT_MS); + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + lineBuffer += chunk; + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() ?? ""; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + try { + const event = JSON.parse(trimmed) as MacCursorEvent; + if (event.type === "ready") { + finish({ + success: true, + granted: event.accessibilityTrusted === true, + status: event.accessibilityTrusted === true ? "granted" : "not-determined", + }); + return; + } + } catch { + // Ignore non-JSON helper output. + } + } + }); + + child.once("error", (error) => { + finish({ + success: false, + granted: false, + status: "error", + error: error.message, + }); + }); + child.once("exit", (code, signal) => { + finish({ + success: false, + granted: false, + status: "exited", + error: `macOS cursor helper exited before ready (code=${code}, signal=${signal})`, + }); + }); + }, + ); +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function normalizeCursorType(value: unknown): NativeCursorType | null { + return value === "arrow" || value === "pointer" || value === "text" ? value : null; +} + +export class MacNativeCursorRecordingSession implements CursorRecordingSession { + private samples: CursorRecordingSample[] = []; + private process: ChildProcessByStdio | null = null; + private lineBuffer = ""; + private startTimeMs = 0; + private fallbackInterval: NodeJS.Timeout | null = null; + private readyResolve: (() => void) | null = null; + private readyReject: ((error: Error) => void) | null = null; + private readyTimer: NodeJS.Timeout | null = null; + private previousLeftButtonDown = false; + private consecutiveOutsideSamples = 0; + // Only hide after this many consecutive out-of-bounds samples (≈100ms at 33ms interval). + // Fast swipes that briefly exit the display are clipped by clip-path instead of disappearing. + private static readonly OUTSIDE_HIDE_THRESHOLD = 3; + + constructor(private readonly options: MacNativeCursorRecordingSessionOptions) {} + + async start(): Promise { + this.samples = []; + this.lineBuffer = ""; + this.startTimeMs = this.options.startTimeMs ?? Date.now(); + this.previousLeftButtonDown = false; + this.consecutiveOutsideSamples = 0; + + try { + systemPreferences.isTrustedAccessibilityClient(true); + } catch { + // Link cursor detection degrades to arrow when Accessibility is unavailable. + } + + const helperPath = findMacCursorHelperPath(); + if (!helperPath) { + this.startPositionOnlyFallback(); + return; + } + + const child = spawn( + helperPath, + [ + JSON.stringify({ + sampleIntervalMs: this.options.sampleIntervalMs, + }), + ], + { + stdio: ["ignore", "pipe", "pipe"], + }, + ); + this.process = child; + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => this.handleStdoutChunk(chunk)); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + const message = chunk.trim(); + if (message) { + console.error("[cursor-macos]", message); + } + }); + child.once("exit", (code, signal) => { + this.rejectReady( + new Error(`macOS cursor helper exited before ready (code=${code}, signal=${signal})`), + ); + this.process = null; + }); + child.once("error", (error) => { + this.rejectReady(error); + this.process = null; + }); + + try { + await this.waitUntilReady(); + } catch (error) { + this.killHelperProcess(child); + this.process = null; + console.warn("[cursor-macos] falling back to position-only cursor telemetry:", error); + this.startPositionOnlyFallback(); + } + } + + async stop(): Promise { + const child = this.process; + this.process = null; + this.clearReadyState(); + + if (this.fallbackInterval) { + clearInterval(this.fallbackInterval); + this.fallbackInterval = null; + } + + if (child) { + this.killHelperProcess(child); + } + + return { + version: 2, + provider: "none", + samples: this.samples, + assets: [], + }; + } + + private startPositionOnlyFallback() { + this.captureSample(Date.now(), null, false, false, false); + this.fallbackInterval = setInterval(() => { + this.captureSample(Date.now(), null, false, false, false); + }, this.options.sampleIntervalMs); + } + + private handleStdoutChunk(chunk: string) { + this.lineBuffer += chunk; + const lines = this.lineBuffer.split(/\r?\n/); + this.lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + this.handleEvent(JSON.parse(trimmedLine) as MacCursorEvent); + } catch (error) { + console.error("Failed to parse macOS cursor helper output:", error, trimmedLine); + } + } + } + + private handleEvent(payload: MacCursorEvent) { + if (payload.type === "ready") { + if (payload.accessibilityTrusted === false) { + console.warn( + "[cursor-macos] Accessibility is not trusted; cursor shape detection will be arrow-only.", + ); + } + this.resolveReady(); + return; + } + + if (payload.type === "sample") { + this.captureSample( + payload.timestampMs, + normalizeCursorType(payload.cursorType), + payload.leftButtonDown === true, + payload.leftButtonPressed === true, + payload.leftButtonReleased === true, + ); + } + } + + private captureSample( + timestampMs: number, + cursorType: NativeCursorType | null, + leftButtonDown: boolean, + leftButtonPressed: boolean, + leftButtonReleased: boolean, + ) { + const cursor = screen.getCursorScreenPoint(); + const bounds = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds; + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + const normalizedX = (cursor.x - bounds.x) / width; + const normalizedY = (cursor.y - bounds.y) / height; + const isOutsideDisplay = + normalizedX < 0 || normalizedX > 1 || normalizedY < 0 || normalizedY > 1; + // Fast swipes that briefly exit the display ( this.options.maxSamples) { + this.samples.shift(); + } + } + + private waitUntilReady() { + return new Promise((resolve, reject) => { + this.readyResolve = resolve; + this.readyReject = reject; + this.readyTimer = setTimeout(() => { + this.rejectReady(new Error("Timed out waiting for macOS cursor helper")); + }, READY_TIMEOUT_MS); + }); + } + + private resolveReady() { + const resolve = this.readyResolve; + this.clearReadyState(); + resolve?.(); + } + + private rejectReady(error: Error) { + const reject = this.readyReject; + this.clearReadyState(); + reject?.(error); + } + + private clearReadyState() { + if (this.readyTimer) { + clearTimeout(this.readyTimer); + this.readyTimer = null; + } + this.readyResolve = null; + this.readyReject = null; + } + + private killHelperProcess(child: ChildProcessByStdio) { + if (child.killed) { + return; + } + + child.kill("SIGTERM"); + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 500).unref(); + } +} diff --git a/electron/native/README.md b/electron/native/README.md index 659829cd9..59930ba36 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -1,5 +1,38 @@ # Native capture helpers +## macOS + +macOS native recording will use a ScreenCaptureKit helper with the same process boundary as the Windows WGC helper: + +1. Electron resolves the selected source, output paths, and user-selected devices. +2. The helper receives one structured JSON request. +3. The helper owns ScreenCaptureKit/AVFoundation capture, timing, encoding, and muxing. +4. Electron persists the resulting media/session manifest and reports helper errors explicitly. + +Helper locations: + +1. `OPENSCREEN_SCK_CAPTURE_EXE`, for local development and diagnostics. +2. `electron/native/screencapturekit/build/openscreen-screencapturekit-helper`, for locally built Swift output. +3. `electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper` or `electron/native/bin/darwin-x64/openscreen-screencapturekit-helper`, for packaged prebuilt helpers. + +The macOS cursor-shape helper is resolved from `OPENSCREEN_MAC_CURSOR_HELPER_EXE` first, then the matching `openscreen-macos-cursor-helper` binary in the same local build and packaged `electron/native/bin/darwin-${arch}` directories. + +Build the macOS helper with: + +```bash +npm run build:native:mac +``` + +On non-macOS hosts this command exits successfully and does not affect Windows/Linux development. On macOS it builds the Swift package at `electron/native/screencapturekit`, writes the development binaries to `electron/native/screencapturekit/build`, and copies redistributable binaries to `electron/native/bin/darwin-${arch}`. + +The current helper implementation supports display/window ScreenCaptureKit video capture, cursor exclusion through `SCStreamConfiguration.showsCursor`, H.264 encoding, MP4 muxing, and ScreenCaptureKit system audio. It also attempts native ScreenCaptureKit microphone capture when the running macOS version exposes that capability. Webcam recording currently stays as an Electron sidecar and is attached to the same recording session after the native screen capture stops. + +Electron exposes `is-native-mac-capture-available` for capability probing. It resolves the same helper locations listed above and reports `missing-helper` until a Swift helper binary is present. When available, macOS recording routes screen/window capture through the native helper so editable cursor recordings do not bake the system cursor into the video. Cursor positions are sampled in Electron; when the cursor helper is available and Accessibility is granted, samples are also tagged with link/text cursor hints such as `pointer`. + +See `docs/engineering/macos-native-recorder-roadmap.md` for the contract, rollout phases, and SSOT rules. + +## Windows + Windows native recording is resolved from one of these locations: 1. `OPENSCREEN_WGC_CAPTURE_EXE`, for local development and diagnostics. diff --git a/electron/native/screencapturekit/Package.swift b/electron/native/screencapturekit/Package.swift new file mode 100644 index 000000000..ec3b1d98d --- /dev/null +++ b/electron/native/screencapturekit/Package.swift @@ -0,0 +1,30 @@ +// swift-tools-version: 5.9 + +import PackageDescription + +let package = Package( + name: "OpenScreenScreenCaptureKitHelper", + platforms: [ + .macOS(.v13) + ], + products: [ + .executable( + name: "openscreen-screencapturekit-helper", + targets: ["OpenScreenScreenCaptureKitHelper"] + ), + .executable( + name: "openscreen-macos-cursor-helper", + targets: ["OpenScreenMacOSCursorHelper"] + ) + ], + targets: [ + .executableTarget( + name: "OpenScreenScreenCaptureKitHelper", + path: "Sources/OpenScreenScreenCaptureKitHelper" + ), + .executableTarget( + name: "OpenScreenMacOSCursorHelper", + path: "Sources/OpenScreenMacOSCursorHelper" + ) + ] +) diff --git a/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift new file mode 100644 index 000000000..672e86f7a --- /dev/null +++ b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift @@ -0,0 +1,268 @@ +import AppKit +import ApplicationServices +import Foundation + +struct CursorHelperRequest: Decodable { + let sampleIntervalMs: Int? +} + +final class MouseButtonTracker { + private let lock = NSLock() + private var leftDownCount = 0 + private var leftUpCount = 0 + private var eventTap: CFMachPort? + private var runLoopSource: CFRunLoopSource? + + struct Events { + let leftDownCount: Int + let leftUpCount: Int + } + + func start() -> Bool { + let mask = + (1 << CGEventType.leftMouseDown.rawValue) | + (1 << CGEventType.leftMouseUp.rawValue) + guard let tap = CGEvent.tapCreate( + tap: .cgSessionEventTap, + place: .headInsertEventTap, + options: .listenOnly, + eventsOfInterest: CGEventMask(mask), + callback: { _, type, event, userInfo in + if let userInfo { + let tracker = Unmanaged.fromOpaque(userInfo).takeUnretainedValue() + tracker.record(type) + } + return Unmanaged.passUnretained(event) + }, + userInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) + ) else { + return false + } + + guard let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) else { + return false + } + + eventTap = tap + runLoopSource = source + CFRunLoopAddSource(CFRunLoopGetCurrent(), source, .commonModes) + CGEvent.tapEnable(tap: tap, enable: true) + return true + } + + func pump() { + CFRunLoopRunInMode(.defaultMode, 0.001, false) + } + + func consume() -> Events { + lock.lock() + defer { lock.unlock() } + let events = Events(leftDownCount: leftDownCount, leftUpCount: leftUpCount) + leftDownCount = 0 + leftUpCount = 0 + return events + } + + private func record(_ type: CGEventType) { + lock.lock() + defer { lock.unlock() } + if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput { + reenableTap() + return + } + if type == .leftMouseDown { + leftDownCount += 1 + } else if type == .leftMouseUp { + leftUpCount += 1 + } + } + + private func reenableTap() { + if let eventTap { + CGEvent.tapEnable(tap: eventTap, enable: true) + } + } +} + +func emit(_ fields: [String: Any?]) { + let compacted = fields.compactMapValues { $0 } + if let data = try? JSONSerialization.data(withJSONObject: compacted, options: []), + let line = String(data: data, encoding: .utf8) + { + print(line) + fflush(stdout) + } +} + +func stringAttribute(_ element: AXUIElement, _ attribute: String) -> String? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) + guard result == .success else { + return nil + } + + return value as? String +} + +func parentElement(_ element: AXUIElement) -> AXUIElement? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, kAXParentAttribute as CFString, &value) + guard result == .success else { + return nil + } + + guard CFGetTypeID(value) == AXUIElementGetTypeID() else { + return nil + } + + return (value as! AXUIElement) +} + +func roleDescription(_ element: AXUIElement) -> String? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, kAXRoleDescriptionAttribute as CFString, &value) + guard result == .success else { + return nil + } + + return value as? String +} + +func actionNames(_ element: AXUIElement) -> [String] { + var value: CFArray? + let result = AXUIElementCopyActionNames(element, &value) + guard result == .success, let value else { + return [] + } + + return (value as NSArray).compactMap { $0 as? String } +} +func isTextInputRole(_ role: String?) -> Bool { + role == "AXTextField" || + role == "AXTextArea" || + role == "AXTextView" || + role == "AXComboBox" +} + +func isPointerRole(_ role: String?, _ subrole: String?, _ description: String?) -> Bool { + if role == "AXLink" || + subrole?.localizedCaseInsensitiveContains("link") == true || + description?.contains("link") == true + { + return true + } + + return role == "AXButton" || + role == "AXMenuButton" || + role == "AXPopUpButton" || + role == "AXCheckBox" || + role == "AXRadioButton" || + role == "AXSwitch" || + role == "AXDisclosureTriangle" || + role == "AXTab" || + role == "AXMenuItem" +} + +func cursorTypeForElement(_ element: AXUIElement) -> String? { + var current: AXUIElement? = element + + for _ in 0..<5 { + guard let element = current else { + break + } + + let role = stringAttribute(element, kAXRoleAttribute) + let subrole = stringAttribute(element, kAXSubroleAttribute) + let description = roleDescription(element)?.lowercased() + + if isTextInputRole(role) { + return "text" + } + + if isPointerRole(role, subrole, description) { + return "pointer" + } + + current = parentElement(element) + } + + return nil +} + +func accessibilityPointForMouse() -> CGPoint { + let mouse = NSEvent.mouseLocation + let primaryHeight = NSScreen.screens.first?.frame.height ?? NSScreen.main?.frame.height ?? 0 + return CGPoint(x: mouse.x, y: primaryHeight - mouse.y) +} + +func currentCursorType() -> String? { + guard AXIsProcessTrusted() else { + return nil + } + + let point = accessibilityPointForMouse() + let systemWide = AXUIElementCreateSystemWide() + var element: AXUIElement? + let result = AXUIElementCopyElementAtPosition( + systemWide, + Float(point.x), + Float(point.y), + &element + ) + + guard result == .success, let element else { + return "arrow" + } + + return cursorTypeForElement(element) ?? "arrow" +} + +func timestampMs() -> Int { + Int(Date().timeIntervalSince1970 * 1000) +} + +func leftButtonDown() -> Bool { + CGEventSource.buttonState(.hidSystemState, button: .left) +} + +func requestAccessibilityTrust() -> Bool { + let options = [ + kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true + ] as CFDictionary + return AXIsProcessTrustedWithOptions(options) +} + +let request: CursorHelperRequest +if CommandLine.arguments.count >= 2, + let data = CommandLine.arguments[1].data(using: .utf8), + let decoded = try? JSONDecoder().decode(CursorHelperRequest.self, from: data) +{ + request = decoded +} else { + request = CursorHelperRequest(sampleIntervalMs: nil) +} + +let intervalMs = max(8, request.sampleIntervalMs ?? 33) +let accessibilityTrusted = requestAccessibilityTrust() +let mouseTracker = MouseButtonTracker() +let mouseTapReady = mouseTracker.start() +emit([ + "type": "ready", + "timestampMs": timestampMs(), + "accessibilityTrusted": accessibilityTrusted, + "mouseTapReady": mouseTapReady, +]) + +while true { + mouseTracker.pump() + let mouseEvents = mouseTracker.consume() + emit([ + "type": "sample", + "timestampMs": timestampMs(), + "cursorType": currentCursorType(), + "leftButtonDown": leftButtonDown(), + "leftButtonPressed": mouseEvents.leftDownCount > 0, + "leftButtonReleased": mouseEvents.leftUpCount > 0, + ]) + Thread.sleep(forTimeInterval: Double(intervalMs) / 1000.0) +} diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift new file mode 100644 index 000000000..14860b03f --- /dev/null +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/main.swift @@ -0,0 +1,673 @@ +import AVFoundation +import CoreGraphics +import CoreMedia +import Foundation +import ScreenCaptureKit + +struct Rectangle: Decodable { + let x: Double + let y: Double + let width: Double + let height: Double +} + +struct RecordingRequest: Decodable { + struct Source: Decodable { + let type: String + let sourceId: String + let displayId: UInt32? + let windowId: UInt32? + let bounds: Rectangle? + } + + struct Video: Decodable { + let fps: Int + let width: Int + let height: Int + let bitrate: Int? + let hideSystemCursor: Bool + } + + struct Audio: Decodable { + struct SystemAudio: Decodable { + let enabled: Bool + } + + struct Microphone: Decodable { + let enabled: Bool + let deviceId: String? + let deviceName: String? + let gain: Double + } + + let system: SystemAudio + let microphone: Microphone + } + + struct Webcam: Decodable { + let enabled: Bool + let deviceId: String? + let deviceName: String? + let width: Int + let height: Int + let fps: Int + } + + struct Cursor: Decodable { + let mode: String + } + + struct Outputs: Decodable { + let screenPath: String + let manifestPath: String? + } + + let schemaVersion: Int? + let recordingId: Int? + let source: Source + let video: Video + let audio: Audio + let webcam: Webcam + let cursor: Cursor + let outputs: Outputs +} + +enum HelperError: Error, CustomStringConvertible { + case invalidArguments + case unsupportedMacOS + case unsupportedFeature(String) + case sourceNotFound(String) + case invalidSourceType(String) + case permissionDenied(String) + case writerSetupFailed(String) + + var description: String { + switch self { + case .invalidArguments: + return "Expected one JSON recording request argument." + case .unsupportedMacOS: + return "ScreenCaptureKit recording requires macOS 13 or newer." + case .unsupportedFeature(let message): + return message + case .sourceNotFound(let message): + return message + case .invalidSourceType(let sourceType): + return "Unsupported source type: \(sourceType)." + case .permissionDenied(let message): + return message + case .writerSetupFailed(let message): + return message + } + } +} + +func emit(_ fields: [String: Any]) { + if let data = try? JSONSerialization.data(withJSONObject: fields, options: []), + let line = String(data: data, encoding: .utf8) + { + print(line) + fflush(stdout) + } +} + +func emitError(code: String, message: String) { + emit([ + "event": "error", + "code": code, + "message": message, + ]) +} + +@available(macOS 13.0, *) +final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { + private struct CaptureTarget { + let filter: SCContentFilter + let width: Int + let height: Int + } + + private let request: RecordingRequest + private let sampleQueue = DispatchQueue(label: "app.openscreen.sck-helper.samples") + private let stateQueue = DispatchQueue(label: "app.openscreen.sck-helper.state") + private var stream: SCStream? + private var writer: AVAssetWriter? + private var videoInput: AVAssetWriterInput? + private var systemAudioInput: AVAssetWriterInput? + private var microphoneAudioInput: AVAssetWriterInput? + private var didStartWriting = false + private var didEmitRecordingStarted = false + private var isStopping = false + private var isPaused = false + private var pauseStartedAt: CMTime? + private var totalPausedDuration = CMTime.zero + private var nativeMicrophoneEnabled = false + private var outputWidth = 1920 + private var outputHeight = 1080 + private let microphoneOutputTypeRawValue = 2 + private let hostClock = CMClockGetHostTimeClock() + + init(request: RecordingRequest) { + self.request = request + } + + func start() async throws { + try ensureRequestedPermissions() + + let content = try await SCShareableContent.excludingDesktopWindows( + false, + onScreenWindowsOnly: true + ) + let target = try makeCaptureTarget(from: content) + outputWidth = target.width + outputHeight = target.height + let configuration = makeStreamConfiguration() + let stream = SCStream(filter: target.filter, configuration: configuration, delegate: self) + + try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: sampleQueue) + if request.audio.system.enabled { + try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue) + } + if nativeMicrophoneEnabled { + guard let microphoneOutputType = SCStreamOutputType(rawValue: microphoneOutputTypeRawValue) else { + throw HelperError.unsupportedFeature( + "Native microphone capture requires a macOS version with ScreenCaptureKit microphone output." + ) + } + try stream.addStreamOutput(self, type: microphoneOutputType, sampleHandlerQueue: sampleQueue) + } + try setupWriter() + + self.stream = stream + emit(["event": "ready", "schemaVersion": 1]) + try await stream.startCapture() + } + + func stop() async { + let shouldStop = stateQueue.sync { + if isStopping { + return false + } + isStopping = true + return true + } + if !shouldStop { + return + } + + do { + try await stream?.stopCapture() + } catch { + emit([ + "event": "warning", + "code": "stop-capture-failed", + "message": "\(error)", + ]) + } + + await finishWriter() + } + + func pause() { + let didPause = stateQueue.sync { + if isStopping || isPaused { + return false + } + + isPaused = true + pauseStartedAt = CMClockGetTime(hostClock) + return true + } + + if didPause { + emit([ + "event": "recording-paused", + "timestampMs": Int(Date().timeIntervalSince1970 * 1000), + ]) + } + } + + func resume() { + let didResume = stateQueue.sync { + if isStopping || !isPaused { + return false + } + + if let pauseStartedAt { + let now = CMClockGetTime(hostClock) + totalPausedDuration = CMTimeAdd( + totalPausedDuration, + CMTimeSubtract(now, pauseStartedAt) + ) + } + isPaused = false + pauseStartedAt = nil + return true + } + + if didResume { + emit([ + "event": "recording-resumed", + "timestampMs": Int(Date().timeIntervalSince1970 * 1000), + ]) + } + } + + func stream(_ stream: SCStream, didStopWithError error: Error) { + emitError(code: "capture-stopped-with-error", message: "\(error)") + Task { + await stop() + } + } + + func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + guard CMSampleBufferDataIsReady(sampleBuffer) else { + return + } + let pauseState = currentPauseState() + if pauseState.paused { + return + } + guard let sampleBuffer = retimedSampleBuffer(sampleBuffer, subtracting: pauseState.offset) else { + return + } + + if type == .audio { + appendAudioSampleBuffer(sampleBuffer, to: systemAudioInput) + return + } + + if type.rawValue == microphoneOutputTypeRawValue { + appendAudioSampleBuffer(sampleBuffer, to: microphoneAudioInput) + return + } + + guard type == .screen else { + return + } + guard isCompleteFrame(sampleBuffer) else { + return + } + guard let videoInput, let writer else { + return + } + let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + if !didStartWriting { + writer.startWriting() + writer.startSession(atSourceTime: presentationTime) + didStartWriting = true + } + + if videoInput.isReadyForMoreMediaData { + if videoInput.append(sampleBuffer), !didEmitRecordingStarted { + didEmitRecordingStarted = true + emit([ + "event": "recording-started", + "timestampMs": Int(Date().timeIntervalSince1970 * 1000), + "width": outputWidth, + "height": outputHeight, + ]) + } + } + } + + private func ensureRequestedPermissions() throws { + if !CGPreflightScreenCaptureAccess() { + let granted = CGRequestScreenCaptureAccess() + if !granted { + throw HelperError.permissionDenied("Screen recording permission is required for ScreenCaptureKit capture.") + } + } + + if request.audio.microphone.enabled { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: + break + case .notDetermined: + let semaphore = DispatchSemaphore(value: 0) + AVCaptureDevice.requestAccess(for: .audio) { _ in + semaphore.signal() + } + let waitResult = semaphore.wait(timeout: .now() + 30) + if waitResult == .timedOut || AVCaptureDevice.authorizationStatus(for: .audio) != .authorized { + throw HelperError.permissionDenied("Microphone permission is required for native microphone capture.") + } + default: + throw HelperError.permissionDenied("Microphone permission is required for native microphone capture.") + } + } + } + + private func makeCaptureTarget(from content: SCShareableContent) throws -> CaptureTarget { + switch request.source.type { + case "display": + guard let displayId = request.source.displayId else { + throw HelperError.sourceNotFound("Display capture requires source.displayId.") + } + guard let display = content.displays.first(where: { $0.displayID == displayId }) else { + throw HelperError.sourceNotFound("No ScreenCaptureKit display found for id \(displayId).") + } + let width = Int(CGDisplayPixelsWide(display.displayID)) + let height = Int(CGDisplayPixelsHigh(display.displayID)) + return CaptureTarget( + filter: SCContentFilter(display: display, excludingWindows: []), + width: clampCaptureDimension(width, fallback: request.video.width), + height: clampCaptureDimension(height, fallback: request.video.height) + ) + case "window": + guard let windowId = request.source.windowId else { + throw HelperError.sourceNotFound("Window capture requires source.windowId.") + } + guard let window = content.windows.first(where: { $0.windowID == windowId }) else { + throw HelperError.sourceNotFound("No ScreenCaptureKit window found for id \(windowId).") + } + let candidateDisplay = content.displays.first { + $0.frame.intersects(window.frame) || $0.frame.contains(CGPoint(x: window.frame.midX, y: window.frame.midY)) + } + let scaleFactor = Self.scaleFactor(for: candidateDisplay?.displayID ?? CGMainDisplayID()) + let width = Int(window.frame.width) * scaleFactor + let height = Int(window.frame.height) * scaleFactor + return CaptureTarget( + filter: SCContentFilter(desktopIndependentWindow: window), + width: clampCaptureDimension(width, fallback: request.video.width), + height: clampCaptureDimension(height, fallback: request.video.height) + ) + default: + throw HelperError.invalidSourceType(request.source.type) + } + } + + private func makeStreamConfiguration() -> SCStreamConfiguration { + let configuration = SCStreamConfiguration() + configuration.width = outputWidth + configuration.height = outputHeight + configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(max(1, request.video.fps))) + configuration.queueDepth = 6 + configuration.showsCursor = !request.video.hideSystemCursor + configuration.pixelFormat = kCVPixelFormatType_32BGRA + configuration.sampleRate = 48_000 + configuration.channelCount = 2 + configuration.excludesCurrentProcessAudio = true + configuration.capturesAudio = request.audio.system.enabled + + if request.audio.microphone.enabled { + guard supportsNativeMicrophoneCapture(streamConfig: configuration) else { + nativeMicrophoneEnabled = false + emit([ + "event": "warning", + "code": "microphone-unavailable", + "message": "Native microphone capture requires ScreenCaptureKit microphone support on this macOS version.", + ]) + return configuration + } + nativeMicrophoneEnabled = true + configuration.capturesAudio = true + configuration.setValue(true, forKey: "captureMicrophone") + if let deviceId = resolveMicrophoneCaptureDeviceID() { + configuration.setValue(deviceId, forKey: "microphoneCaptureDeviceID") + } + } else { + nativeMicrophoneEnabled = false + } + + return configuration + } + + private func setupWriter() throws { + let outputUrl = URL(fileURLWithPath: request.outputs.screenPath) + try? FileManager.default.removeItem(at: outputUrl) + try FileManager.default.createDirectory( + at: outputUrl.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + + let writer = try AVAssetWriter(outputURL: outputUrl, fileType: .mp4) + let settings: [String: Any] = [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: outputWidth, + AVVideoHeightKey: outputHeight, + AVVideoCompressionPropertiesKey: [ + AVVideoAverageBitRateKey: request.video.bitrate ?? 18_000_000, + AVVideoExpectedSourceFrameRateKey: request.video.fps, + ], + ] + let input = AVAssetWriterInput(mediaType: .video, outputSettings: settings) + input.expectsMediaDataInRealTime = true + + guard writer.canAdd(input) else { + throw HelperError.writerSetupFailed("Unable to add H.264 video input to AVAssetWriter.") + } + + writer.add(input) + self.writer = writer + self.videoInput = input + + if request.audio.system.enabled { + systemAudioInput = try addAudioInput(to: writer, bitRate: 192_000) + } + if nativeMicrophoneEnabled { + microphoneAudioInput = try addAudioInput(to: writer, bitRate: 128_000) + } + } + + private func finishWriter() async { + guard let writer else { + return + } + + videoInput?.markAsFinished() + systemAudioInput?.markAsFinished() + microphoneAudioInput?.markAsFinished() + + await withCheckedContinuation { continuation in + writer.finishWriting { + continuation.resume() + } + } + + if writer.status == .completed { + emit([ + "event": "recording-stopped", + "screenPath": request.outputs.screenPath, + ]) + } else { + emitError( + code: "writer-failed", + message: writer.error.map { "\($0)" } ?? "AVAssetWriter failed with status \(writer.status.rawValue)." + ) + } + } + + private func addAudioInput(to writer: AVAssetWriter, bitRate: Int) throws -> AVAssetWriterInput { + let settings: [String: Any] = [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVSampleRateKey: 48_000, + AVNumberOfChannelsKey: 2, + AVEncoderBitRateKey: bitRate, + ] + let input = AVAssetWriterInput(mediaType: .audio, outputSettings: settings) + input.expectsMediaDataInRealTime = true + + guard writer.canAdd(input) else { + throw HelperError.writerSetupFailed("Unable to add AAC audio input to AVAssetWriter.") + } + + writer.add(input) + return input + } + + private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput?) { + guard didStartWriting else { + return + } + guard let input, input.isReadyForMoreMediaData else { + return + } + + input.append(sampleBuffer) + } + + private func currentPauseState() -> (paused: Bool, offset: CMTime) { + stateQueue.sync { + (isPaused, totalPausedDuration) + } + } + + private func retimedSampleBuffer(_ sampleBuffer: CMSampleBuffer, subtracting offset: CMTime) -> CMSampleBuffer? { + if !offset.isValid || offset == .zero { + return sampleBuffer + } + + let sampleCount = CMSampleBufferGetNumSamples(sampleBuffer) + if sampleCount <= 0 { + return sampleBuffer + } + + var timing = Array(repeating: CMSampleTimingInfo(), count: sampleCount) + let timingStatus = CMSampleBufferGetSampleTimingInfoArray( + sampleBuffer, + entryCount: sampleCount, + arrayToFill: &timing, + entriesNeededOut: nil + ) + if timingStatus != noErr { + emit([ + "event": "warning", + "code": "sample-retime-failed", + "message": "Unable to read sample timing info: \(timingStatus).", + ]) + return sampleBuffer + } + + for index in timing.indices { + if timing[index].presentationTimeStamp.isValid { + timing[index].presentationTimeStamp = CMTimeSubtract( + timing[index].presentationTimeStamp, + offset + ) + } + if timing[index].decodeTimeStamp.isValid { + timing[index].decodeTimeStamp = CMTimeSubtract(timing[index].decodeTimeStamp, offset) + } + } + + var retimedBuffer: CMSampleBuffer? + let copyStatus = CMSampleBufferCreateCopyWithNewTiming( + allocator: kCFAllocatorDefault, + sampleBuffer: sampleBuffer, + sampleTimingEntryCount: sampleCount, + sampleTimingArray: &timing, + sampleBufferOut: &retimedBuffer + ) + if copyStatus != noErr { + emit([ + "event": "warning", + "code": "sample-retime-failed", + "message": "Unable to copy sample timing info: \(copyStatus).", + ]) + return sampleBuffer + } + + return retimedBuffer + } + + private func isCompleteFrame(_ sampleBuffer: CMSampleBuffer) -> Bool { + guard let attachments = CMSampleBufferGetSampleAttachmentsArray( + sampleBuffer, + createIfNecessary: false + ) as? [[SCStreamFrameInfo: Any]], + let attachment = attachments.first, + let statusRawValue = attachment[SCStreamFrameInfo.status] as? Int, + let status = SCFrameStatus(rawValue: statusRawValue) + else { + return true + } + + return status == .complete + } + + private func clampCaptureDimension(_ value: Int, fallback: Int) -> Int { + let requested = max(2, fallback) + let candidate = value > 0 ? value : requested + let clamped = min(candidate, requested) + return max(2, clamped - (clamped % 2)) + } + + private static func scaleFactor(for displayId: CGDirectDisplayID) -> Int { + guard let mode = CGDisplayCopyDisplayMode(displayId) else { + return 1 + } + + return max(1, mode.pixelWidth / max(1, mode.width)) + } + + private func supportsNativeMicrophoneCapture(streamConfig: SCStreamConfiguration) -> Bool { + streamConfig.responds(to: Selector(("setCaptureMicrophone:"))) && + streamConfig.responds(to: Selector(("setMicrophoneCaptureDeviceID:"))) && + SCStreamOutputType(rawValue: microphoneOutputTypeRawValue) != nil + } + + private func resolveMicrophoneCaptureDeviceID() -> String? { + let devices = AVCaptureDevice.devices(for: .audio) + + if let deviceName = request.audio.microphone.deviceName?.trimmingCharacters(in: .whitespacesAndNewlines), + !deviceName.isEmpty, + let device = devices.first(where: { $0.localizedName == deviceName }) + { + return device.uniqueID + } + + if let deviceId = request.audio.microphone.deviceId?.trimmingCharacters(in: .whitespacesAndNewlines), + !deviceId.isEmpty, + devices.contains(where: { $0.uniqueID == deviceId }) + { + return deviceId + } + + return nil + } +} + +@main +struct OpenScreenScreenCaptureKitHelper { + static func main() async { + do { + guard CommandLine.arguments.count == 2 else { + throw HelperError.invalidArguments + } + + guard #available(macOS 13.0, *) else { + throw HelperError.unsupportedMacOS + } + + let requestData = Data(CommandLine.arguments[1].utf8) + let decoder = JSONDecoder() + let request = try decoder.decode(RecordingRequest.self, from: requestData) + let recorder = ScreenCaptureRecorder(request: request) + let stopTask = Task.detached { + while let line = readLine() { + let command = line.trimmingCharacters(in: .whitespacesAndNewlines) + switch command { + case "pause": + recorder.pause() + case "resume": + recorder.resume() + case "stop": + await recorder.stop() + exit(0) + default: + break + } + } + } + + try await recorder.start() + await stopTask.value + } catch let error as HelperError { + emitError(code: "helper-error", message: error.description) + exit(1) + } catch { + emitError(code: "helper-error", message: "\(error)") + exit(1) + } + } +} diff --git a/electron/preload.ts b/electron/preload.ts index 8302b959f..361eb18de 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,4 +1,5 @@ import { contextBridge, ipcRenderer } from "electron"; +import type { NativeMacRecordingRequest } from "../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording"; import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession"; import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts"; @@ -24,6 +25,9 @@ contextBridge.exposeInMainWorld("electronAPI", { setHudOverlayIgnoreMouseEvents: (ignore: boolean) => { ipcRenderer.send("hud-overlay-ignore-mouse-events", ignore); }, + moveHudOverlayBy: (deltaX: number, deltaY: number) => { + ipcRenderer.send("hud-overlay-move-by", deltaX, deltaY); + }, getSources: async (opts: Electron.SourcesOptions) => { return await ipcRenderer.invoke("get-sources", opts); }, @@ -48,6 +52,12 @@ contextBridge.exposeInMainWorld("electronAPI", { requestCameraAccess: () => { return ipcRenderer.invoke("request-camera-access"); }, + requestScreenAccess: () => { + return ipcRenderer.invoke("request-screen-access"); + }, + requestNativeMacCursorAccess: () => { + return ipcRenderer.invoke("request-native-mac-cursor-access"); + }, storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => { return ipcRenderer.invoke("store-recorded-video", videoData, fileName); }, @@ -68,12 +78,35 @@ contextBridge.exposeInMainWorld("electronAPI", { isNativeWindowsCaptureAvailable: () => { return ipcRenderer.invoke("is-native-windows-capture-available"); }, + isNativeMacCaptureAvailable: () => { + return ipcRenderer.invoke("is-native-mac-capture-available"); + }, startNativeWindowsRecording: (request: NativeWindowsRecordingRequest) => { return ipcRenderer.invoke("start-native-windows-recording", request); }, stopNativeWindowsRecording: (discard?: boolean) => { return ipcRenderer.invoke("stop-native-windows-recording", discard); }, + startNativeMacRecording: (request: NativeMacRecordingRequest) => { + return ipcRenderer.invoke("start-native-mac-recording", request); + }, + pauseNativeMacRecording: () => { + return ipcRenderer.invoke("pause-native-mac-recording"); + }, + resumeNativeMacRecording: () => { + return ipcRenderer.invoke("resume-native-mac-recording"); + }, + stopNativeMacRecording: (discard?: boolean) => { + return ipcRenderer.invoke("stop-native-mac-recording", discard); + }, + attachNativeMacWebcamRecording: (payload: { + screenVideoPath: string; + recordingId: number; + webcam: { fileName: string; videoData: ArrayBuffer }; + cursorCaptureMode?: import("../src/lib/recordingSession").CursorCaptureMode; + }) => { + return ipcRenderer.invoke("attach-native-mac-webcam-recording", payload); + }, getCursorTelemetry: (videoPath?: string) => { return ipcRenderer.invoke("get-cursor-telemetry", videoPath); }, @@ -112,6 +145,9 @@ contextBridge.exposeInMainWorld("electronAPI", { readBinaryFile: (filePath: string) => { return ipcRenderer.invoke("read-binary-file", filePath); }, + preparePreviewAudioTrack: (filePath: string) => { + return ipcRenderer.invoke("prepare-preview-audio-track", filePath); + }, clearCurrentVideoPath: () => { return ipcRenderer.invoke("clear-current-video-path"); }, diff --git a/electron/windows.ts b/electron/windows.ts index 4d4e75206..3a7350edf 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -30,6 +30,20 @@ ipcMain.on("hud-overlay-ignore-mouse-events", (_event, ignore: boolean) => { } }); +ipcMain.on("hud-overlay-move-by", (_event, deltaX: number, deltaY: number) => { + if ( + !hudOverlayWindow || + hudOverlayWindow.isDestroyed() || + !Number.isFinite(deltaX) || + !Number.isFinite(deltaY) + ) { + return; + } + + const [x, y] = hudOverlayWindow.getPosition(); + hudOverlayWindow.setPosition(Math.round(x + deltaX), Math.round(y + deltaY), false); +}); + /** * Creates the always-on-top HUD overlay window centred at the bottom of the * primary display. The window is frameless, transparent, and follows the user diff --git a/package.json b/package.json index 9388bcd31..fd0c4cf3d 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "format": "biome format --write .", "i18n:check": "node scripts/i18n-check.mjs", "preview": "vite preview", - "build:mac": "tsc && vite build && electron-builder --mac", + "build:native:mac": "node scripts/build-macos-screencapturekit-helper.mjs", + "build:mac": "npm run build:native:mac && tsc && vite build && electron-builder --mac", "build:native:win": "node scripts/build-windows-wgc-helper.mjs", "build:win": "npm run build:native:win && tsc && vite build && electron-builder --win --config.npmRebuild=false", "build:linux": "tsc && vite build && electron-builder --linux AppImage deb pacman --config.npmRebuild=false", diff --git a/scripts/build-macos-screencapturekit-helper.mjs b/scripts/build-macos-screencapturekit-helper.mjs new file mode 100644 index 000000000..8e7c97396 --- /dev/null +++ b/scripts/build-macos-screencapturekit-helper.mjs @@ -0,0 +1,91 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +if (process.platform !== "darwin") { + console.log("Skipping macOS ScreenCaptureKit helper build: host platform is not macOS."); + process.exit(0); +} + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const root = path.resolve(__dirname, ".."); +const helperName = "openscreen-screencapturekit-helper"; +const cursorHelperName = "openscreen-macos-cursor-helper"; +const packageDir = path.join(root, "electron", "native", "screencapturekit"); +const buildDir = path.join(packageDir, "build"); +const swiftBuildDir = path.join(buildDir, "swiftpm"); +const builtHelperPath = path.join(swiftBuildDir, "release", helperName); +const localHelperPath = path.join(buildDir, helperName); +const builtCursorHelperPath = path.join(swiftBuildDir, "release", cursorHelperName); +const localCursorHelperPath = path.join(buildDir, cursorHelperName); +const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; +const distributableDir = path.join(root, "electron", "native", "bin", archTag); +const distributablePath = path.join(distributableDir, helperName); +const distributableCursorHelperPath = path.join(distributableDir, cursorHelperName); + +const xcodebuildVersion = spawnSync("xcodebuild", ["-version"], { + cwd: root, + encoding: "utf8", +}); + +if (xcodebuildVersion.status !== 0) { + const message = `${xcodebuildVersion.stderr ?? ""}${xcodebuildVersion.stdout ?? ""}`.trim(); + console.error( + [ + "Unable to build the macOS ScreenCaptureKit helper because full Xcode is not active.", + "", + message, + "", + "Install Xcode from the App Store or Apple Developer downloads, then run:", + " sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer", + " sudo xcodebuild -license accept", + "", + "Command Line Tools alone may not include the Swift SDK/platform metadata required by SwiftPM.", + ].join("\n"), + ); + process.exit(1); +} + +const result = spawnSync( + "swift", + ["build", "-c", "release", "--package-path", packageDir, "--build-path", swiftBuildDir], + { + cwd: root, + stdio: "inherit", + }, +); + +if (result.error) { + console.error(`Failed to start Swift build: ${result.error.message}`); + process.exit(1); +} + +if (result.status !== 0) { + process.exit(result.status ?? 1); +} + +fs.mkdirSync(buildDir, { recursive: true }); +fs.mkdirSync(distributableDir, { recursive: true }); +for (const artifactPath of [builtHelperPath, builtCursorHelperPath]) { + if (!fs.existsSync(artifactPath)) { + console.error(`Swift build completed but expected artifact was not found: ${artifactPath}`); + process.exit(1); + } +} +fs.copyFileSync(builtHelperPath, localHelperPath); +fs.copyFileSync(builtHelperPath, distributablePath); +fs.copyFileSync(builtCursorHelperPath, localCursorHelperPath); +fs.copyFileSync(builtCursorHelperPath, distributableCursorHelperPath); +fs.chmodSync(localHelperPath, 0o755); +fs.chmodSync(distributablePath, 0o755); +fs.chmodSync(localCursorHelperPath, 0o755); +fs.chmodSync(distributableCursorHelperPath, 0o755); + +console.log(`Built macOS ScreenCaptureKit helper: ${localHelperPath}`); +console.log(`Copied redistributable helper: ${distributablePath}`); +console.log(`Built macOS cursor helper: ${localCursorHelperPath}`); +console.log(`Copied redistributable cursor helper: ${distributableCursorHelperPath}`); diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 056a4fd83..570ec2809 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -1,5 +1,5 @@ import { Check, ChevronDown, Languages } from "lucide-react"; -import { useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; import { BsPauseCircle, BsPlayCircle, BsRecordCircle } from "react-icons/bs"; import { FaRegStopCircle } from "react-icons/fa"; @@ -98,6 +98,7 @@ export function LaunchWindow() { elapsedSeconds, toggleRecording, togglePaused, + canPauseRecording, restartRecording, cancelRecording, microphoneEnabled, @@ -127,7 +128,7 @@ export function LaunchWindow() { const [isWebcamFocused, setIsWebcamFocused] = useState(false); const webcamExpanded = isWebcamHovered || isWebcamFocused; const [isLanguageMenuOpen, setIsLanguageMenuOpen] = useState(false); - const [isWindows, setIsWindows] = useState(false); + const [supportsCursorModeToggle, setSupportsCursorModeToggle] = useState(false); const languageTriggerRef = useRef(null); const languageMenuPanelRef = useRef(null); const [languageMenuStyle, setLanguageMenuStyle] = useState<{ @@ -192,12 +193,12 @@ export function LaunchWindow() { .getPlatform() .then((platform) => { if (!cancelled) { - setIsWindows(platform === "win32"); + setSupportsCursorModeToggle(platform === "win32" || platform === "darwin"); } }) .catch(() => { if (!cancelled) { - setIsWindows(false); + setSupportsCursorModeToggle(false); } }); @@ -281,12 +282,25 @@ export function LaunchWindow() { return () => cancelAnimationFrame(id); }, [isLanguageMenuOpen]); + const hudMouseEventsEnabledRef = useRef(undefined); + const setHudMouseEventsEnabled = useCallback((enabled: boolean) => { + if (hudMouseEventsEnabledRef.current === enabled) { + return; + } + hudMouseEventsEnabledRef.current = enabled; + window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!enabled); + }, []); + useEffect(() => { - window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true); + setHudMouseEventsEnabled(false); return () => { window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(false); }; - }, []); + }, [setHudMouseEventsEnabled]); + + useEffect(() => { + setHudMouseEventsEnabled(isLanguageMenuOpen); + }, [isLanguageMenuOpen, setHudMouseEventsEnabled]); const [selectedSource, setSelectedSource] = useState("Screen"); const [hasSelectedSource, setHasSelectedSource] = useState(false); @@ -357,6 +371,29 @@ export function LaunchWindow() { setMicrophoneEnabled(!microphoneEnabled); } }; + const dragLastPositionRef = useRef<{ x: number; y: number } | null>(null); + const handleHudDragPointerDown = (event: React.PointerEvent) => { + event.preventDefault(); + event.stopPropagation(); + setHudMouseEventsEnabled(true); + event.currentTarget.setPointerCapture(event.pointerId); + dragLastPositionRef.current = { x: event.screenX, y: event.screenY }; + }; + const handleHudDragPointerMove = (event: React.PointerEvent) => { + const lastPosition = dragLastPositionRef.current; + if (!lastPosition) return; + const deltaX = event.screenX - lastPosition.x; + const deltaY = event.screenY - lastPosition.y; + dragLastPositionRef.current = { x: event.screenX, y: event.screenY }; + window.electronAPI?.moveHudOverlayBy?.(deltaX, deltaY); + }; + const handleHudDragPointerEnd = (event: React.PointerEvent) => { + dragLastPositionRef.current = null; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + setHudMouseEventsEnabled(false); + }; return ( // Root fills the HUD window only. Avoid w-screen/h-screen (100vw/100vh): @@ -367,10 +404,15 @@ export function LaunchWindow() { className={`h-full w-full min-w-0 max-w-full overflow-x-hidden overflow-y-hidden bg-transparent ${styles.electronDrag}`} onPointerMove={(event) => { const target = event.target as HTMLElement | null; - const shouldCapture = Boolean(target?.closest("[data-hud-interactive='true']")); - window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(!shouldCapture); + const shouldCapture = + isLanguageMenuOpen || Boolean(target?.closest("[data-hud-interactive='true']")); + setHudMouseEventsEnabled(shouldCapture); + }} + onPointerLeave={() => { + if (!isLanguageMenuOpen) { + setHudMouseEventsEnabled(false); + } }} - onPointerLeave={() => window.electronAPI?.setHudOverlayIgnoreMouseEvents?.(true)} > {systemLocaleSuggestion && (
setHudMouseEventsEnabled(true)} + onPointerDown={() => setHudMouseEventsEnabled(true)} + onMouseEnter={() => setHudMouseEventsEnabled(true)} + onMouseLeave={() => { + if (!isLanguageMenuOpen) { + setHudMouseEventsEnabled(false); + } + }} > {/* Drag handle */} -
+
{getIcon("drag", "text-white/30")}
@@ -609,7 +665,7 @@ export function LaunchWindow() { ? getIcon("webcamOn", "text-green-400") : getIcon("webcamOff", "text-white/40")} - {isWindows && ( + {supportsCursorModeToggle && ( - + {canPauseRecording && ( + + + + )}