Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
4 changes: 4 additions & 0 deletions Fluid.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */; };
7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */; };
803000000000000000000002 /* MediaPlaybackServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 803000000000000000000001 /* MediaPlaybackServiceTests.swift */; };
80A000000000000000000002 /* SystemAudioMuteServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 80A000000000000000000001 /* SystemAudioMuteServiceTests.swift */; };
0B05F11E000000000000A002 /* SupportedFileExtensionsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */; };
803000000000000000000003 /* MediaRemoteAdapter in Frameworks */ = {isa = PBXBuildFile; productRef = 803000000000000000000004 /* MediaRemoteAdapter */; };
B51800000000000000000002 /* SpokenSendTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B51800000000000000000001 /* SpokenSendTests.swift */; };
Expand Down Expand Up @@ -86,6 +87,7 @@
CD1C7A0000000000000000B1 /* CustomDictionaryManualEntryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CustomDictionaryManualEntryTests.swift; sourceTree = "<group>"; };
7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TypingServiceTransientPasteboardTests.swift; sourceTree = "<group>"; };
803000000000000000000001 /* MediaPlaybackServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaPlaybackServiceTests.swift; sourceTree = "<group>"; };
80A000000000000000000001 /* SystemAudioMuteServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SystemAudioMuteServiceTests.swift; sourceTree = "<group>"; };
0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SupportedFileExtensionsTests.swift; sourceTree = "<group>"; };
7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioFixtureLoader.swift; sourceTree = "<group>"; };
7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */ = {isa = PBXFileReference; lastKnownFileType = audio.wav; path = dictation_fixture.wav; sourceTree = "<group>"; };
Expand Down Expand Up @@ -170,6 +172,7 @@
DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */,
7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */,
803000000000000000000001 /* MediaPlaybackServiceTests.swift */,
80A000000000000000000001 /* SystemAudioMuteServiceTests.swift */,
B51800000000000000000001 /* SpokenSendTests.swift */,
B51900000000000000000001 /* PrivateAIProviderPromptFormatTests.swift */,
B52000000000000000000001 /* PrivateAIDictationTokenBudgetTests.swift */,
Expand Down Expand Up @@ -347,6 +350,7 @@
DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */,
7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */,
803000000000000000000002 /* MediaPlaybackServiceTests.swift in Sources */,
80A000000000000000000002 /* SystemAudioMuteServiceTests.swift in Sources */,
B51800000000000000000002 /* SpokenSendTests.swift in Sources */,
B51900000000000000000002 /* PrivateAIProviderPromptFormatTests.swift in Sources */,
C0DE63700000000000000002 /* WhisperLanguageSelectionTests.swift in Sources */,
Expand Down
3 changes: 3 additions & 0 deletions Sources/Fluid/Persistence/BackupService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,10 @@ struct SettingsBackupPayload: Codable, Equatable {
let continuousDictationModeEnabled: Bool?
let continuousDictationSpacingEnabled: Bool?
let contextAwareCapitalizationEnabled: Bool?
/// Kept for backups restored by builds that predate the three-way choice.
let pauseMediaDuringTranscription: Bool
// Optional so backups created before the three-way playback choice still decode.
let recordingPlaybackBehavior: SettingsStore.RecordingPlaybackBehavior?
let automaticDictionaryLearningEnabled: Bool?
let automaticDictionarySuggestionFrequency: SettingsStore.AutomaticDictionarySuggestionFrequency?
let pronunciationMatchingEnabled: Bool?
Expand Down
52 changes: 45 additions & 7 deletions Sources/Fluid/Persistence/SettingsStore.swift
Original file line number Diff line number Diff line change
Expand Up @@ -3338,7 +3338,8 @@ final class SettingsStore: ObservableObject {
continuousDictationModeEnabled: self.continuousDictationModeEnabled,
continuousDictationSpacingEnabled: self.continuousDictationSpacingEnabled,
contextAwareCapitalizationEnabled: self.contextAwareCapitalizationEnabled,
pauseMediaDuringTranscription: self.pauseMediaDuringTranscription,
pauseMediaDuringTranscription: self.recordingPlaybackBehavior == .pause,
recordingPlaybackBehavior: self.recordingPlaybackBehavior,
automaticDictionaryLearningEnabled: self.automaticDictionaryLearningEnabled,
automaticDictionarySuggestionFrequency: self.automaticDictionarySuggestionFrequency,
pronunciationMatchingEnabled: self.pronunciationMatchingEnabled,
Expand Down Expand Up @@ -3515,7 +3516,8 @@ final class SettingsStore: ObservableObject {
self.continuousDictationModeEnabled = restoredContinuousDictationModeEnabled
self.continuousDictationSpacingEnabled = payload.continuousDictationSpacingEnabled ?? restoredContinuousDictationModeEnabled
self.contextAwareCapitalizationEnabled = payload.contextAwareCapitalizationEnabled ?? restoredContinuousDictationModeEnabled
self.pauseMediaDuringTranscription = payload.pauseMediaDuringTranscription
self.recordingPlaybackBehavior = payload.recordingPlaybackBehavior
?? (payload.pauseMediaDuringTranscription ? .pause : .keepPlaying)
if let automaticDictionaryLearningEnabled = payload.automaticDictionaryLearningEnabled {
self.automaticDictionaryLearningEnabled = automaticDictionaryLearningEnabled
}
Expand Down Expand Up @@ -4535,16 +4537,50 @@ final class SettingsStore: ObservableObject {

// MARK: - Media Playback Control

/// When enabled, automatically pauses system media playback when transcription starts.
/// Only resumes if FluidVoice was the one that paused it.
var pauseMediaDuringTranscription: Bool {
get { self.defaults.object(forKey: Keys.pauseMediaDuringTranscription) as? Bool ?? false }
/// What happens to audio already playing on the Mac while a recording runs.
enum RecordingPlaybackBehavior: String, Codable, CaseIterable, Identifiable {
/// Leave other audio untouched.
case keepPlaying
/// Ask the Now Playing app to pause, and resume it afterwards.
case pause
/// Silence the output device, and restore it afterwards.
case mute

var id: String { self.rawValue }

var displayName: String {
switch self {
case .keepPlaying: "Keep Playing"
case .pause: "Pause"
case .mute: "Mute"
}
}
}

/// Chooses between leaving playback alone, pausing it, and muting the output
/// device. Either intervention is reverted only when FluidVoice was the one
/// that made it.
var recordingPlaybackBehavior: RecordingPlaybackBehavior {
get {
if let stored = self.defaults.string(forKey: Keys.recordingPlaybackBehavior),
let behavior = RecordingPlaybackBehavior(rawValue: stored)
{
return behavior
}
return self.legacyPauseMediaDuringTranscription ? .pause : .keepPlaying
}
set {
objectWillChange.send()
self.defaults.set(newValue, forKey: Keys.pauseMediaDuringTranscription)
self.defaults.set(newValue.rawValue, forKey: Keys.recordingPlaybackBehavior)
}
}

/// The boolean toggle this setting replaced. Read only, to migrate users who
/// had media pausing enabled before the behavior became a three-way choice.
private var legacyPauseMediaDuringTranscription: Bool {
self.defaults.object(forKey: Keys.pauseMediaDuringTranscription) as? Bool ?? false
}

// MARK: - Custom Dictionary

/// A custom dictionary entry that maps multiple misheard/alternate spellings to a correct replacement.
Expand Down Expand Up @@ -5558,6 +5594,8 @@ private extension SettingsStore {
static let transcriptionPreviewCharLimit = "TranscriptionPreviewCharLimit"

/// Media Playback Control
static let recordingPlaybackBehavior = "RecordingPlaybackBehavior"
/// Replaced by `recordingPlaybackBehavior`; still read to migrate existing users.
static let pauseMediaDuringTranscription = "PauseMediaDuringTranscription"

/// Custom Dictation Prompt
Expand Down
58 changes: 56 additions & 2 deletions Sources/Fluid/Services/ASRService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,17 @@ final class ASRService: ObservableObject {
if self.isRunning {
await self.stopWithoutTranscription()
}

// Only once capture has ended, so terminating mid-recording cannot make
// playback audible while the pipeline is still running. A stop already
// in flight owns the buffer handoff, which makes the call above return
// immediately, so it is left to restore after it freezes its own capture
// boundary. What remains here is ownership left over from a device that
// was unplugged during an earlier teardown, held only in memory.
if self.isRunning == false, self.isStoppingFinalTranscription == false {
SystemAudioMuteService.shared.restoreIfMuted()
Comment thread
emilevictorportenart marked this conversation as resolved.
}

let audioEngineShutdownStartedAt = Date().timeIntervalSince1970
await self.retireAudioEngineAndWait(reason: "app_termination")
self.benchmarkLog(
Expand Down Expand Up @@ -2150,6 +2161,34 @@ final class ASRService: ObservableObject {
self.benchmarkCompletedStreamingChunks = 0
self.benchmarkLastChunkSampleCount = 0
(self.transcriptionProvider as? FluidAudioProvider)?.resetStreamingPreviewCache()

// Read once and use for both interventions below. They are applied
// either side of the wait for first PCM, so reading the setting twice
// would let a change made during that wait produce a recording with
// both policies or neither.
let playbackBehavior = SettingsStore.shared.recordingPlaybackBehavior

// Hand back any device an earlier recording could not restore, whatever
// the current setting is. Gating this on `.mute` would leave a device
// silent through a whole recording after the user switched away from it.
SystemAudioMuteService.shared.restoreIfMuted()

// Silence the output before the pipeline retains its first packet.
// Muting after capture is live would let the playback this mode exists
// to suppress into the opening of the recording. Setting a CoreAudio
// property is synchronous and cheap, unlike the Now Playing query
// below, so it does not delay the first PCM packet. It also silences
// our own start cue, which is inherent to muting the output device.
// Every start-failure and teardown path below restores it.
if playbackBehavior == .mute {
// A cue from the previous recording can still owe the system volume
// a restore. Let it land before sampling, or we would record the
// lowered cue volume as the value to put back and the deferred write
// would raise the volume again mid-recording.
TranscriptionSoundPlayer.shared.finishPendingVolumeRestore()
SystemAudioMuteService.shared.muteIfAudible()
Comment thread
emilevictorportenart marked this conversation as resolved.
}

self.audioCapturePipeline.setRecordingEnabled(
true,
sessionID: captureSessionID,
Expand Down Expand Up @@ -2357,12 +2396,13 @@ final class ASRService: ObservableObject {
"✅ Audio capture running after first PCM (session=\(captureSessionID))",
source: "ASRService"
)

onCaptureStarted?()

// Pause only after capture is live so media control cannot delay the
// first PCM packet. A quick stop while this await is in flight is
// handled explicitly below.
if SettingsStore.shared.pauseMediaDuringTranscription {
if playbackBehavior == .pause {
let didPause = await MediaPlaybackService.shared.pauseIfPlaying()
guard self.isRunning, self.isStoppingFinalTranscription == false else {
if didPause {
Expand Down Expand Up @@ -2426,7 +2466,8 @@ final class ASRService: ObservableObject {
DebugLogger.shared.error("Failed to start ASR session: \(error)", source: "ASRService")
}

// Resume media if we paused it before the failure
// Restore audio we silenced or paused before the failure
SystemAudioMuteService.shared.restoreIfMuted()
if self.didPauseMediaForThisSession {
await MediaPlaybackService.shared.resumeIfWePaused(true)
self.didPauseMediaForThisSession = false
Expand Down Expand Up @@ -2761,6 +2802,14 @@ final class ASRService: ObservableObject {
// without appending audio from the next session.
self.audioCapturePipeline.markRecordingEnd(atHostTime: mach_absolute_time())

// Restore audio only once that boundary is frozen, so re-enabling the
// speakers cannot contaminate the final packet. This must stay ahead of
// onCaptureStopped below: the stop cue is played there and relies on the
// output already being audible, which is why TranscriptionSoundPlayer
// suppresses only the start cue under Mute. It also sits ahead of every
// later return path, and is a no-op when this session silenced nothing.
SystemAudioMuteService.shared.restoreIfMuted()

// Stop monitoring device to prevent callbacks after stop
DebugLogger.shared.debug("👁️ Stopping device monitoring...", source: "ASRService")
self.stopMonitoringDevice()
Expand Down Expand Up @@ -3311,6 +3360,10 @@ final class ASRService: ObservableObject {
self.isRunning = false
self.audioCapturePipeline.setRecordingEnabled(false)

// Restore only once the pipeline has stopped retaining audio. Covers
// cancellation and app termination, which both land here.
SystemAudioMuteService.shared.restoreIfMuted()

// Stop monitoring device
self.stopMonitoringDevice()

Expand Down Expand Up @@ -4252,6 +4305,7 @@ final class ASRService: ObservableObject {
if invalidation.reason == "audio_service_restarted" {
self.reestablishAudioHardwareListenersAfterServiceReset()
AppServices.shared.audioObserver.restartObservingAfterAudioServiceReset()
SystemAudioMuteService.shared.reestablishObserversAfterAudioServiceReset()
}
DebugLogger.shared.warning(
"Direct capture generation \(invalidation.generation) invalidated by " +
Expand Down
Loading
Loading