diff --git a/Info.plist b/Info.plist index 6b6d430..2a1e688 100644 --- a/Info.plist +++ b/Info.plist @@ -19,7 +19,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 2.06 + 2.07 CFBundleVersion 1 LSMinimumSystemVersion diff --git a/package.sh b/package.sh index 5443d45..22bce18 100755 --- a/package.sh +++ b/package.sh @@ -13,7 +13,7 @@ rm -rf "$APP_BUNDLE" # 1. Compile Swift sources (Pure Native Swift) swiftc -O -target arm64-apple-macosx13.0 \ - -framework Cocoa -framework AVFoundation -framework Carbon -framework AppIntents \ + -framework Cocoa -framework AVFoundation -framework Carbon -framework AppIntents -framework Network \ "$PROJECT_DIR"/src/*.swift \ -o "$BUILD_DIR/SonioxVoice" diff --git a/src/AppDelegate.swift b/src/AppDelegate.swift index 5617108..96b5cbb 100644 --- a/src/AppDelegate.swift +++ b/src/AppDelegate.swift @@ -1,51 +1,68 @@ import Cocoa import AVFoundation -import ApplicationServices public final class AppDelegate: NSObject, NSApplicationDelegate { - public static private(set) var shared: AppDelegate? - - public override init() { - super.init() - AppDelegate.shared = self - } + public static weak var shared: AppDelegate? private var statusBarController: StatusBarController! - private var audioRecorder = AudioRecorder() + private let audioRecorder = AudioRecorder() private var activeSession: SonioxLiveSession? private var isBusyFinalizing = false + private var isNetworkDegraded = false private var currentAudioLevel: Float = 0.0 + private var lastRecordedAudio = Data() + private var recordingStartTime = Date() + private var lastVoiceDetectedTime = Date() + private var hasSpokenInCurrentSession = false private var handsFreeSilenceTimer: Timer? - private var lastVoiceDetectedTime: Date = Date() - private var recordingStartTime: Date = Date() - private var hasSpokenInCurrentSession: Bool = false - private var pendingStopWorkItem: DispatchWorkItem? = nil - private var isNetworkDegraded: Bool = false - private var lastRecordedAudio: Data = Data() - private var finalizeWatchdogWorkItem: DispatchWorkItem? = nil + private var pendingStopWorkItem: DispatchWorkItem? + private var finalizeWatchdogWorkItem: DispatchWorkItem? + + // Cancellation token incremented on every Escape press or new recording session + private var operationGeneration: UInt64 = 0 + private let genLock = NSLock() + + private func currentGeneration() -> UInt64 { + genLock.lock() + defer { genLock.unlock() } + return operationGeneration + } + + @discardableResult + private func bumpGeneration() -> UInt64 { + genLock.lock() + defer { genLock.unlock() } + operationGeneration &+= 1 + return operationGeneration + } public func applicationDidFinishLaunching(_ notification: Notification) { - setbuf(stdout, nil) - setbuf(stderr, nil) - if UserDefaults.standard.object(forKey: "SonioxPlaySounds") == nil { - UserDefaults.standard.set(true, forKey: "SonioxPlaySounds") - } + AppDelegate.shared = self + NSApp.setActivationPolicy(.accessory) + + // Register URL handler for soniox://start, soniox://stop, soniox://toggle, soniox://cancel + NSAppleEventManager.shared().setEventHandler( + self, + andSelector: #selector(handleGetURLEvent(_:withReplyEvent:)), + forEventClass: AEEventClass(kInternetEventClass), + andEventID: AEEventID(kAEGetURL) + ) - // Setup DistributedNotificationCenter observers for Shortcuts AppIntents & external triggers + // Register DistributedNotificationCenter for Siri / Shortcuts / CLI trigger DistributedNotificationCenter.default().addObserver( self, - selector: #selector(handleDistributedStartRecording), + selector: #selector(handleExternalStartNotification(_:)), name: NSNotification.Name("com.soniox.voice.start"), object: nil ) DistributedNotificationCenter.default().addObserver( self, - selector: #selector(handleDistributedStopRecording), + selector: #selector(handleExternalStopNotification(_:)), name: NSNotification.Name("com.soniox.voice.stop"), object: nil ) DistributedNotificationCenter.default().addObserver( self, - selector: #selector(handleDistributedToggleRecording), + selector: #selector(handleExternalToggleNotification(_:)), name: NSNotification.Name("com.soniox.voice.toggle"), object: nil ) @@ -55,7 +72,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { self?.toggleRecording() } - // Setup Hotkey manager callbacks (Push-To-Talk & Toggle) + // Setup Hotkey manager callbacks (Push-To-Talk, Toggle, and Escape Cancel) HotkeyManager.shared.onHotkeyPressed = { [weak self] in guard let self = self else { return } self.pendingStopWorkItem?.cancel() @@ -73,97 +90,156 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { guard let self = self else { return } if HotkeyManager.shared.currentMode == .pushToTalk { if self.audioRecorder.isRecording { + let holdDuration = Date().timeIntervalSince(self.recordingStartTime) + if holdDuration < 0.25 && !self.hasSpokenInCurrentSession { + self.cancelShortTapRecording() + return + } self.pendingStopWorkItem?.cancel() - let workItem = DispatchWorkItem { [weak self] in - guard let self = self else { return } - if self.audioRecorder.isRecording { - self.stopRecording() - } + let work = DispatchWorkItem { [weak self] in + guard let self = self, self.audioRecorder.isRecording else { return } + self.stopRecording() } - self.pendingStopWorkItem = workItem - // Trailing grace period: 0.35s (350ms) buffer after key release so trailing fast speech is never clipped - DispatchQueue.main.asyncAfter(deadline: .now() + 0.35, execute: workItem) + self.pendingStopWorkItem = work + DispatchQueue.main.asyncAfter(deadline: .now() + 0.65, execute: work) } } } - // Initial Hotkey Registration (immediate and deferred for AX permission readiness) - HotkeyManager.shared.registerHotkeys() - DispatchQueue.main.asyncAfter(deadline: .now() + 0.8) { - HotkeyManager.shared.registerHotkeys() + // Escape Key Handler: Immediately aborts active recording, transcription, recovery, and hides the green orb! + HotkeyManager.shared.onEscapePressed = { [weak self] in + guard let self = self else { return false } + let isActive = self.audioRecorder.isRecording || self.isBusyFinalizing || (self.activeSession != nil) || HUDOverlayController.shared.isVisible + guard isActive else { return false } + + DispatchQueue.main.async { + self.cancelEverythingImmediately() + } + return true // Consume Escape so it only cancels Soniox Voice } - // Listen to live audio levels & update HUD + HotkeyManager.shared.registerHotkeys() + + // Wire audio level updates -> HUD Orb + Silence Detection + Silent Stall Check audioRecorder.onAudioLevelUpdate = { [weak self] level in guard let self = self else { return } self.currentAudioLevel = level - if level > 0.07 { - self.hasSpokenInCurrentSession = true - self.lastVoiceDetectedTime = Date() - } if self.audioRecorder.isRecording { - HUDOverlayController.shared.show(state: .recording(level: level, isLagging: self.isNetworkDegraded, pingQuality: PingMonitor.shared.currentQuality)) + if level > 0.07 { + self.lastVoiceDetectedTime = Date() + self.hasSpokenInCurrentSession = true + self.activeSession?.checkSilentStallWhileSpeaking() + } + HUDOverlayController.shared.show( + state: .recording( + level: level, + isLagging: self.isNetworkDegraded, + pingQuality: PingMonitor.shared.currentQuality + ) + ) } } - // Listen to live ping updates and push to HUD + // Wire live 16kHz PCM audio chunks directly to active WebSocket session + audioRecorder.onAudioChunkAvailable = { [weak self] chunk in + self?.activeSession?.sendAudioChunk(chunk) + } + + // Start live ping monitor & wire network restoration hooks PingMonitor.shared.onPingUpdate = { [weak self] quality in guard let self = self else { return } if self.audioRecorder.isRecording { - HUDOverlayController.shared.show(state: .recording(level: self.currentAudioLevel, isLagging: self.isNetworkDegraded, pingQuality: quality)) + HUDOverlayController.shared.show( + state: .recording( + level: self.currentAudioLevel, + isLagging: self.isNetworkDegraded, + pingQuality: quality + ) + ) } } - PingMonitor.shared.start() - // Stream audio chunk live directly to active Soniox session in real-time - audioRecorder.onAudioChunkAvailable = { [weak self] chunk in - self?.activeSession?.sendAudioChunk(chunk) + PingMonitor.shared.onNetworkRestored = { [weak self] in + guard let self = self else { return } + SonioxSessionPool.shared.reconnectAll() + let gen = self.currentGeneration() + OfflineAudioQueueManager.shared.processPendingQueueIfNeeded(isCancelled: { [weak self] in + guard let self = self else { return true } + return self.currentGeneration() != gen + }) } - // Pre-warm the background WebSocket session for 0ms startup delay - SonioxSessionPool.shared.prewarmNextSession() + PingMonitor.shared.onInterfaceChanged = { + SonioxSessionPool.shared.reconnectAll() + } + + OfflineAudioQueueManager.shared.onOfflineAudioRecovered = { [weak self] recoveredText in + guard let self = self else { return } + let processed = TextProcessor.shared.process(text: recoveredText) + guard !processed.isEmpty else { return } + HistoryManager.shared.addEntry(processed) + let pb = NSPasteboard.general + pb.clearContents() + pb.setString(processed, forType: .string) + self.playSound(named: "Glass") + print("AppDelegate: ✅ Offline queued speech recovered and placed on clipboard & history!") + } - // Pre-configure audio hardware capture graph (kept STOPPED until key is pressed) - try? audioRecorder.ensureSessionConfigured() + PingMonitor.shared.start() - // Check permissions on start - checkInitialPermissions() - setupDarwinNotifications() - } - - private func checkInitialPermissions() { - // 1. Microphone - audioRecorder.requestMicrophonePermission { granted in - if !granted { - print("Microphone permission not granted yet.") + // Initialize Window State Manager + _ = WindowStateManager.shared + + // Request Microphone Permission & Warm up Audio + WebSocket Pool + audioRecorder.requestMicrophonePermission { [weak self] granted in + print("Microphone permission granted: \(granted)") + self?.statusBarController.buildMenu() + if granted { + try? self?.audioRecorder.ensureSessionConfigured() + SonioxSessionPool.shared.prewarmNextSession() + OfflineAudioQueueManager.shared.processPendingQueueIfNeeded() } } - // 2. Accessibility - if !AXIsProcessTrusted() { - let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary - AXIsProcessTrustedWithOptions(options) - } + let isAx = AXIsProcessTrusted() + print("Accessibility trusted: \(isAx)") } - private func playSystemSound(name: String) { - guard SonioxSettings.shared.playSounds else { return } - DispatchQueue.main.async { - if let sound = NSSound(named: NSSound.Name(name)) { - sound.volume = 1.0 - sound.play() - } else { - DispatchQueue.global(qos: .userInitiated).async { - let p = Process() - p.executableURL = URL(fileURLWithPath: "/usr/bin/afplay") - p.arguments = ["/System/Library/Sounds/\(name).aiff"] - try? p.run() - } - } - } + /// Completely aborts any ongoing recording, finalization, or network recovery when Escape is pressed + public func cancelEverythingImmediately() { + print("AppDelegate: 🛑 ESCAPE pressed! Aborting all active recording, transcription, and recovery immediately.") + bumpGeneration() + + pendingStopWorkItem?.cancel() + pendingStopWorkItem = nil + finalizeWatchdogWorkItem?.cancel() + finalizeWatchdogWorkItem = nil + handsFreeSilenceTimer?.invalidate() + handsFreeSilenceTimer = nil + + isBusyFinalizing = false + isNetworkDegraded = false + hasSpokenInCurrentSession = false + lastRecordedAudio.removeAll() + + activeSession?.abort() + activeSession = nil + + audioRecorder.abortRecording() + InputBoxManager.shared.resetSession() + HotkeyManager.shared.resetKeyState() + OfflineAudioQueueManager.shared.clearAllPending() + + PingMonitor.shared.setActiveRecordingMode(false) + statusBarController.updateIcon(state: .idle) + statusBarController.buildMenu(isRecording: false) + HUDOverlayController.shared.hideImmediately() + + // Pre-warm a fresh standby WebSocket session so the app is 100% ready for the next voice input + SonioxSessionPool.shared.prewarmNextSession() } - public func toggleRecording() { + @objc public func toggleRecording() { if audioRecorder.isRecording { stopRecording() } else { @@ -173,10 +249,17 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { public func startRecording() { if audioRecorder.isRecording { return } + + // Cancel any lingering background finalization/recovery from previous speech so user never gets blocked + let myGen = bumpGeneration() + pendingStopWorkItem?.cancel() + pendingStopWorkItem = nil finalizeWatchdogWorkItem?.cancel() finalizeWatchdogWorkItem = nil if isBusyFinalizing { - print("AppDelegate: isBusyFinalizing was active when startRecording invoked. Clearing stale lock.") + print("AppDelegate: Cancelling previous finalizing/recovery session to start new recording immediately.") + activeSession?.abort() + activeSession = nil isBusyFinalizing = false } @@ -186,13 +269,16 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { lastVoiceDetectedTime = Date() hasSpokenInCurrentSession = false + // Switch PingMonitor to 1.0s active cadence while HUD is shown + PingMonitor.shared.setActiveRecordingMode(true) + // Start Hands-free silence watchdog timer (checks every 250ms) handsFreeSilenceTimer?.invalidate() handsFreeSilenceTimer = Timer.scheduledTimer(withTimeInterval: 0.25, repeats: true) { [weak self] _ in self?.checkHandsFreeSilence() } - // 1. Ensure target input box is focused without stealing user cursor + // 1. Reset session state without stealing focus InputBoxManager.shared.ensureFocusedInputBox() // 2. Acquire pre-warmed / fast-connected WebSocket session @@ -202,20 +288,28 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { // Handle network degradation warning session.onNetworkDegraded = { [weak self] degraded in - guard let self = self else { return } + guard let self = self, self.currentGeneration() == myGen else { return } self.isNetworkDegraded = degraded if self.audioRecorder.isRecording { - HUDOverlayController.shared.show(state: .recording(level: self.currentAudioLevel, isLagging: degraded)) + HUDOverlayController.shared.show(state: .recording(level: self.currentAudioLevel, isLagging: degraded, pingQuality: PingMonitor.shared.currentQuality)) } } + session.onEndTriggerDetected = { [weak self] in + guard let self = self, self.currentGeneration() == myGen else { return } + print("AppDelegate: In-flight end trigger ('and'/'تمام') detected! Stopping recording immediately...") + self.stopRecording() + } + // Stream live finalized tokens directly into input box word-by-word session.onCommittedDelta = { [weak self] delta in - guard let self = self else { return } + guard let self = self, self.currentGeneration() == myGen else { return } + + self.hasSpokenInCurrentSession = true + self.lastVoiceDetectedTime = Date() - // Check for in-stream end trigger ("and", "تمام", "end") let deltaWords = delta.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } - if let lastWord = deltaWords.last?.replacingOccurrences(of: #"[.,!?:;،؛؟"'()\[\]{}«»–—\-]"#, with: "", options: .regularExpression).lowercased() { + if let lastWord = deltaWords.last.map({ InputBoxManager.stripPunctuation($0).lowercased() }) { if lastWord == "and" || lastWord == "تمام" || lastWord == "end" { print("AppDelegate: In-speech end trigger detected ('\(lastWord)'). Finalizing speech...") DispatchQueue.main.async { @@ -225,395 +319,392 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { } } - self.hasSpokenInCurrentSession = true - self.lastVoiceDetectedTime = Date() - - // If it's an executive slash command, do NOT type it into the input box if SonioxSettings.shared.voiceCommands && VoiceCommandExecutor.shared.isSlashCommand(text: delta) { return } InputBoxManager.shared.streamDelta(text: delta) } - session.onEndTriggerDetected = { [weak self] in - guard let self = self else { return } - print("AppDelegate: In-flight end trigger ('and'/'تمام') detected. Stopping recording immediately...") - DispatchQueue.main.async { - self.stopRecording() - } - } - - session.onFinalResult = { [weak self] result in - DispatchQueue.main.async { - guard let self = self else { return } - if self.audioRecorder.isRecording { - if HotkeyManager.shared.isPhysicalKeyDown { - print("AppDelegate: Session finished/failed while user is physically holding hotkey. Marking degraded; recovery will run upon release.") - self.isNetworkDegraded = true - HUDOverlayController.shared.show(state: .recording(level: self.currentAudioLevel, isLagging: true)) - return - } else { - print("AppDelegate: Session finished in hands-free/Siri mode. Stopping recording and finalizing.") - self.stopRecording() - return - } - } - self.handleFinalResult(result) + session.onPartialText = { [weak self] partial in + guard let self = self, self.currentGeneration() == myGen else { return } + let trimmed = partial.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { + self.hasSpokenInCurrentSession = true + self.lastVoiceDetectedTime = Date() } } do { try audioRecorder.startRecording() - playSystemSound(name: "Tink") statusBarController.updateIcon(state: .recording) statusBarController.buildMenu(isRecording: true) - HUDOverlayController.shared.show(state: .recording(level: 0.1, isLagging: false, pingQuality: PingMonitor.shared.currentQuality)) + HUDOverlayController.shared.show( + state: .recording( + level: 0.05, + isLagging: false, + pingQuality: PingMonitor.shared.currentQuality + ) + ) + playStartSound() } catch { - print("Failed to start audio recording:", error) - activeSession?.cancel() + print("Failed to start recording: \(error)") + PingMonitor.shared.setActiveRecordingMode(false) + activeSession?.abort() activeSession = nil - HUDOverlayController.shared.show(state: .error(message: "REC ERROR")) + HUDOverlayController.shared.show(state: .error(message: "MIC ERR")) } } - private func hasAudibleSpeech(pcmData: Data) -> Bool { - // Less than 0.25s of audio is just a momentary key tap/click - guard pcmData.count >= 8000 else { return false } - var sumSquares: Double = 0 - var peak: Int16 = 0 - let count = pcmData.count / 2 - pcmData.withUnsafeBytes { raw in - let ptr = raw.bindMemory(to: Int16.self) - for i in 0.. peak { peak = absSample } - let normalized = Double(sample) / 32768.0 - sumSquares += normalized * normalized - } - } - let rms = sqrt(sumSquares / Double(count)) - // Raised thresholds: filter out mechanical keyclicks and keyboard taps - return peak > 2500 && rms > 0.015 + private func cancelShortTapRecording() { + print("AppDelegate: Short hotkey tap (<0.25s) with no speech detected. Cancelling immediately.") + bumpGeneration() + pendingStopWorkItem?.cancel() + pendingStopWorkItem = nil + handsFreeSilenceTimer?.invalidate() + handsFreeSilenceTimer = nil + finalizeWatchdogWorkItem?.cancel() + finalizeWatchdogWorkItem = nil + isBusyFinalizing = false + + audioRecorder.abortRecording() + lastRecordedAudio.removeAll() + activeSession?.abort() + activeSession = nil + + PingMonitor.shared.setActiveRecordingMode(false) + statusBarController.updateIcon(state: .idle) + statusBarController.buildMenu(isRecording: false) + HUDOverlayController.shared.hideImmediately() } public func stopRecording() { + guard audioRecorder.isRecording, !isBusyFinalizing else { return } + let myGen = currentGeneration() + isBusyFinalizing = true + pendingStopWorkItem?.cancel() + pendingStopWorkItem = nil handsFreeSilenceTimer?.invalidate() handsFreeSilenceTimer = nil - HotkeyManager.shared.resetKeyState() - guard audioRecorder.isRecording else { return } - activeSession?.setRecordingActive(false) + statusBarController.updateIcon(state: .transcribing) + statusBarController.buildMenu(isRecording: false) + HUDOverlayController.shared.show(state: .transcribing) + playSound(named: "Pop") + let pcmData = audioRecorder.stopRecording() self.lastRecordedAudio = pcmData - // Fast-path: If user tapped key or did NOT speak (silence / keyclick), dismiss immediately with ZERO delay! - let hasSpeech = hasAudibleSpeech(pcmData: pcmData) - if (!hasSpeech && !InputBoxManager.shared.hasStreamedText) || pcmData.count < 8000 { - print("AppDelegate: Short tap or silence detected. Immediate 0ms dismissal.") - activeSession?.cancel() - activeSession = nil - isBusyFinalizing = false - statusBarController.updateIcon(state: .idle) - statusBarController.buildMenu(isRecording: false) - HUDOverlayController.shared.hide() + guard pcmData.count > 1600 else { // Less than 0.05s of audio + finishFinalization(text: nil, error: nil, generation: myGen) return } - playSystemSound(name: "Pop") - statusBarController.updateIcon(state: .transcribing) - statusBarController.buildMenu(isRecording: false) + guard let session = activeSession else { + attemptFullAudioRecovery(reason: "No active live session", generation: myGen) + return + } - isBusyFinalizing = true + session.setRecordingActive(false) + let audioSeconds = Double(pcmData.count) / 32000.0 - // Check if the live session suffered a network failure during recording - let sessionDropped = (self.activeSession == nil) || (self.activeSession?.hasNetworkFailure == true) || self.isNetworkDegraded - if sessionDropped { - print("AppDelegate: Live session dropped/degraded during recording. Triggering Smart Audio Recovery immediately.") - startSmartAudioRecovery(pcmData: pcmData, reason: "Live session dropped mid-recording") - return + // FAST PATH FOR SILENT STALL / OFFLINE NETWORK: + // If user spoke (>0.8s) and socket is dead, disconnected, or had a silent TCP stall with 0 tokens, + // do NOT wait 4.0s-10s for finalize timeout — jump straight into Smart Audio Recovery in 0ms! + if hasSpokenInCurrentSession && audioSeconds >= 0.8 { + if !PingMonitor.shared.isPathSatisfied || !session.isAlive || (session.hasSilentStall && session.totalReceivedTokensCount == 0) { + print("AppDelegate: ⚡️ Fast-path recovery triggered immediately (Silent stall / dead socket detected during speech).") + session.abort() + self.activeSession = nil + attemptFullAudioRecovery(reason: "Fast-path stall detection", generation: myGen) + return + } } - HUDOverlayController.shared.show(state: .transcribing) + var completed = false + let lock = NSLock() - // Dynamic safety timeout watchdog: scale with audio length & network transit - let durationSec = Double(pcmData.count) / 32000.0 - let finalizeTimeout = max(4.0, min(12.0, durationSec * 0.25 + 3.5)) + session.onFinalResult = { [weak self] result in + guard let self = self, self.currentGeneration() == myGen else { return } + lock.lock() + guard !completed else { lock.unlock(); return } + completed = true + lock.unlock() + + self.finalizeWatchdogWorkItem?.cancel() + self.finalizeWatchdogWorkItem = nil + + switch result { + case .success(let text): + let cleanText = text.trimmingCharacters(in: .whitespacesAndNewlines) + if cleanText.isEmpty && !InputBoxManager.shared.hasStreamedText && (self.hasSpokenInCurrentSession || pcmData.count > 16000) { + print("AppDelegate: Live session returned empty transcript despite recorded audio (\(pcmData.count) bytes). Triggering Smart Audio Recovery...") + self.attemptFullAudioRecovery(reason: "Empty final result", generation: myGen) + } else { + self.finishFinalization(text: text, error: nil, generation: myGen) + } + case .failure(let err): + let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines) + print("AppDelegate: Live session error (\(err.localizedDescription)). Launching full buffered audio recovery...") + self.attemptFullAudioRecovery(reason: err.localizedDescription, partialFallback: bestEffort, generation: myGen) + } + } + // Trigger WebSocket finalization + session.finalizeStream() + + // Dynamic timeout scaled to audio length: 3.5s min up to 8.5s + let finalizeTimeout = min(8.5, max(3.5, 2.5 + (audioSeconds * 0.25))) let watchdog = DispatchWorkItem { [weak self] in - guard let self = self, self.isBusyFinalizing else { return } - print("AppDelegate: Finalize safety watchdog triggered (\(String(format: "%.1f", finalizeTimeout))s) — starting Smart Audio Recovery.") - self.finalizeWatchdogWorkItem = nil - self.startSmartAudioRecovery(pcmData: self.lastRecordedAudio, reason: "TIMEOUT") + guard let self = self, self.currentGeneration() == myGen else { return } + lock.lock() + guard !completed else { lock.unlock(); return } + completed = true + lock.unlock() + + let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines) + session.abort() + + if bestEffort.isEmpty && !InputBoxManager.shared.hasStreamedText && (self.hasSpokenInCurrentSession || pcmData.count > 16000) { + print("AppDelegate: Finalize timeout with empty text. Triggering Smart Audio Recovery...") + self.attemptFullAudioRecovery(reason: "Finalize timeout", generation: myGen) + } else { + self.finishFinalization(text: bestEffort, error: nil, generation: myGen) + } } self.finalizeWatchdogWorkItem = watchdog DispatchQueue.main.asyncAfter(deadline: .now() + finalizeTimeout, execute: watchdog) - - activeSession?.finalizeStream() } - private func handleFinalResult(_ result: Result) { - finalizeWatchdogWorkItem?.cancel() - finalizeWatchdogWorkItem = nil - guard isBusyFinalizing else { return } - self.activeSession = nil - self.statusBarController.updateIcon(state: .idle) - - var rawText = "" - var isNetworkError = false - switch result { - case .success(let text): - rawText = text.trimmingCharacters(in: .whitespacesAndNewlines) - case .failure(let error): - isNetworkError = true - print("Soniox primary finalize error:", error) - } - - let hasSpeech = hasAudibleSpeech(pcmData: self.lastRecordedAudio) - - // 1. Primary Success Path: Non-empty transcript returned - if !rawText.isEmpty { - self.isBusyFinalizing = false - deliverOutputText(rawText) - return - } - - // 2. Resilient Recovery Path: If user actually spoke but server returned empty/error/dropped - if hasSpeech && self.lastRecordedAudio.count >= 8000 { - let reason = isNetworkError ? "NET ERROR" : "RETRYING..." - print("AppDelegate: Primary STT returned empty/error for audible speech. Triggering Smart Audio Recovery...") - startSmartAudioRecovery(pcmData: self.lastRecordedAudio, reason: reason) - return - } + private func attemptFullAudioRecovery(reason: String, partialFallback: String = "", generation: UInt64) { + guard currentGeneration() == generation else { return } + let audioToRecover = self.lastRecordedAudio + let audioSec = Double(audioToRecover.count) / 32000.0 - // 3. Strict Silence / Key-tap Gating: Pure silence or tap - self.isBusyFinalizing = false - HUDOverlayController.shared.hide() - } - - private func startSmartAudioRecovery(pcmData: Data, reason: String) { - finalizeWatchdogWorkItem?.cancel() - finalizeWatchdogWorkItem = nil - guard !pcmData.isEmpty else { - self.isBusyFinalizing = false - self.statusBarController.updateIcon(state: .idle) - HUDOverlayController.shared.hide() + guard audioToRecover.count > 6400 else { + finishFinalization(text: partialFallback.isEmpty ? nil : partialFallback, error: reason, generation: generation) return } - let durationSec = Double(pcmData.count) / 32000.0 - print("AppDelegate: Starting Smart Audio Recovery [\(reason)] with \(pcmData.count) bytes (\(String(format: "%.1f", durationSec))s)...") - - // Show glowing amber recovery state on HUD with English reason - let hudReason = reason.contains("TIMEOUT") ? "TIMEOUT" : "RETRYING..." - HUDOverlayController.shared.show(state: .retrying(reason: hudReason)) - statusBarController.updateIcon(state: .transcribing) - - // Cancel active session to prevent resource leaks - self.activeSession?.cancel() - self.activeSession = nil + HUDOverlayController.shared.show(state: .retrying(reason: "RETRYING...")) - // Concurrently prime a fresh session pool in background for the NEXT speech - SonioxSessionPool.shared.reconnectAll() - - let hints = SonioxSettings.shared.language.hints - let target = SonioxSettings.shared.targetLanguage.isEnabled ? SonioxSettings.shared.targetLanguage.rawValue : nil + let isCancelledCheck: () -> Bool = { [weak self] in + guard let self = self else { return true } + return self.currentGeneration() != generation + } - SonioxLiveSession.retryFullBufferedAudio(pcmData: pcmData, languageHints: hints, targetLanguage: target, onProgress: nil) { [weak self] retryResult in - guard let self = self else { return } - self.isBusyFinalizing = false - self.statusBarController.updateIcon(state: .idle) + SonioxLiveSession.retryFullBufferedAudio( + pcmData: audioToRecover, + languageHints: SonioxSettings.shared.language.hints, + isCancelled: isCancelledCheck, + onStatusUpdate: { [weak self] statusText in + guard let self = self, self.currentGeneration() == generation else { return } + HUDOverlayController.shared.show(state: .retrying(reason: statusText)) + } + ) { [weak self] result in + guard let self = self, self.currentGeneration() == generation else { return } + switch result { + case .success(let recovered): + let trimmed = recovered.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { + VoiceCommandLogger.shared.logRecovery( + audioBytes: audioToRecover.count, + durationSec: audioSec, + reason: reason, + outcome: "SUCCESS (Full Buffered Audio Recovered)", + recoveredText: trimmed + ) + self.finishFinalization(text: trimmed, error: nil, generation: generation) + return + } + case .failure: + break + } - switch retryResult { - case .success(let recoveredText): - let trimmed = recoveredText.trimmingCharacters(in: .whitespacesAndNewlines) - print("AppDelegate: Smart Audio Recovery SUCCEEDED! Recovered \(trimmed.count) chars.") + if !partialFallback.isEmpty { VoiceCommandLogger.shared.logRecovery( - audioBytes: pcmData.count, - durationSec: durationSec, + audioBytes: audioToRecover.count, + durationSec: audioSec, reason: reason, - outcome: "SUCCESS", - recoveredText: trimmed + outcome: "PARTIAL FALLBACK USED", + recoveredText: partialFallback ) - self.deliverOutputText(trimmed) - - case .failure(let error): - print("AppDelegate: Smart Audio Recovery failed completely: \(error)") + self.finishFinalization(text: partialFallback, error: nil, generation: generation) + } else { + // Save to persistent offline queue so when internet comes back in a few minutes, it is automatically transcribed! + OfflineAudioQueueManager.shared.enqueueFailedAudio(pcmData: audioToRecover) VoiceCommandLogger.shared.logRecovery( - audioBytes: pcmData.count, - durationSec: durationSec, + audioBytes: audioToRecover.count, + durationSec: audioSec, reason: reason, - outcome: "FAILED: \(error.localizedDescription)", + outcome: "QUEUED FOR OFFLINE RECOVERY", recoveredText: "" ) - let errMsg: String - let errStr = error.localizedDescription.lowercased() - if errStr.contains("timed out") || errStr.contains("timeout") { - errMsg = "TIMEOUT" - } else if errStr.contains("network") || errStr.contains("connection") || errStr.contains("offline") { - errMsg = "NET ERROR" - } else { - errMsg = "STT ERROR" - } - HUDOverlayController.shared.show(state: .error(message: errMsg)) + self.finishFinalization(text: nil, error: "QUEUED", generation: generation) } } } - private func deliverOutputText(_ text: String) { - guard !text.isEmpty else { + private func finishFinalization(text: String?, error: String?, generation: UInt64) { + guard currentGeneration() == generation else { return } + finalizeWatchdogWorkItem?.cancel() + finalizeWatchdogWorkItem = nil + activeSession?.abort() + activeSession = nil + isBusyFinalizing = false + lastRecordedAudio.removeAll() + + PingMonitor.shared.setActiveRecordingMode(false) + statusBarController.updateIcon(state: .idle) + + if let err = error { + print("Soniox Error: \(err)") + playSound(named: "Basso") + let shortMsg: String + if err == "QUEUED" { + shortMsg = "SAVED" + } else if err.lowercased().contains("timeout") { + shortMsg = "TIMEOUT" + } else if err.lowercased().contains("key") || err.lowercased().contains("offline") || err.lowercased().contains("internet") { + shortMsg = "NET ERR" + } else { + shortMsg = "RETRY ERR" + } + HUDOverlayController.shared.show(state: .error(message: shortMsg)) + return + } + + let rawText = (text ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !rawText.isEmpty else { HUDOverlayController.shared.hide() return } - // 1. Check for Executive Voice Slash Commands - if SonioxSettings.shared.voiceCommands && VoiceCommandExecutor.shared.isSlashCommand(text: text) { - HUDOverlayController.shared.show(state: .success) - VoiceCommandExecutor.shared.execute(rawText: text) - return // STRICT: Slash commands are executed, never pasted! + let isBotCmd = BotCommandProcessor.isBotCommand(text: rawText) + + if !isBotCmd && SonioxSettings.shared.voiceCommands { + if VoiceCommandExecutor.shared.isSlashCommand(text: rawText) { + HUDOverlayController.shared.show(state: .success) + VoiceCommandExecutor.shared.execute(rawText: rawText) + playSound(named: "Glass") + return + } } - // 2. Normal dictation processing & final commit - let processed = TextProcessor.shared.process(text: text) - if !processed.isEmpty { - VoiceCommandLogger.shared.logDictation(rawSpeech: text, processed: processed) - HistoryManager.shared.addEntry(processed) - InputBoxManager.shared.commitFinalText(fullText: processed, addTrailingSpace: SonioxSettings.shared.addTrailingSpace) - HUDOverlayController.shared.show(state: .success) - } else { + let processedText = TextProcessor.shared.process(text: rawText) + guard !processedText.isEmpty else { HUDOverlayController.shared.hide() + return } + + VoiceCommandLogger.shared.logDictation(rawSpeech: rawText, processed: processedText) + HistoryManager.shared.addEntry(processedText) + + let addSpace = isBotCmd ? false : SonioxSettings.shared.addTrailingSpace + InputBoxManager.shared.commitFinalText( + fullText: processedText, + addTrailingSpace: addSpace + ) + + playSound(named: "Glass") + HUDOverlayController.shared.show(state: .success) } - public func applicationShouldHandleReopen(_ sender: NSApplication, hasVisibleWindows flag: Bool) -> Bool { - print("AppDelegate: Application reopen triggered (Siri / Shortcut).") - DispatchQueue.main.async { [weak self] in - guard let self = self else { return } - if self.audioRecorder.isRecording { - print("AppDelegate: Already recording on reopen — stopping and finalizing.") - self.stopRecording() - } else { - self.startRecording() - } - } - return true - } - - public func application(_ application: NSApplication, open urls: [URL]) { - for url in urls { - if url.scheme == "soniox" { - let action = url.host ?? url.path - if action.contains("start") { - DispatchQueue.main.async { [weak self] in - self?.startRecording() - } - } else if action.contains("stop") { - DispatchQueue.main.async { [weak self] in - self?.stopRecording() - } - } else if action.contains("toggle") { - DispatchQueue.main.async { [weak self] in - self?.toggleRecording() - } - } + private func playStartSound() { + guard SonioxSettings.shared.playSounds else { return } + if let sound = NSSound(named: NSSound.Name("Tink")) { + sound.stop() + sound.volume = 1.0 + sound.play() + } else { + let path = "/System/Library/Sounds/Tink.aiff" + if FileManager.default.fileExists(atPath: path) { + let task = Process() + task.executableURL = URL(fileURLWithPath: "/usr/bin/afplay") + task.arguments = ["-v", "1.0", path] + try? task.run() } } } - - public func applicationWillTerminate(_ notification: Notification) { - HotkeyManager.shared.unregisterHotkeys() - } - - private func setupDarwinNotifications() { - let notifyCenter = CFNotificationCenterGetDarwinNotifyCenter() - let selfPtr = Unmanaged.passUnretained(self).toOpaque() - - let startCb: CFNotificationCallback = { (_, observer, _, _, _) in - guard let observer = observer else { return } - let app = Unmanaged.fromOpaque(observer).takeUnretainedValue() - DispatchQueue.main.async { - if !app.audioRecorder.isRecording { - print("AppDelegate: Received Darwin notification 'com.soniox.voice.start'. Starting recording...") - app.startRecording() - } - } - } - - let stopCb: CFNotificationCallback = { (_, observer, _, _, _) in - guard let observer = observer else { return } - let app = Unmanaged.fromOpaque(observer).takeUnretainedValue() - DispatchQueue.main.async { - if app.audioRecorder.isRecording { - print("AppDelegate: Received Darwin notification 'com.soniox.voice.stop'. Stopping recording...") - app.stopRecording() - } - } - } - - let toggleCb: CFNotificationCallback = { (_, observer, _, _, _) in - guard let observer = observer else { return } - let app = Unmanaged.fromOpaque(observer).takeUnretainedValue() - DispatchQueue.main.async { - print("AppDelegate: Received Darwin notification 'com.soniox.voice.toggle'.") - app.toggleRecording() + + private func playSound(named name: String) { + guard SonioxSettings.shared.playSounds else { return } + if let sound = NSSound(named: NSSound.Name(name)) { + sound.volume = 0.75 + sound.play() + } else { + let path = "/System/Library/Sounds/\(name).aiff" + if FileManager.default.fileExists(atPath: path) { + let task = Process() + task.executableURL = URL(fileURLWithPath: "/usr/bin/afplay") + task.arguments = ["-v", "0.75", path] + try? task.run() } } - - CFNotificationCenterAddObserver(notifyCenter, selfPtr, startCb, "com.soniox.voice.start" as CFString, nil, .deliverImmediately) - CFNotificationCenterAddObserver(notifyCenter, selfPtr, stopCb, "com.soniox.voice.stop" as CFString, nil, .deliverImmediately) - CFNotificationCenterAddObserver(notifyCenter, selfPtr, toggleCb, "com.soniox.voice.toggle" as CFString, nil, .deliverImmediately) } - + + // MARK: - Hands-Free Silence Auto-Stop & External Triggers private func checkHandsFreeSilence() { - guard audioRecorder.isRecording else { + guard audioRecorder.isRecording, !isBusyFinalizing else { handsFreeSilenceTimer?.invalidate() handsFreeSilenceTimer = nil return } - // If user is physically holding down a hotkey, do NOT auto-stop - if HotkeyManager.shared.isPhysicalKeyDown { + if HotkeyManager.shared.currentMode == .pushToTalk && HotkeyManager.shared.isPhysicalKeyDown { return } let now = Date() - let elapsedSinceStart = now.timeIntervalSince(recordingStartTime) - - // 1. Initial silence timeout: if no speech detected for 5.0s after start, auto-dismiss - if !hasSpokenInCurrentSession && !InputBoxManager.shared.hasStreamedText { - if elapsedSinceStart >= 5.0 { - print("AppDelegate: Hands-free initial silence timeout (5.0s). Auto-dismissing.") - DispatchQueue.main.async { [weak self] in - self?.stopRecording() - } - return + let totalElapsed = now.timeIntervalSince(recordingStartTime) + let silenceElapsed = now.timeIntervalSince(lastVoiceDetectedTime) + + if hasSpokenInCurrentSession { + if silenceElapsed >= 2.2 { + print("AppDelegate: Hands-free silence detected (2.2s after speech). Auto-stopping recording...") + stopRecording() + } + } else { + if totalElapsed >= 6.0 { + print("AppDelegate: Hands-free initial silence timeout (6.0s without speech). Cancelling...") + stopRecording() } } - - // 2. Trailing silence timeout: if speech was detected and now silent for 2.0s, auto-finalize! - if hasSpokenInCurrentSession || InputBoxManager.shared.hasStreamedText { - let silenceDuration = now.timeIntervalSince(lastVoiceDetectedTime) - if silenceDuration >= 2.0 { - print("AppDelegate: Hands-free trailing silence detected (2.0s). Auto-finalizing.") - DispatchQueue.main.async { [weak self] in - self?.stopRecording() - } - return + } + + public func applicationShouldHandleReopen(_ sender: NSApplication, hasVisibleWindows flag: Bool) -> Bool { + print("AppDelegate: App reopened (via Siri / Spotlight / Shortcuts). Starting hands-free recording!") + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + if !self.audioRecorder.isRecording { + self.startRecording() } } + return true + } + + @objc private func handleGetURLEvent(_ event: NSAppleEventDescriptor, withReplyEvent replyEvent: NSAppleEventDescriptor) { + guard let urlStr = event.paramDescriptor(forKeyword: AEKeyword(keyDirectObject))?.stringValue, + let url = URL(string: urlStr) else { return } + let host = (url.host ?? "").lowercased() + print("AppDelegate: Received URL trigger: \(urlStr)") - // 3. Absolute maximum duration guard (60s) - if elapsedSinceStart >= 60.0 { - print("AppDelegate: Hands-free max recording duration reached (60s). Auto-finalizing.") - DispatchQueue.main.async { [weak self] in - self?.stopRecording() + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + switch host { + case "stop": + if self.audioRecorder.isRecording { self.stopRecording() } + case "cancel": + self.cancelEverythingImmediately() + case "toggle": + self.toggleRecording() + default: + if !self.audioRecorder.isRecording { self.startRecording() } } - return } } - - @objc private func handleDistributedStartRecording() { - print("AppDelegate: Distributed notification 'com.soniox.voice.start' received.") + + @objc private func handleExternalStartNotification(_ notification: Notification) { + print("AppDelegate: Received DistributedNotification 'com.soniox.voice.start'") DispatchQueue.main.async { [weak self] in guard let self = self else { return } if !self.audioRecorder.isRecording { @@ -621,16 +712,19 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { } } } - - @objc private func handleDistributedStopRecording() { - print("AppDelegate: Distributed notification 'com.soniox.voice.stop' received.") + + @objc private func handleExternalStopNotification(_ notification: Notification) { + print("AppDelegate: Received DistributedNotification 'com.soniox.voice.stop'") DispatchQueue.main.async { [weak self] in - self?.stopRecording() + guard let self = self else { return } + if self.audioRecorder.isRecording { + self.stopRecording() + } } } - - @objc private func handleDistributedToggleRecording() { - print("AppDelegate: Distributed notification 'com.soniox.voice.toggle' received.") + + @objc private func handleExternalToggleNotification(_ notification: Notification) { + print("AppDelegate: Received DistributedNotification 'com.soniox.voice.toggle'") DispatchQueue.main.async { [weak self] in self?.toggleRecording() } diff --git a/src/AudioRecorder.swift b/src/AudioRecorder.swift index ad739f4..9af39ba 100644 --- a/src/AudioRecorder.swift +++ b/src/AudioRecorder.swift @@ -12,6 +12,11 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer private var pcmBuffer = Data() private let lock = NSLock() private let captureQueue = DispatchQueue(label: "com.soniox.audiocapture", qos: .userInteractive) + private let sessionQueue = DispatchQueue(label: "com.soniox.audiosession", qos: .userInitiated) + + // Pre-allocated reusable buffers to eliminate 50Hz per-frame heap allocations + private var reusableSrcBuffer: AVAudioPCMBuffer? + private var reusableOutBuffer: AVAudioPCMBuffer? public private(set) var isRecording = false public var onAudioLevelUpdate: ((Float) -> Void)? @@ -50,16 +55,22 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer public func resetSession() { lock.lock() - defer { lock.unlock() } isRecording = false - if let session = captureSession { - if session.isRunning { - session.stopRunning() - } - } + let session = captureSession captureSession = nil audioOutput = nil audioConverter = nil + reusableSrcBuffer = nil + reusableOutBuffer = nil + lock.unlock() + + if let session = session { + sessionQueue.async { + if session.isRunning { + session.stopRunning() + } + } + } print("AudioRecorder: Hardware capture session reset.") } @@ -124,27 +135,39 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer public func startRecording() throws { lock.lock() - defer { lock.unlock() } - - if isRecording { return } - pcmBuffer.removeAll() - - try ensureSessionConfigured() - if let session = captureSession, !session.isRunning { - session.startRunning() + if isRecording { + lock.unlock() + return } + pcmBuffer.removeAll(keepingCapacity: true) + try ensureSessionConfigured() + let session = captureSession isRecording = true + lock.unlock() + + if let session = session { + sessionQueue.async { + if !session.isRunning { + session.startRunning() + } + } + } print("AudioRecorder: Fast recording started (mic hardware ON).") } public func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { - guard isRecording else { return } + lock.lock() + let active = isRecording + lock.unlock() + guard active else { return } guard let formatDesc = CMSampleBufferGetFormatDescription(sampleBuffer) else { return } let srcFormat = AVAudioFormat(cmAudioFormatDescription: formatDesc) if audioConverter == nil || audioConverter?.inputFormat != srcFormat { audioConverter = AVAudioConverter(from: srcFormat, to: targetFormat) + reusableSrcBuffer = nil + reusableOutBuffer = nil } guard let converter = self.audioConverter else { return } @@ -152,8 +175,12 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer let numSamples = CMSampleBufferGetNumSamples(sampleBuffer) guard numSamples > 0 else { return } - guard let srcBuffer = AVAudioPCMBuffer(pcmFormat: srcFormat, frameCapacity: AVAudioFrameCount(numSamples)) else { return } - srcBuffer.frameLength = AVAudioFrameCount(numSamples) + let requiredSrcCap = AVAudioFrameCount(numSamples) + if reusableSrcBuffer == nil || reusableSrcBuffer!.frameCapacity < requiredSrcCap || reusableSrcBuffer!.format != srcFormat { + reusableSrcBuffer = AVAudioPCMBuffer(pcmFormat: srcFormat, frameCapacity: max(requiredSrcCap, 4096)) + } + guard let srcBuffer = reusableSrcBuffer else { return } + srcBuffer.frameLength = requiredSrcCap var lengthAtOffset = 0 var totalLength = 0 @@ -169,7 +196,10 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer } let outCapacity = AVAudioFrameCount(Double(numSamples) * (16000.0 / srcFormat.sampleRate)) + 128 - guard let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outCapacity) else { return } + if reusableOutBuffer == nil || reusableOutBuffer!.frameCapacity < outCapacity { + reusableOutBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: max(outCapacity, 2048)) + } + guard let outBuffer = reusableOutBuffer else { return } var error: NSError? var haveData = true @@ -191,6 +221,10 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer let data = Data(bytes: int16Ptr, count: bytesCount) lock.lock() + guard isRecording else { + lock.unlock() + return + } pcmBuffer.append(data) lock.unlock() @@ -215,6 +249,7 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer } } + /// Non-blocking drain of the serial capture queue + async hardware shutdown public func stopRecording() -> Data { lock.lock() if !isRecording { @@ -224,16 +259,46 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer } lock.unlock() - // Allow in-flight audio sample buffers on captureQueue to finish processing - usleep(60000) + // Synchronize with serial captureQueue (<1ms) so any in-flight sampleBuffer finishes cleanly without 60ms usleep! + captureQueue.sync { + self.lock.lock() + self.isRecording = false + self.lock.unlock() + } lock.lock() - isRecording = false - captureSession?.stopRunning() let data = pcmBuffer + let session = captureSession lock.unlock() + // Stop hardware capture asynchronously on background queue so main thread & WebSocket finalize have 0ms blocking + if let session = session { + sessionQueue.async { + if session.isRunning { + session.stopRunning() + } + } + } + print("AudioRecorder: Stopped. Microphone hardware OFF. Total PCM captured: \(data.count) bytes.") return data } + + /// Immediate abort (e.g. Escape key pressed) — discards buffered audio immediately + public func abortRecording() { + lock.lock() + isRecording = false + pcmBuffer.removeAll(keepingCapacity: true) + let session = captureSession + lock.unlock() + + if let session = session { + sessionQueue.async { + if session.isRunning { + session.stopRunning() + } + } + } + print("AudioRecorder: Aborted immediately.") + } } diff --git a/src/HUDOverlay.swift b/src/HUDOverlay.swift index e6648bd..08e006f 100644 --- a/src/HUDOverlay.swift +++ b/src/HUDOverlay.swift @@ -15,13 +15,15 @@ public final class HUDOverlayController { private var window: NSPanel? private var orbView: OutwardExpandingOrbView? - private var isVisible = false + public private(set) var isVisible = false + private var autoHideWorkItem: DispatchWorkItem? // Large transparent canvas so outward expansion and shadows never clip private let canvasSize: CGFloat = 200 private init() { - DispatchQueue.main.async { + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } self.setupWindow() NotificationCenter.default.addObserver(forName: NSApplication.didChangeScreenParametersNotification, object: nil, queue: .main) { [weak self] _ in self?.reposition() @@ -68,6 +70,9 @@ public final class HUDOverlayController { public func show(state: HUDState) { DispatchQueue.main.async { [weak self] in guard let self = self else { return } + self.autoHideWorkItem?.cancel() + self.autoHideWorkItem = nil + if self.window == nil { self.setupWindow() } @@ -76,7 +81,7 @@ public final class HUDOverlayController { switch state { case .hidden: - self.hide() + self.hideImmediately() case .recording(let level, let isLagging, let pingQuality): if !self.isVisible || window.alphaValue < 0.95 { @@ -104,9 +109,11 @@ public final class HUDOverlayController { case .success: orb.showSuccessState() - DispatchQueue.main.asyncAfter(deadline: .now() + 0.35) { - self.hide() + let work = DispatchWorkItem { [weak self] in + self?.hide() } + self.autoHideWorkItem = work + DispatchQueue.main.asyncAfter(deadline: .now() + 0.35, execute: work) case .error(let message): if !self.isVisible || window.alphaValue < 0.95 { @@ -115,16 +122,35 @@ public final class HUDOverlayController { self.isVisible = true } orb.showErrorState(message: message) - DispatchQueue.main.asyncAfter(deadline: .now() + 1.40) { - self.hide() + let work = DispatchWorkItem { [weak self] in + self?.hide() } + self.autoHideWorkItem = work + DispatchQueue.main.asyncAfter(deadline: .now() + 1.40, execute: work) + } + } + } + + /// Instant 0ms dismissal (used when Escape is pressed or short key tap is cancelled) + public func hideImmediately() { + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + self.autoHideWorkItem?.cancel() + self.autoHideWorkItem = nil + self.isVisible = false + if let window = self.window { + window.alphaValue = 0.0 + window.orderOut(nil) } + self.orbView?.reset() } } public func hide() { DispatchQueue.main.async { [weak self] in guard let self = self, let window = self.window, self.isVisible else { return } + self.autoHideWorkItem?.cancel() + self.autoHideWorkItem = nil self.isVisible = false NSAnimationContext.runAnimationGroup({ context in context.duration = 0.15 diff --git a/src/HotkeyManager.swift b/src/HotkeyManager.swift index c8056ce..6fbbb6a 100644 --- a/src/HotkeyManager.swift +++ b/src/HotkeyManager.swift @@ -63,12 +63,15 @@ public final class HotkeyManager { public var onHotkeyPressed: (() -> Void)? public var onHotkeyReleased: (() -> Void)? + public var onEscapePressed: (() -> Bool)? // Returns true if Soniox was active and consumed Escape + private let stateLock = NSLock() private var hotKeyRef: EventHotKeyRef? private var eventHandlerRef: EventHandlerRef? private var eventTapPort: CFMachPort? private var runLoopSource: CFRunLoopSource? private var globalMonitor: Any? + private var localMonitor: Any? private var isOptionPhysicallyDown = false private var isCapsLockPhysicallyDown = false @@ -76,6 +79,8 @@ public final class HotkeyManager { private var isFnPhysicallyDown = false public var isPhysicalKeyDown: Bool { + stateLock.lock() + defer { stateLock.unlock() } return isOptionPhysicallyDown || isCapsLockPhysicallyDown || isF5PhysicallyDown || isFnPhysicallyDown } private var lastFnPressTime: TimeInterval = 0 @@ -136,10 +141,12 @@ public final class HotkeyManager { } public func resetKeyState() { + stateLock.lock() isOptionPhysicallyDown = false isF5PhysicallyDown = false isCapsLockPhysicallyDown = false isFnPhysicallyDown = false + stateLock.unlock() } public func registerHotkeys() { @@ -187,17 +194,23 @@ public final class HotkeyManager { ) } - // 2. Global Event Tap for high-priority interception (Microphone Key, Option, CapsLock, Fn/Globe) + // 2. Global Event Tap for high-priority interception (Microphone Key, Option, CapsLock, Fn/Globe, Escape) setupEventTap() - // 3. Dual-layer fallback NSEvent monitor (catches flagsChanged/Option even after wake/sleep) + // 3. Fallback NSEvent monitor (active ONLY if CGEventTap is disabled/unavailable) setupGlobalMonitor() } private func triggerAction() { + stateLock.lock() let now = Date().timeIntervalSince1970 - guard (now - lastTriggerTime) > 0.10 else { return } + guard (now - lastTriggerTime) > 0.10 else { + stateLock.unlock() + return + } lastTriggerTime = now + stateLock.unlock() + DispatchQueue.main.async { [weak self] in self?.onHotkeyPressed?() } @@ -223,17 +236,29 @@ public final class HotkeyManager { let flags = event.flags.rawValue let keyCode = event.getIntegerValueField(.keyboardEventKeycode) + // 0. Escape Key (kVK_Escape = 53): Instant Cancel of active recording / transcription / recovery / HUD + if keyCode == 53 && type == .keyDown { + if let consumed = manager.onEscapePressed?(), consumed { + return nil // Swallow Escape so it only cancels Soniox and doesn't close user's dialog/page + } + } + // 1. Microphone Key (🎙️ / F5) if manager.currentPreset == .micKey { if keyCode == 96 { // 96 = kVK_F5 if type == .keyDown { - if !manager.isF5PhysicallyDown { - manager.isF5PhysicallyDown = true + manager.stateLock.lock() + let wasDown = manager.isF5PhysicallyDown + manager.isF5PhysicallyDown = true + manager.stateLock.unlock() + if !wasDown { manager.triggerAction() } return nil // Swallow F5 so it doesn't leak to apps } else if type == .keyUp { + manager.stateLock.lock() manager.isF5PhysicallyDown = false + manager.stateLock.unlock() if manager.currentMode == .pushToTalk { DispatchQueue.main.async { manager.onHotkeyReleased?() @@ -248,8 +273,14 @@ public final class HotkeyManager { else if manager.currentPreset == .option { if keyCode == 58 || keyCode == 61 { // 58 = Left Option, 61 = Right Option let isAlt = (flags & CGEventFlags.maskAlternate.rawValue) != 0 - if isAlt != manager.isOptionPhysicallyDown { + manager.stateLock.lock() + let changed = (isAlt != manager.isOptionPhysicallyDown) + if changed { manager.isOptionPhysicallyDown = isAlt + } + manager.stateLock.unlock() + + if changed { if isAlt { manager.triggerAction() } else if manager.currentMode == .pushToTalk { @@ -264,11 +295,14 @@ public final class HotkeyManager { // 3. CapsLock Key else if manager.currentPreset == .capsLock { if keyCode == 57 { // 57 = CapsLock - if !manager.isCapsLockPhysicallyDown { - manager.isCapsLockPhysicallyDown = true + manager.stateLock.lock() + let nowDown = !manager.isCapsLockPhysicallyDown + manager.isCapsLockPhysicallyDown = nowDown + manager.stateLock.unlock() + + if nowDown { manager.triggerAction() } else { - manager.isCapsLockPhysicallyDown = false if manager.currentMode == .pushToTalk { DispatchQueue.main.async { manager.onHotkeyReleased?() @@ -294,9 +328,14 @@ public final class HotkeyManager { else if manager.currentPreset == .doubleTapFn || manager.currentPreset == .holdFn { if keyCode == 63 { // 63 = kVK_Function (Fn / Globe) let isFnActive = (flags & CGEventFlags.maskSecondaryFn.rawValue) != 0 - if isFnActive != manager.isFnPhysicallyDown { + manager.stateLock.lock() + let changed = (isFnActive != manager.isFnPhysicallyDown) + if changed { manager.isFnPhysicallyDown = isFnActive - + } + manager.stateLock.unlock() + + if changed { if isFnActive { let now = Date().timeIntervalSince1970 let interval = now - manager.lastFnPressTime @@ -350,18 +389,36 @@ public final class HotkeyManager { self.globalMonitor = nil } - // Active backup monitor that never dies on sleep/wake + // Active backup monitor: ONLY acts if CGEventTap is nil or disabled by OS, eliminating dual-trigger race conditions! globalMonitor = NSEvent.addGlobalMonitorForEvents(matching: [.flagsChanged, .keyDown, .keyUp]) { [weak self] event in guard let self = self else { return } + if let tap = self.eventTapPort, CGEvent.tapIsEnabled(tap: tap) { + return // Primary CGEventTap is healthy and handling events; do not double-process + } + + // Wake up eventTap if it was disabled + if let tap = self.eventTapPort { + CGEvent.tapEnable(tap: tap, enable: true) + } + let flags = event.modifierFlags let keyCode = event.keyCode + if keyCode == 53 && event.type == .keyDown { + _ = self.onEscapePressed?() + return + } + if self.currentPreset == .option { - if keyCode == 58 || keyCode == 61 { // Left or Right Option + if keyCode == 58 || keyCode == 61 { let isAlt = flags.contains(.option) - if isAlt != self.isOptionPhysicallyDown { - self.isOptionPhysicallyDown = isAlt + self.stateLock.lock() + let changed = (isAlt != self.isOptionPhysicallyDown) + if changed { self.isOptionPhysicallyDown = isAlt } + self.stateLock.unlock() + + if changed { if isAlt { self.triggerAction() } else if self.currentMode == .pushToTalk { @@ -378,8 +435,12 @@ public final class HotkeyManager { } else if self.currentPreset == .capsLock { if keyCode == 57 { let isCaps = flags.contains(.capsLock) - if isCaps != self.isCapsLockPhysicallyDown { - self.isCapsLockPhysicallyDown = isCaps + self.stateLock.lock() + let changed = (isCaps != self.isCapsLockPhysicallyDown) + if changed { self.isCapsLockPhysicallyDown = isCaps } + self.stateLock.unlock() + + if changed { if isCaps { self.triggerAction() } else if self.currentMode == .pushToTalk { diff --git a/src/InputBoxManager.swift b/src/InputBoxManager.swift index abbb4af..18c0306 100644 --- a/src/InputBoxManager.swift +++ b/src/InputBoxManager.swift @@ -9,17 +9,31 @@ public final class InputBoxManager { private var pendingPrefixBuffer: String = "" private var isPrefixResolved: Bool = false private var needsNewlineBeforeNextDelta: Bool = false + private var sessionGeneration: UInt64 = 0 + private var clipboardRestoreWorkItem: DispatchWorkItem? private let pasteQueue = DispatchQueue(label: "com.soniox.pasteQueue", qos: .userInteractive) + private static let punctRegex: NSRegularExpression = { + return try! NSRegularExpression(pattern: #"[.,!?:;،؛؟"'()\[\]{}«»–—\-]"#, options: []) + }() + private init() {} + public static func stripPunctuation(_ str: String) -> String { + let range = NSRange(location: 0, length: str.utf16.count) + return punctRegex.stringByReplacingMatches(in: str, options: [], range: range, withTemplate: "") + } + /// Reset session state at the beginning of speech public func resetSession() { lock.lock() + sessionGeneration &+= 1 accumulatedStreamedText = "" pendingPrefixBuffer = "" isPrefixResolved = false needsNewlineBeforeNextDelta = false + clipboardRestoreWorkItem?.cancel() + clipboardRestoreWorkItem = nil lock.unlock() } @@ -32,8 +46,8 @@ public final class InputBoxManager { return !s.isEmpty || !p.isEmpty } - /// Prepares and guarantees that an input box is focused before recording starts. - /// STRICT RULE: If an input box is already focused, NEVER click or steal focus! + /// Prepares session state before recording starts. + /// STRICT RULE: Focus is 100% user-directed. Never steal focus to browser address bar or toolbar. public func ensureFocusedInputBox() { resetSession() @@ -43,9 +57,6 @@ public final class InputBoxManager { print("InputBoxManager: Accessibility not trusted.") return } - - // Focus is 100% user-directed. We preserve whatever the user focused (web text input, - // native text box, code editor, or address bar). Never forcibly steal focus to any toolbar or omnibox. } /// Streams word-by-word delta text directly into the focused input box in real time @@ -54,22 +65,28 @@ public final class InputBoxManager { // Suppress trailing end triggers ("and", "تمام", "end") from live typing let triggerWords = text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } - if let last = triggerWords.last?.replacingOccurrences(of: #"[.,!?:;،؛؟"'()\[\]{}«»–—\-]"#, with: "", options: .regularExpression).lowercased(), - last == "and" || last == "تمام" || last == "end" { - if triggerWords.count <= 1 { - return // Drop completely if it was just the end word + if let lastRaw = triggerWords.last { + let last = InputBoxManager.stripPunctuation(lastRaw).lowercased() + if last == "and" || last == "تمام" || last == "end" { + if triggerWords.count <= 1 { + return // Drop completely if it was just the end word + } + let textWithoutEnd = triggerWords.dropLast().joined(separator: " ") + " " + streamDelta(text: textWithoutEnd) + return } - let textWithoutEnd = triggerWords.dropLast().joined(separator: " ") + " " - streamDelta(text: textWithoutEnd) - return } + // Apply lightweight inline domain vocabulary so streamed words match final text + let normalizedDelta = SonioxSettings.shared.domainVocabulary ? TextProcessor.shared.applyDomainVocabularyInline(text) : text + lock.lock() + let expectedGen = sessionGeneration if isPrefixResolved { - var textToPaste = text + var textToPaste = normalizedDelta if needsNewlineBeforeNextDelta { needsNewlineBeforeNextDelta = false - let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + let trimmed = normalizedDelta.trimmingCharacters(in: .whitespacesAndNewlines) if !trimmed.isEmpty { textToPaste = "\n" + trimmed } @@ -79,6 +96,11 @@ public final class InputBoxManager { pasteQueue.async { [weak self] in guard let self = self else { return } + self.lock.lock() + let valid = (self.sessionGeneration == expectedGen) + self.lock.unlock() + guard valid else { return } + let pb = NSPasteboard.general pb.clearContents() pb.setString(textToPaste, forType: .string) @@ -89,7 +111,7 @@ public final class InputBoxManager { } // Not resolved yet: buffer to check if speech starts with a slash command - pendingPrefixBuffer += text + pendingPrefixBuffer += normalizedDelta let trimmed = pendingPrefixBuffer.trimmingCharacters(in: .whitespacesAndNewlines) let words = trimmed.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } @@ -111,6 +133,11 @@ public final class InputBoxManager { pasteQueue.async { [weak self] in guard let self = self else { return } + self.lock.lock() + let valid = (self.sessionGeneration == expectedGen) + self.lock.unlock() + guard valid else { return } + let pb = NSPasteboard.general pb.clearContents() pb.setString(toFlush, forType: .string) @@ -137,6 +164,11 @@ public final class InputBoxManager { pasteQueue.async { [weak self] in guard let self = self else { return } + self.lock.lock() + let valid = (self.sessionGeneration == expectedGen) + self.lock.unlock() + guard valid else { return } + let pb = NSPasteboard.general pb.clearContents() pb.setString(textToPaste, forType: .string) @@ -164,6 +196,11 @@ public final class InputBoxManager { pasteQueue.async { [weak self] in guard let self = self else { return } + self.lock.lock() + let valid = (self.sessionGeneration == expectedGen) + self.lock.unlock() + guard valid else { return } + let pb = NSPasteboard.general pb.clearContents() pb.setString(toFlush, forType: .string) @@ -189,6 +226,7 @@ public final class InputBoxManager { pb.setString(textWithTrailing, forType: .string) lock.lock() + let expectedGen = sessionGeneration // Flush any pending buffer if recording ended before prefix resolution if !isPrefixResolved && !pendingPrefixBuffer.isEmpty { let pending = pendingPrefixBuffer @@ -207,14 +245,29 @@ public final class InputBoxManager { pasteQueue.async { [weak self] in guard let self = self else { return } + self.lock.lock() + let valid = (self.sessionGeneration == expectedGen) + self.lock.unlock() + guard valid else { return } - // Helper to restore full text onto clipboard for the user after injection completes - let restoreClipboardFullText = { - DispatchQueue.main.asyncAfter(deadline: .now() + 0.12) { + // Safe 250ms clipboard restore with generation cancellation to prevent double-paste in heavy apps + let restoreClipboardFullText = { [weak self] in + guard let self = self else { return } + let work = DispatchWorkItem { [weak self] in + guard let self = self else { return } + self.lock.lock() + let stillValid = (self.sessionGeneration == expectedGen) + self.lock.unlock() + guard stillValid else { return } let pb = NSPasteboard.general pb.clearContents() pb.setString(textWithTrailing, forType: .string) } + self.lock.lock() + self.clipboardRestoreWorkItem?.cancel() + self.clipboardRestoreWorkItem = work + self.lock.unlock() + DispatchQueue.main.asyncAfter(deadline: .now() + 0.25, execute: work) } // Case 1: Nothing was streamed live -> Paste entire text @@ -251,7 +304,7 @@ public final class InputBoxManager { return } - // Case 4: Intelligent Word-level Tail-Anchor Matching (robust to domain vocabulary, number formatting, punctuation, ZWNJ) + // Case 4: Intelligent Word-level Tail-Anchor Matching let streamedWords = cleanStreamed.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } let finalWords = trimmed.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } @@ -261,10 +314,8 @@ public final class InputBoxManager { } let cleanWord: (String) -> String = { w in - w.replacingOccurrences(of: "\u{200C}", with: "") - .replacingOccurrences(of: "\u{200D}", with: "") - .replacingOccurrences(of: "[.,!?:;،؛؟\\\"'()\\[\\]{}«»–—\\-]", with: "", options: .regularExpression) - .lowercased() + let noZwnj = w.replacingOccurrences(of: "\u{200C}", with: "").replacingOccurrences(of: "\u{200D}", with: "") + return InputBoxManager.stripPunctuation(noZwnj).lowercased() } let normStreamed = streamedWords.map(cleanWord).filter { !$0.isEmpty } let normFinal = finalWords.map(cleanWord).filter { !$0.isEmpty } @@ -336,10 +387,9 @@ public final class InputBoxManager { unstreamedStartIndex = min(streamedWords.count, finalWords.count) } - // Suffix Evaluation: NEVER suppress paste! Always paste all remaining unstreamed words! + // Suffix Evaluation: Always paste all remaining unstreamed words if let startIndex = unstreamedStartIndex { if startIndex < finalWords.count { - // Only paste unstreamed trailing words let remainingWords = Array(finalWords.dropFirst(startIndex)) let suffix = remainingWords.joined(separator: " ") let suffixToPaste = addTrailingSpace ? (" " + suffix + " ") : (" " + suffix) @@ -350,7 +400,6 @@ public final class InputBoxManager { restoreClipboardFullText() return } else { - // startIndex >= finalWords.count: All words were ALREADY streamed! if addTrailingSpace { let pb = NSPasteboard.general pb.clearContents() @@ -377,6 +426,4 @@ public final class InputBoxManager { up.post(tap: .cgAnnotatedSessionEventTap) } } - - } diff --git a/src/PingMonitor.swift b/src/PingMonitor.swift index 1ce6942..7b24539 100644 --- a/src/PingMonitor.swift +++ b/src/PingMonitor.swift @@ -1,5 +1,6 @@ import Foundation import AppKit +import Network public enum PingQuality { case good(ms: Int) // < 450 ms -> Emerald Green @@ -20,6 +21,11 @@ public enum PingQuality { } } } + + public var isConnected: Bool { + if case .disconnected = self { return false } + return true + } } public final class PingMonitor { @@ -30,21 +36,62 @@ public final class PingMonitor { private var timer: Timer? private var isRunning = false private var isMeasuring = false + private var isActiveCadence = false + + private let pathMonitor = NWPathMonitor() + private let monitorQueue = DispatchQueue(label: "com.soniox.pathmonitor", qos: .utility) + public private(set) var isPathSatisfied: Bool = true + private var lastInterfaceSignature: String = "" public var onPingUpdate: ((PingQuality) -> Void)? + public var onNetworkRestored: (() -> Void)? + public var onInterfaceChanged: (() -> Void)? public private(set) var currentQuality: PingQuality = .medium(ms: 380) private init() { - let config = URLSessionConfiguration.default - config.timeoutIntervalForRequest = 2.5 - config.timeoutIntervalForResource = 2.5 + let config = URLSessionConfiguration.ephemeral + config.timeoutIntervalForRequest = 2.2 + config.timeoutIntervalForResource = 2.2 config.waitsForConnectivity = false config.requestCachePolicy = .reloadIgnoringLocalCacheData self.session = URLSession(configuration: config) + setupPathMonitor() setupSleepWakeObservers() } + private func setupPathMonitor() { + pathMonitor.pathUpdateHandler = { [weak self] path in + guard let self = self else { return } + let satisfied = (path.status == .satisfied) + let sig = path.availableInterfaces.map { "\($0.name):\($0.type)" }.joined(separator: ",") + + DispatchQueue.main.async { + let wasSatisfied = self.isPathSatisfied + let prevSig = self.lastInterfaceSignature + self.isPathSatisfied = satisfied + self.lastInterfaceSignature = sig + + if !satisfied { + print("PingMonitor: NWPathMonitor detected network offline.") + self.currentQuality = .disconnected(ms: nil) + self.onPingUpdate?(.disconnected(ms: nil)) + } else { + if !wasSatisfied { + print("PingMonitor: NWPathMonitor detected network RESTORED!") + self.measurePing() + self.onNetworkRestored?() + } else if !prevSig.isEmpty && prevSig != sig { + print("PingMonitor: Network interface changed (\(prevSig) -> \(sig)). Re-measuring...") + self.measurePing() + self.onInterfaceChanged?() + } + } + } + } + pathMonitor.start(queue: monitorQueue) + } + private func setupSleepWakeObservers() { NSWorkspace.shared.notificationCenter.addObserver(forName: NSWorkspace.willSleepNotification, object: nil, queue: .main) { [weak self] _ in print("PingMonitor: Mac entering sleep. Pausing ping measurements.") @@ -52,20 +99,43 @@ public final class PingMonitor { } NSWorkspace.shared.notificationCenter.addObserver(forName: NSWorkspace.didWakeNotification, object: nil, queue: .main) { [weak self] _ in - print("PingMonitor: Mac woke from sleep. Resuming ping in 2.5s after network settles...") - DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) { + print("PingMonitor: Mac woke from sleep. Resuming ping in 2.0s after network settles...") + DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { self?.start() } } } + /// Starts monitoring in low-power idle mode (0% CPU drain when not recording) public func start() { guard !isRunning else { return } isRunning = true measurePing() + scheduleTimer(interval: isActiveCadence ? 1.0 : 45.0) + } + + /// Switches between fast 1.0s cadence (while HUD is active) and 45.0s eco cadence (when idle) + public func setActiveRecordingMode(_ active: Bool) { DispatchQueue.main.async { [weak self] in - self?.timer?.invalidate() - self?.timer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { [weak self] _ in + guard let self = self else { return } + guard self.isActiveCadence != active else { return } + self.isActiveCadence = active + if self.isRunning { + if active { + self.measurePing() + self.scheduleTimer(interval: 1.0) + } else { + self.scheduleTimer(interval: 45.0) + } + } + } + } + + private func scheduleTimer(interval: TimeInterval) { + DispatchQueue.main.async { [weak self] in + guard let self = self, self.isRunning else { return } + self.timer?.invalidate() + self.timer = Timer.scheduledTimer(withTimeInterval: interval, repeats: true) { [weak self] _ in self?.measurePing() } } @@ -74,12 +144,22 @@ public final class PingMonitor { public func stop() { isRunning = false isMeasuring = false - timer?.invalidate() - timer = nil + isActiveCadence = false + DispatchQueue.main.async { [weak self] in + self?.timer?.invalidate() + self?.timer = nil + } } - private func measurePing() { + public func measurePing() { guard isRunning, !isMeasuring else { return } + if !isPathSatisfied { + DispatchQueue.main.async { [weak self] in + self?.currentQuality = .disconnected(ms: nil) + self?.onPingUpdate?(.disconnected(ms: nil)) + } + return + } isMeasuring = true var req = URLRequest(url: targetURL) @@ -88,9 +168,11 @@ public final class PingMonitor { let task = session.dataTask(with: req) { [weak self] _, resp, _ in defer { self?.isMeasuring = false } + guard let self = self else { return } let elapsed = Date().timeIntervalSince(t0) * 1000.0 let ms = Int(elapsed) + let wasDisconnected = !self.currentQuality.isConnected let quality: PingQuality if let http = resp as? HTTPURLResponse, (200...399).contains(http.statusCode) { if ms < 450 { @@ -107,8 +189,11 @@ public final class PingMonitor { } DispatchQueue.main.async { - self?.currentQuality = quality - self?.onPingUpdate?(quality) + self.currentQuality = quality + self.onPingUpdate?(quality) + if wasDisconnected && quality.isConnected { + self.onNetworkRestored?() + } } } task.resume() diff --git a/src/SonioxClient.swift b/src/SonioxClient.swift index e773422..b6daa84 100644 --- a/src/SonioxClient.swift +++ b/src/SonioxClient.swift @@ -7,6 +7,15 @@ public final class SonioxLiveSession { private static let userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" private static let origin = "https://translate.compare.soniox.com" + // Pre-compiled static regexes for zero-allocation hot path + private static let faRegex: NSRegularExpression = { + return try! NSRegularExpression(pattern: "[\\u0600-\\u06FF\\u0750-\\u077F\\uFB50-\\uFDFF\\uFE70-\\uFEFF]", options: []) + }() + private static let enRegex: NSRegularExpression = { + return try! NSRegularExpression(pattern: "[a-zA-Z]", options: []) + }() + private static let punctCharSet = CharacterSet(charactersIn: ".,!?:;،؛؟\"'()[]{}«»-–—") + public let targetLanguage: String? public var isTranslationMode: Bool { guard let target = targetLanguage, !target.isEmpty, target != "off" else { return false } @@ -18,12 +27,16 @@ public final class SonioxLiveSession { private var isFinalizing = false private var isClosed = false private var isConnected = false + private var isAborted = false private let lock = NSLock() - // Transcripts tracking + // Transcripts & Stall tracking private var committedFinalTokens: [String] = [] private var currentNonFinalTokens: [String] = [] private var streamedPrefix: String = "" + private var sentAudioBytes: Int = 0 + public private(set) var totalReceivedTokensCount: Int = 0 + public private(set) var hasSilentStall: Bool = false // Callbacks public var onPartialText: ((String) -> Void)? @@ -41,13 +54,22 @@ public final class SonioxLiveSession { public func setRecordingActive(_ active: Bool) { lock.lock() isRecordingActive = active + if active { + lastDataTime = Date() + } lock.unlock() } public var isReady: Bool { lock.lock() defer { lock.unlock() } - return isConnected && !isClosed && !isFinalizing + return isConnected && !isClosed && !isFinalizing && !isAborted + } + + public var isAlive: Bool { + lock.lock() + defer { lock.unlock() } + return !isClosed && !isAborted && !hasNetworkFailure } public func startKeepAliveHeartbeat() { @@ -57,12 +79,13 @@ public final class SonioxLiveSession { self.heartbeatTimer = Timer.scheduledTimer(withTimeInterval: 6.0, repeats: true) { [weak self] _ in guard let self = self else { return } self.lock.lock() - guard !self.isClosed, !self.isFinalizing, let task = self.webSocketTask else { + guard !self.isClosed, !self.isFinalizing, !self.isAborted, let task = self.webSocketTask else { self.lock.unlock() return } self.lock.unlock() + let t0 = Date() task.sendPing { [weak self] error in guard let self = self else { return } if let error = error { @@ -73,6 +96,14 @@ public final class SonioxLiveSession { self.isClosed = true self.lock.unlock() SonioxSessionPool.shared.prewarmNextSession() + } else { + let rtt = Date().timeIntervalSince(t0) + if rtt > 0.95 { + self.lock.lock() + self.hasSilentStall = true + self.lock.unlock() + self.onNetworkDegraded?(true) + } } } } @@ -112,10 +143,10 @@ public final class SonioxLiveSession { var request = URLRequest(url: url) request.setValue(SonioxLiveSession.userAgent, forHTTPHeaderField: "User-Agent") request.setValue(SonioxLiveSession.origin, forHTTPHeaderField: "Origin") - request.timeoutInterval = 25.0 + request.timeoutInterval = 12.0 let config = URLSessionConfiguration.default - config.waitsForConnectivity = true + config.waitsForConnectivity = false // Fail fast on dead interface so recovery/failover triggers immediately config.requestCachePolicy = .reloadIgnoringLocalCacheData let session = URLSession(configuration: config) @@ -128,8 +159,9 @@ public final class SonioxLiveSession { task.sendPing { [weak self] error in guard let self = self else { return } self.lock.lock() - if error == nil && !self.isClosed { + if error == nil && !self.isClosed && !self.isAborted { self.isConnected = true + self.lastDataTime = Date() self.lock.unlock() self.onConnectionStateChanged?(true) print("SonioxLiveSession: Connected successfully.") @@ -146,27 +178,59 @@ public final class SonioxLiveSession { startReceiving() } + /// Called while user is actively speaking to detect half-open TCP silent stalls + public func checkSilentStallWhileSpeaking() { + lock.lock() + let connected = isConnected && !isClosed && !isAborted + let sent = sentAudioBytes + let elapsedSinceData = Date().timeIntervalSince(lastDataTime) + lock.unlock() + + // If >1.5s of audio (48KB) was sent while speaking, but 0 frames arrived for >3.0s: + if connected && sent > 48000 && elapsedSinceData > 3.0 { + lock.lock() + hasSilentStall = true + lock.unlock() + DispatchQueue.main.async { [weak self] in + self?.onNetworkDegraded?(true) + } + } + } + public func sendAudioChunk(_ data: Data) { lock.lock() - defer { lock.unlock() } - guard !isFinalizing, !isClosed, let task = webSocketTask else { return } + guard !isFinalizing, !isClosed, !isAborted, let task = webSocketTask else { + lock.unlock() + return + } + lock.unlock() + let t0 = Date() let message = URLSessionWebSocketTask.Message.data(data) task.send(message) { [weak self] error in + guard let self = self else { return } if let error = error { print("SonioxLiveSession: Error streaming audio chunk:", error) - self?.lock.lock() - self?.hasNetworkFailure = true - self?.isConnected = false - self?.lock.unlock() - self?.onNetworkDegraded?(true) + self.lock.lock() + self.hasNetworkFailure = true + self.isConnected = false + self.lock.unlock() + self.onNetworkDegraded?(true) + } else { + self.lock.lock() + self.sentAudioBytes += data.count + let sendDur = Date().timeIntervalSince(t0) + if sendDur > 0.70 { + self.hasSilentStall = true + } + self.lock.unlock() } } } public func finalizeStream() { lock.lock() - guard !isFinalizing, !isClosed, let task = webSocketTask else { + guard !isFinalizing, !isClosed, !isAborted, let task = webSocketTask else { lock.unlock() return } @@ -175,7 +239,6 @@ public final class SonioxLiveSession { print("SonioxLiveSession: Sending finalize packet...") // 1. Send trailing 300ms (9600 bytes at 16kHz 16-bit PCM) silence frame - // so acoustic CTC model cleanly flushes and resolves trailing fast speech syllables let silenceChunk = Data(count: 9600) let silenceMsg = URLSessionWebSocketTask.Message.data(silenceChunk) task.send(silenceMsg) { _ in } @@ -192,14 +255,14 @@ public final class SonioxLiveSession { } } - // Safety timeout fallback: allow up to 4.5s for STT inference & network transit - DispatchQueue.global().asyncAfter(deadline: .now() + 4.5) { [weak self] in + // Safety timeout fallback: allow up to 4.0s for STT inference & network transit + DispatchQueue.global().asyncAfter(deadline: .now() + 4.0) { [weak self] in guard let self = self else { return } self.lock.lock() - let stillOpen = !self.isClosed + let stillOpen = !self.isClosed && !self.isAborted self.lock.unlock() if stillOpen { - print("SonioxLiveSession: Finalize timeout (4.5s) reached — completing.") + print("SonioxLiveSession: Finalize timeout (4.0s) reached — completing.") self.completeWithCurrentText() } } @@ -211,7 +274,7 @@ public final class SonioxLiveSession { guard let self = self else { return } self.lock.lock() - if self.isClosed { + if self.isClosed || self.isAborted { self.lock.unlock() return } @@ -219,7 +282,12 @@ public final class SonioxLiveSession { switch result { case .success(let message): + self.lock.lock() self.lastDataTime = Date() + self.hasSilentStall = false + self.lock.unlock() + self.onNetworkDegraded?(false) + var textReceived: String? switch message { case .string(let str): @@ -244,8 +312,6 @@ public final class SonioxLiveSession { self.lock.unlock() self.onNetworkDegraded?(true) - // If recording is still active (user speaking), keep buffering in AudioRecorder; - // do NOT complete or deliver empty text prematurely! if !activeRecording { self.completeWithCurrentText() } else { @@ -261,19 +327,16 @@ public final class SonioxLiveSession { var gotFin = false var newFinals: [String] = [] var newNonFinals: [String] = [] + var tokenCountDelta = 0 if let type = json["type"] as? String, type == "data", let parts = json["parts"] as? [[String: Any]] { for part in parts { let isTranslation = (part["translation_status"] as? String) == "translation" if self.isTranslationMode { - if !isTranslation { - continue - } + if !isTranslation { continue } } else { - if isTranslation { - continue - } + if isTranslation { continue } } let pText = part["text"] as? String ?? "" @@ -284,19 +347,27 @@ public final class SonioxLiveSession { let clean = pText.replacingOccurrences(of: "", with: "") if !clean.isEmpty { newFinals.append(clean) + tokenCountDelta += 1 } } else if isFinal { if !pText.isEmpty { newFinals.append(pText) + tokenCountDelta += 1 } } else { if !pText.isEmpty { newNonFinals.append(pText) + tokenCountDelta += 1 } } } lock.lock() + if isAborted { + lock.unlock() + return + } + totalReceivedTokensCount += tokenCountDelta if !newFinals.isEmpty { committedFinalTokens.append(contentsOf: newFinals) } @@ -311,7 +382,7 @@ public final class SonioxLiveSession { let trimmedCombined = combined.trimmingCharacters(in: .whitespacesAndNewlines) let currentWords = trimmedCombined.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } if let lastRaw = currentWords.last { - let cleanLast = lastRaw.replacingOccurrences(of: #"[.,!?:;،؛؟"'()\[\]{}«»–—\-]"#, with: "", options: .regularExpression).lowercased() + let cleanLast = InputBoxManager.stripPunctuation(lastRaw).lowercased() if cleanLast == "and" || cleanLast == "تمام" || cleanLast == "end" { triggerDetected = true } @@ -360,9 +431,18 @@ public final class SonioxLiveSession { } } + public func currentBestTranscript() -> String { + lock.lock() + let fullCommitted = committedFinalTokens.joined() + let fullNonFinal = currentNonFinalTokens.joined() + let rawCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal) + lock.unlock() + return SonioxLiveSession.sanitizeText(rawCombined) + } + public func completeWithCurrentText() { lock.lock() - if isClosed { + if isClosed || isAborted { lock.unlock() return } @@ -376,9 +456,12 @@ public final class SonioxLiveSession { let cb = onFinalResult webSocketTask?.cancel(with: .normalClosure, reason: nil) webSocketTask = nil + urlSession?.finishTasksAndInvalidate() urlSession = nil lock.unlock() + stopKeepAliveHeartbeat() + DispatchQueue.main.async { if cleaned.isEmpty && failureOccurred { cb?(.failure(NSError(domain: "SonioxLiveSession", code: -101, userInfo: [NSLocalizedDescriptionKey: "WebSocket network failure with empty transcript"]))) @@ -388,13 +471,33 @@ public final class SonioxLiveSession { } } - public func cancel() { + /// Immediate hard cancellation (Escape key pressed) — suppresses all further callbacks + public func abort() { lock.lock() + isAborted = true isClosed = true - webSocketTask?.cancel(with: .normalClosure, reason: nil) + isConnected = false + isRecordingActive = false + isFinalizing = false + onPartialText = nil + onCommittedDelta = nil + onEndTriggerDetected = nil + onFinalResult = nil + onConnectionStateChanged = nil + onNetworkDegraded = nil + let task = webSocketTask + let session = urlSession webSocketTask = nil urlSession = nil lock.unlock() + + stopKeepAliveHeartbeat() + task?.cancel() + session?.invalidateAndCancel() + } + + public func cancel() { + abort() } // MARK: - Smart Audio Recovery / Fast Burst Retry @@ -402,13 +505,15 @@ public final class SonioxLiveSession { /// Re-transcribes the full audio buffer when the live stream had packet drops, disconnects, or missing trailing words. public static func retryFullBufferedAudio( pcmData: Data, - languageHints: [String], + languageHints: [String] = SonioxSettings.shared.language.hints, targetLanguage: String? = nil, + isCancelled: @escaping () -> Bool = { false }, onProgress: ((String) -> Void)? = nil, + onStatusUpdate: ((String) -> Void)? = nil, completion: @escaping (Result) -> Void ) { - guard !pcmData.isEmpty else { - completion(.failure(NSError(domain: "SonioxLiveSession", code: -102, userInfo: [NSLocalizedDescriptionKey: "Empty audio buffer for recovery"]))) + guard !pcmData.isEmpty, !isCancelled() else { + completion(.failure(NSError(domain: "SonioxLiveSession", code: -102, userInfo: [NSLocalizedDescriptionKey: "Cancelled or empty audio buffer"]))) return } @@ -423,25 +528,62 @@ public final class SonioxLiveSession { // Attempt 4: Fallback WS func executeAttempt(_ step: Int) { + if isCancelled() { + completion(.failure(NSError(domain: "SonioxLiveSession", code: -999, userInfo: [NSLocalizedDescriptionKey: "Cancelled by Escape"]))) + return + } let isPrimary = (step == 1 || step == 3) let endpoint = isPrimary ? primaryWsBase : fallbackWsBase let name = isPrimary ? "Primary" : "Fallback" print("SonioxLiveSession: Smart Recovery Attempt \(step)/4 on \(name) endpoint...") - executeBurstSession(endpoint: endpoint, pcmData: pcmData, languageHints: languageHints, targetLanguage: effectiveTarget, onProgress: onProgress) { result in + if step > 1 { + DispatchQueue.main.async { + onStatusUpdate?("RETRY \(step)/4") + } + } + + // If network interface is completely offline on attempt >= 2, wait briefly for Wi-Fi/VPN handoff + if !PingMonitor.shared.isPathSatisfied && step >= 2 { + DispatchQueue.global(qos: .userInitiated).asyncAfter(deadline: .now() + 1.0) { + if isCancelled() { + completion(.failure(NSError(domain: "SonioxLiveSession", code: -999, userInfo: [NSLocalizedDescriptionKey: "Cancelled"]))) + return + } + if step < 4 { + executeAttempt(step + 1) + } else { + completion(.failure(NSError(domain: "SonioxLiveSession", code: -1009, userInfo: [NSLocalizedDescriptionKey: "Network offline"]))) + } + } + return + } + + executeBurstSession( + endpoint: endpoint, + pcmData: pcmData, + languageHints: languageHints, + targetLanguage: effectiveTarget, + isCancelled: isCancelled, + onProgress: onProgress + ) { result in + if isCancelled() { + completion(.failure(NSError(domain: "SonioxLiveSession", code: -999, userInfo: [NSLocalizedDescriptionKey: "Cancelled"]))) + return + } switch result { case .success(let text): - print("SonioxLiveSession: Recovery Attempt \(step) SUCCEEDED (\(text.count) chars).") + print("SonioxLiveSession: ✅ Recovery Attempt \(step) SUCCEEDED (\(text.count) chars).") completion(.success(text)) case .failure(let error): print("SonioxLiveSession: Recovery Attempt \(step) failed: \(error)") if step < 4 { - let delay = (step == 2) ? 0.8 : 0.2 // 0.8s pause before attempt 3 for network interface stabilization - DispatchQueue.global().asyncAfter(deadline: .now() + delay) { + let delay = (step == 2) ? 0.8 : 0.25 + DispatchQueue.global(qos: .userInitiated).asyncAfter(deadline: .now() + delay) { executeAttempt(step + 1) } } else { - print("SonioxLiveSession: All 4 recovery attempts exhausted: \(error)") + print("SonioxLiveSession: ❌ All 4 recovery attempts exhausted: \(error)") completion(.failure(error)) } } @@ -456,6 +598,7 @@ public final class SonioxLiveSession { pcmData: Data, languageHints: [String], targetLanguage: String?, + isCancelled: @escaping () -> Bool, onProgress: ((String) -> Void)?, completion: @escaping (Result) -> Void ) { @@ -483,12 +626,14 @@ public final class SonioxLiveSession { request.setValue(origin, forHTTPHeaderField: "Origin") let durationSec = Double(pcmData.count) / 32000.0 - // Dynamic watchdog: allow generous time for network reconnect + fast upload + STT inference - let watchdogTimeout = max(8.0, min(30.0, durationSec * 0.75 + 6.0)) - request.timeoutInterval = watchdogTimeout + 5.0 + let watchdogTimeout = max(6.5, min(25.0, durationSec * 0.60 + 5.0)) + request.timeoutInterval = 5.0 // Fast handshake timeout so dead endpoint fails over in <5s + + let config = URLSessionConfiguration.ephemeral + config.timeoutIntervalForRequest = watchdogTimeout + config.timeoutIntervalForResource = watchdogTimeout + 3.0 + config.waitsForConnectivity = false // Fail fast on dead interface so next endpoint attempt runs immediately! - let config = URLSessionConfiguration.default - config.waitsForConnectivity = true let session = URLSession(configuration: config) let task = session.webSocketTask(with: request) task.resume() @@ -511,11 +656,18 @@ public final class SonioxLiveSession { } } - // Receive handler func receiveNext() { + if isCancelled() { + finishOnce(with: .failure(NSError(domain: "Soniox", code: -999, userInfo: [NSLocalizedDescriptionKey: "Cancelled"]))) + return + } task.receive { result in lock.lock() - if isDone { lock.unlock(); return } + if isDone || isCancelled() { + lock.unlock() + finishOnce(with: .failure(NSError(domain: "Soniox", code: -999, userInfo: [NSLocalizedDescriptionKey: "Cancelled"]))) + return + } lock.unlock() switch result { @@ -532,6 +684,7 @@ public final class SonioxLiveSession { var gotFin = false var packetNonFinal: [String] = [] + var packetCommitted: [String] = [] if let parts = json["parts"] as? [[String: Any]] { for p in parts { let isTranslation = (p["translation_status"] as? String) == "translation" @@ -545,9 +698,9 @@ public final class SonioxLiveSession { if t.contains("") { gotFin = true let clean = t.replacingOccurrences(of: "", with: "") - if !clean.isEmpty { fullCommitted.append(clean) } + if !clean.isEmpty { packetCommitted.append(clean) } } else if isFinal { - if !t.isEmpty { fullCommitted.append(t) } + if !t.isEmpty { packetCommitted.append(t) } } else { if !t.isEmpty { packetNonFinal.append(t) } } @@ -555,6 +708,9 @@ public final class SonioxLiveSession { } lock.lock() + if !packetCommitted.isEmpty { + fullCommitted.append(contentsOf: packetCommitted) + } latestNonFinal = packetNonFinal let currentProgress = fullCommitted.joined() + (gotFin ? "" : latestNonFinal.joined()) lock.unlock() @@ -598,12 +754,15 @@ public final class SonioxLiveSession { receiveNext() - // Handshake verification before pumping audio to avoid early drops & empty transcripts + // Handshake verification + Chained Backpressure Upload (32KB / 1.0s chunks) task.sendPing { pingError in lock.lock() let alreadyDone = isDone lock.unlock() - if alreadyDone { return } + if alreadyDone || isCancelled() { + finishOnce(with: .failure(NSError(domain: "Soniox", code: -999, userInfo: nil))) + return + } if let pingErr = pingError { print("SonioxLiveSession: Burst connection ping failed on \(endpoint): \(pingErr)") @@ -611,34 +770,33 @@ public final class SonioxLiveSession { return } - // Connection is active: stream audio in high-speed paced bursts - DispatchQueue.global(qos: .userInitiated).async { - let chunkSize = 16000 // 0.5s audio - var offset = 0 - while offset < pcmData.count { - lock.lock() - let done = isDone - lock.unlock() - if done { break } - - let length = min(chunkSize, pcmData.count - offset) - let chunk = pcmData.subdata(in: offset..<(offset + length)) - task.send(URLSessionWebSocketTask.Message.data(chunk)) { _ in } - offset += length - usleep(12000) // 12ms pace - } - + let chunkSize = 32000 // 1.0s of 16kHz 16-bit PCM + func sendNextChunk(offset: Int) { lock.lock() let done = isDone lock.unlock() - if !done { - // Send trailing silence + finalize - let silence = Data(count: 4800) - task.send(URLSessionWebSocketTask.Message.data(silence)) { _ in } - let finalizeMsg = URLSessionWebSocketTask.Message.string("{\"type\": \"finalize\"}") - task.send(finalizeMsg) { _ in } + if done || isCancelled() { return } + + if offset >= pcmData.count { + let silence = Data(count: 6400) + task.send(.data(silence)) { _ in + task.send(.string("{\"type\": \"finalize\"}")) { _ in } + } + return + } + + let end = min(offset + chunkSize, pcmData.count) + let slice = pcmData.subdata(in: offset.. String { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) @@ -666,18 +824,16 @@ public final class SonioxLiveSession { let words = trimmed.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } if words.isEmpty { return "" } - let faPattern = "[\\u0600-\\u06FF\\u0750-\\u077F\\uFB50-\\uFDFF\\uFE70-\\uFEFF]" - let enPattern = "[a-zA-Z]" - - func matches(_ pattern: String, in str: String) -> Bool { - return str.range(of: pattern, options: .regularExpression) != nil + func matches(_ regex: NSRegularExpression, in str: String) -> Bool { + let range = NSRange(location: 0, length: str.utf16.count) + return regex.firstMatch(in: str, options: [], range: range) != nil } var faCount = 0 var enCount = 0 for w in words { - if matches(faPattern, in: w) { faCount += 1 } - if matches(enPattern, in: w) { enCount += 1 } + if matches(faRegex, in: w) { faCount += 1 } + if matches(enRegex, in: w) { enCount += 1 } } let total = faCount + enCount @@ -696,8 +852,8 @@ public final class SonioxLiveSession { if faRatio >= 0.25 { for w in words { - if matches(enPattern, in: w) && !matches(faPattern, in: w) { - let cleanW = w.lowercased().trimmingCharacters(in: CharacterSet(charactersIn: ".,!?:;،؛؟\"'()[]{}«»-–—")) + if matches(enRegex, in: w) && !matches(faRegex, in: w) { + let cleanW = w.lowercased().trimmingCharacters(in: punctCharSet) if stopWords.contains(cleanW) { continue } if faRatio >= 0.70 && !preserveWords.contains(cleanW) { continue } } @@ -744,13 +900,14 @@ public final class SonioxSessionPool { public func discardPrewarmedSession() { lock.lock() prewarmedSession?.stopKeepAliveHeartbeat() - prewarmedSession?.cancel() + prewarmedSession?.abort() prewarmedSession = nil isPrewarming = false lock.unlock() } public func prewarmNextSession() { + guard PingMonitor.shared.isPathSatisfied else { return } lock.lock() defer { lock.unlock() } @@ -765,7 +922,7 @@ public final class SonioxSessionPool { isPrewarming = true prewarmedSession?.stopKeepAliveHeartbeat() - prewarmedSession?.cancel() + prewarmedSession?.abort() prewarmedSession = nil DispatchQueue.global(qos: .userInitiated).async { [weak self] in @@ -789,8 +946,7 @@ public final class SonioxSessionPool { } } - // Watchdog in case connection hangs indefinitely - DispatchQueue.global().asyncAfter(deadline: .now() + 15.0) { [weak self] in + DispatchQueue.global().asyncAfter(deadline: .now() + 12.0) { [weak self] in guard let self = self else { return } self.lock.lock() if self.isPrewarming && self.prewarmedSession == nil { @@ -813,20 +969,19 @@ public final class SonioxSessionPool { prewarmedSession = nil lock.unlock() - // Immediately start pre-warming the NEXT session in the background DispatchQueue.global(qos: .userInitiated).async { [weak self] in self?.prewarmNextSession() } let age = Date().timeIntervalSince(createdAt) - if let session = session, session.isReady && age < 60.0 { + if let session = session, session.isReady && age < 55.0 { session.stopKeepAliveHeartbeat() print("SonioxSessionPool: Acquired HOT prewarmed session (age: \(String(format: "%.1f", age))s, 0ms connect delay!)") return session } session?.stopKeepAliveHeartbeat() - session?.cancel() + session?.abort() print("SonioxSessionPool: Prewarmed session not ready/stale (age: \(String(format: "%.1f", age))s), creating immediate on-demand session.") let hints = SonioxSettings.shared.language.hints @@ -834,3 +989,123 @@ public final class SonioxSessionPool { return SonioxLiveSession(languageHints: hints, targetLanguage: target) } } + +// MARK: - Persistent Offline Audio Queue (Zero Data Loss on Multi-Minute Internet Outages) +public final class OfflineAudioQueueManager { + public static let shared = OfflineAudioQueueManager() + + private let queueDir: URL + private let workerQueue = DispatchQueue(label: "com.soniox.offlinequeue", qos: .utility) + private var isProcessing = false + private let lock = NSLock() + + public var onOfflineAudioRecovered: ((String) -> Void)? + + private init() { + let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! + self.queueDir = appSupport.appendingPathComponent("SonioxVoice/PendingAudioQueue") + try? FileManager.default.createDirectory(at: queueDir, withIntermediateDirectories: true) + } + + public func enqueueFailedAudio(pcmData: Data) { + guard pcmData.count >= 16000 else { return } // Only save meaningful speech (>= 0.5s) + workerQueue.async { [weak self] in + guard let self = self else { return } + try? FileManager.default.createDirectory(at: self.queueDir, withIntermediateDirectories: true) + let timestamp = Int(Date().timeIntervalSince1970 * 1000) + let fileURL = self.queueDir.appendingPathComponent("speech_\(timestamp).pcm") + do { + try pcmData.write(to: fileURL, options: .atomic) + print("OfflineAudioQueueManager: 💾 Saved \(pcmData.count) bytes of speech to disk queue (\(fileURL.lastPathComponent)) for automatic recovery when internet returns.") + } catch { + print("OfflineAudioQueueManager: Failed to write offline audio:", error) + } + } + } + + public func clearAllPending() { + workerQueue.async { [weak self] in + guard let self = self else { return } + if let files = try? FileManager.default.contentsOfDirectory(at: self.queueDir, includingPropertiesForKeys: nil) { + for f in files where f.pathExtension == "pcm" { + try? FileManager.default.removeItem(at: f) + } + } + } + } + + public func processPendingQueueIfNeeded(isCancelled: @escaping () -> Bool = { false }) { + guard PingMonitor.shared.isPathSatisfied else { return } + lock.lock() + if isProcessing { + lock.unlock() + return + } + isProcessing = true + lock.unlock() + + workerQueue.async { [weak self] in + guard let self = self else { return } + guard let files = try? FileManager.default.contentsOfDirectory(at: self.queueDir, includingPropertiesForKeys: [.creationDateKey]), + !files.isEmpty else { + self.lock.lock() + self.isProcessing = false + self.lock.unlock() + return + } + + let pcmFiles = files.filter { $0.pathExtension == "pcm" }.sorted { $0.lastPathComponent < $1.lastPathComponent } + guard let firstFile = pcmFiles.first else { + self.lock.lock() + self.isProcessing = false + self.lock.unlock() + return + } + + // Drop stale files older than 2 hours + if let attrs = try? FileManager.default.attributesOfItem(atPath: firstFile.path), + let created = attrs[.creationDate] as? Date, + Date().timeIntervalSince(created) > 7200 { + try? FileManager.default.removeItem(at: firstFile) + self.lock.lock() + self.isProcessing = false + self.lock.unlock() + self.processPendingQueueIfNeeded(isCancelled: isCancelled) + return + } + + guard let pcmData = try? Data(contentsOf: firstFile), !pcmData.isEmpty else { + try? FileManager.default.removeItem(at: firstFile) + self.lock.lock() + self.isProcessing = false + self.lock.unlock() + return + } + + print("OfflineAudioQueueManager: 🌐 Network is back! Recovering queued speech file \(firstFile.lastPathComponent) (\(pcmData.count) bytes)...") + SonioxLiveSession.retryFullBufferedAudio(pcmData: pcmData, isCancelled: isCancelled) { [weak self] result in + guard let self = self else { return } + switch result { + case .success(let recoveredText): + let trimmed = recoveredText.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { + try? FileManager.default.removeItem(at: firstFile) + DispatchQueue.main.async { + self.onOfflineAudioRecovered?(trimmed) + } + self.lock.lock() + self.isProcessing = false + self.lock.unlock() + self.processPendingQueueIfNeeded(isCancelled: isCancelled) + return + } + case .failure: + break + } + self.lock.lock() + self.isProcessing = false + self.lock.unlock() + } + } + } +} diff --git a/src/TextProcessor.swift b/src/TextProcessor.swift index b56ae53..2d1d5cd 100644 --- a/src/TextProcessor.swift +++ b/src/TextProcessor.swift @@ -3,6 +3,14 @@ import Foundation public final class TextProcessor { public static let shared = TextProcessor() + private static let trailingEndRegex: NSRegularExpression = { + return try! NSRegularExpression(pattern: #"[،,:\s]+(and|تمام|end)[.!?]?$"#, options: .caseInsensitive) + }() + + private static let singleEndRegex: NSRegularExpression = { + return try! NSRegularExpression(pattern: #"^(and|تمام|end)[.!?]?$"#, options: .caseInsensitive) + }() + private init() {} public func process(text: String) -> String { @@ -63,6 +71,10 @@ public final class TextProcessor { return result.trimmingCharacters(in: .whitespacesAndNewlines) } + public func applyDomainVocabularyInline(_ text: String) -> String { + return applyDomainVocabulary(text) + } + private func applyVoicePunctuation(_ text: String) -> String { var t = " " + text + " " @@ -120,51 +132,51 @@ public final class TextProcessor { return t } + private static let domainTerms: [(String, String)] = [ + ("این اپ پرچس", "In-App Purchase"), + ("این اپ پرشرز", "In-App Purchase"), + ("اپ پرشرز", "In-App Purchases"), + ("اپل کانکت", "App Store Connect"), + ("اپ استور کانکت", "App Store Connect"), + ("حجزی مرچنت", "Hajzee Merchant"), + ("حجزی یوزر", "Hajzee User"), + ("حجزی", "Hajzee"), + ("نیوهورایزن", "NewHorizon"), + ("نیو هورایزن", "NewHorizon"), + ("نیوهوریزن", "NewHorizon"), + ("زر محک", "ZarMahak"), + ("زرمحک", "ZarMahak"), + ("سانی اوکس", "Soniox"), + ("سانیوکس", "Soniox"), + ("سانی اکس", "Soniox"), + ("تیم بای", "TeamBy"), + ("تیمبی", "TeamBy"), + ("دوودی", "Dovodi"), + ("داودی", "Dovodi"), + ("داکر", "Docker"), + ("کوبرنتیز", "Kubernetes"), + ("کوبرنتس", "Kubernetes"), + ("ای اس ایکس آی", "ESXi"), + ("ای اس ایکس ای", "ESXi"), + ("فست ای پی آی", "FastAPI"), + ("فست ای پی ای", "FastAPI"), + ("پستگرس کیو ال", "PostgreSQL"), + ("پستگرس", "PostgreSQL"), + ("ردیس", "Redis"), + ("فلاتر", "Flutter"), + ("وایارگارد", "WireGuard"), + ("وایرگارد", "WireGuard"), + ("میکروتیک", "MikroTik"), + ("هرمس", "Hermes"), + ("نکست جی اس", "Next.js"), + ("ری اکت", "React"), + ("پایتون", "Python"), + ("سویفت", "Swift") + ] + private func applyDomainVocabulary(_ text: String) -> String { var t = text - let terms: [(String, String)] = [ - ("این اپ پرچس", "In-App Purchase"), - ("این اپ پرشرز", "In-App Purchase"), - ("اپ پرشرز", "In-App Purchases"), - ("اپل کانکت", "App Store Connect"), - ("اپ استور کانکت", "App Store Connect"), - ("حجزی مرچنت", "Hajzee Merchant"), - ("حجزی یوزر", "Hajzee User"), - ("حجزی", "Hajzee"), - ("نیوهورایزن", "NewHorizon"), - ("نیو هورایزن", "NewHorizon"), - ("نیوهوریزن", "NewHorizon"), - ("زر محک", "ZarMahak"), - ("زرمحک", "ZarMahak"), - ("سانی اوکس", "Soniox"), - ("سانیوکس", "Soniox"), - ("سانی اکس", "Soniox"), - ("تیم بای", "TeamBy"), - ("تیمبی", "TeamBy"), - ("دوودی", "Dovodi"), - ("داودی", "Dovodi"), - ("داکر", "Docker"), - ("کوبرنتیز", "Kubernetes"), - ("کوبرنتس", "Kubernetes"), - ("ای اس ایکس آی", "ESXi"), - ("ای اس ایکس ای", "ESXi"), - ("فست ای پی آی", "FastAPI"), - ("فست ای پی ای", "FastAPI"), - ("پستگرس کیو ال", "PostgreSQL"), - ("پستگرس", "PostgreSQL"), - ("ردیس", "Redis"), - ("فلاتر", "Flutter"), - ("وایارگارد", "WireGuard"), - ("وایرگارد", "WireGuard"), - ("میکروتیک", "MikroTik"), - ("هرمس", "Hermes"), - ("نکست جی اس", "Next.js"), - ("ری اکت", "React"), - ("پایتون", "Python"), - ("سویفت", "Swift") - ] - - for (target, canonical) in terms { + for (target, canonical) in TextProcessor.domainTerms { t = t.replacingOccurrences(of: target, with: canonical, options: .caseInsensitive) } return t @@ -176,10 +188,8 @@ public final class TextProcessor { return text case .digits, .persianDigits: - // Convert spoken Persian numbers to digits var t = " " + text + " " - // Compound tens and units let compounds: [(String, String)] = [ (" بیست و نه ", " 29 "), (" بیست و هشت ", " 28 "), (" بیست و هفت ", " 27 "), (" بیست و شش ", " 26 "), (" بیست و پنج ", " 25 "), (" بیست و چهار ", " 24 "), (" بیست و سه ", " 23 "), (" بیست و دو ", " 22 "), (" بیست و یک ", " 21 "), @@ -203,7 +213,6 @@ public final class TextProcessor { t = t.replacingOccurrences(of: word, with: num) } - // Standard single numbers & round numbers let simples: [(String, String)] = [ (" ده ", " 10 "), (" یازده ", " 11 "), (" دوازده ", " 12 "), (" سیزده ", " 13 "), (" چهارده ", " 14 "), (" پانزده ", " 15 "), (" شانزده ", " 16 "), (" هفده ", " 17 "), (" هجده ", " 18 "), (" نوزده ", " 19 "), @@ -245,20 +254,16 @@ public final class TextProcessor { private func normalizeText(_ text: String) -> String { var t = text - // Clean double spaces while t.contains(" ") { t = t.replacingOccurrences(of: " ", with: " ") } - // Remove space before punctuation let punctuationList = [".", "،", "؛", "؟", "!", ":", ")", "»"] for p in punctuationList { t = t.replacingOccurrences(of: " " + p, with: p) } - // Ensure space after punctuation (unless end of line) for p in [".", "،", "؛", "؟", "!"] { t = t.replacingOccurrences(of: p, with: p + " ") } - // Common ASR stutter & merged word fixes let speechFixes: [(String, String)] = [ (" ماا ", " ما "), (" اینپیگیری ", " این پیگیری "), @@ -270,7 +275,6 @@ public final class TextProcessor { t = t.replacingOccurrences(of: bad, with: good) } - // Fix spaces inside parentheses / quotes t = t.replacingOccurrences(of: "( ", with: "(") t = t.replacingOccurrences(of: "« ", with: "«") @@ -280,20 +284,13 @@ public final class TextProcessor { return t.trimmingCharacters(in: .whitespacesAndNewlines) } - public func stripTrailingEndTrigger(_ text: String) -> String { var t = text.trimmingCharacters(in: .whitespacesAndNewlines) - let pattern = #"[،,:\s]+(and|تمام|end)[.!?]?$"# - if let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) { - let range = NSRange(location: 0, length: t.utf16.count) - t = regex.stringByReplacingMatches(in: t, options: [], range: range, withTemplate: "") - } - let singlePattern = #"^(and|تمام|end)[.!?]?$"# - if let regex = try? NSRegularExpression(pattern: singlePattern, options: .caseInsensitive) { - let range = NSRange(location: 0, length: t.utf16.count) - t = regex.stringByReplacingMatches(in: t, options: [], range: range, withTemplate: "") - } + let range1 = NSRange(location: 0, length: t.utf16.count) + t = TextProcessor.trailingEndRegex.stringByReplacingMatches(in: t, options: [], range: range1, withTemplate: "") + + let range2 = NSRange(location: 0, length: t.utf16.count) + t = TextProcessor.singleEndRegex.stringByReplacingMatches(in: t, options: [], range: range2, withTemplate: "") return t.trimmingCharacters(in: .whitespacesAndNewlines) } - } diff --git a/src/VoiceCommandLogger.swift b/src/VoiceCommandLogger.swift index 75f456f..f897f30 100644 --- a/src/VoiceCommandLogger.swift +++ b/src/VoiceCommandLogger.swift @@ -5,6 +5,7 @@ public final class VoiceCommandLogger { public static let shared = VoiceCommandLogger() public let logFileURL: URL + private let maxLogSizeBytes: UInt64 = 5 * 1024 * 1024 // 5 MB max log size private let queue = DispatchQueue(label: "com.soniox.voicecommandlogger", qos: .utility) private let dateFormatter: DateFormatter @@ -106,7 +107,18 @@ public final class VoiceCommandLogger { } } + private func rotateIfNeeded() { + if let attrs = try? FileManager.default.attributesOfItem(atPath: logFileURL.path), + let size = attrs[.size] as? UInt64, size > maxLogSizeBytes { + let backupURL = logFileURL.deletingLastPathComponent().appendingPathComponent("voice_commands.old.log") + try? FileManager.default.removeItem(at: backupURL) + try? FileManager.default.moveItem(at: logFileURL, to: backupURL) + ensureLogHeader() + } + } + private func appendToFile(text: String) { + rotateIfNeeded() guard let data = text.data(using: .utf8) else { return } if let fileHandle = try? FileHandle(forWritingTo: logFileURL) { fileHandle.seekToEndOfFile() @@ -133,7 +145,6 @@ public final class VoiceCommandLogger { } public func openLogInViewer() { - // Open log in default text editor or Console NSWorkspace.shared.open(logFileURL) }