diff --git a/Info.plist b/Info.plist index e4836b0..b4e9dde 100644 --- a/Info.plist +++ b/Info.plist @@ -19,7 +19,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 2.14 + 2.15 CFBundleVersion 1 LSMinimumSystemVersion diff --git a/src/AppDelegate.swift b/src/AppDelegate.swift index 06ff40c..24313d6 100644 --- a/src/AppDelegate.swift +++ b/src/AppDelegate.swift @@ -159,6 +159,9 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { PingMonitor.shared.onNetworkRestored = { [weak self] in guard let self = self else { return } + if self.audioRecorder.isRecording, let active = self.activeSession, !active.isReady { + active.triggerMidRecordingReconnect(cleanArchive: false, reason: "Network restored while recording") + } SonioxSessionPool.shared.reconnectAll() let gen = self.currentGeneration() OfflineAudioQueueManager.shared.processPendingQueueIfNeeded(isCancelled: { [weak self] in @@ -167,7 +170,10 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { }) } - PingMonitor.shared.onInterfaceChanged = { + PingMonitor.shared.onInterfaceChanged = { [weak self] in + if let self = self, self.audioRecorder.isRecording { + self.activeSession?.triggerMidRecordingReconnect(cleanArchive: false, reason: "Network interface changed while recording") + } SonioxSessionPool.shared.reconnectAll() } @@ -295,6 +301,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { session.onEndTriggerDetected = { [weak self] in guard let self = self, self.currentGeneration() == myGen else { return } + guard HotkeyManager.shared.currentMode != .pushToTalk else { return } print("AppDelegate: In-flight end trigger ('and'/'تمام') detected! Stopping recording immediately...") self.stopRecording() } @@ -306,14 +313,16 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { self.hasSpokenInCurrentSession = true self.lastVoiceDetectedTime = Date() - let deltaWords = delta.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } - if let lastWord = deltaWords.last.map({ InputBoxManager.stripPunctuation($0).lowercased() }) { - if lastWord == "and" || lastWord == "تمام" || lastWord == "end" { - print("AppDelegate: In-speech end trigger detected ('\(lastWord)'). Finalizing speech...") - DispatchQueue.main.async { - self.stopRecording() + if HotkeyManager.shared.currentMode != .pushToTalk { + let deltaWords = delta.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } + if let lastWord = deltaWords.last.map({ InputBoxManager.stripPunctuation($0).lowercased() }) { + if lastWord == "and" || lastWord == "تمام" || lastWord == "end" { + print("AppDelegate: In-speech end trigger detected ('\(lastWord)'). Finalizing speech...") + DispatchQueue.main.async { + self.stopRecording() + } + return } - return } } @@ -405,15 +414,24 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { session.setRecordingActive(false) let audioSeconds = Double(pcmData.count) / 32000.0 - // FAST PATH FOR SILENT STALL / OFFLINE NETWORK: - // If user spoke (>0.8s) and socket is dead, disconnected, or had a silent TCP stall with 0 tokens, + // FAST PATH FOR SILENT STALL / OFFLINE NETWORK / UNTRANSCRIBED TAIL AUDIO: + // If user spoke (>0.8s) and socket is dead, disconnected, or has untranscribed tail audio from a mid-speech drop, // do NOT wait 4.0s-10s for finalize timeout — jump straight into Smart Audio Recovery in 0ms! if hasSpokenInCurrentSession && audioSeconds >= 0.8 { - if !PingMonitor.shared.isPathSatisfied || !session.isAlive || (session.hasSilentStall && session.totalReceivedTokensCount == 0) { - print("AppDelegate: ⚡️ Fast-path recovery triggered immediately (Silent stall / dead socket detected during speech).") + if !PingMonitor.shared.isPathSatisfied || !session.isAlive || session.hasUntranscribedTailAudio || (session.hasSilentStall && session.totalReceivedTokensCount == 0) { + let confirmedPrefix = session.confirmedTranscriptPrefix() + let tailAudio = session.untranscribedTailAudioData() + let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines) + print("AppDelegate: ⚡️ Fast-path recovery triggered immediately (confirmedPrefix: \(confirmedPrefix.count) chars, tailAudio: \(tailAudio.count) bytes).") session.abort() self.activeSession = nil - attemptFullAudioRecovery(reason: "Fast-path stall detection", generation: myGen) + attemptFullAudioRecovery( + reason: "Fast-path stall/disconnect detection", + partialFallback: bestEffort, + confirmedPrefix: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? confirmedPrefix : "", + customAudioData: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? tailAudio : nil, + generation: myGen + ) return } } @@ -441,9 +459,17 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { self.finishFinalization(text: text, error: nil, generation: myGen) } case .failure(let err): + let confirmedPrefix = session.confirmedTranscriptPrefix() + let tailAudio = session.untranscribedTailAudioData() let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines) - print("AppDelegate: Live session error (\(err.localizedDescription)). Launching full buffered audio recovery...") - self.attemptFullAudioRecovery(reason: err.localizedDescription, partialFallback: bestEffort, generation: myGen) + print("AppDelegate: Live session error (\(err.localizedDescription)). Launching buffered audio recovery (prefix: \(confirmedPrefix.count) chars, tail: \(tailAudio.count) bytes)...") + self.attemptFullAudioRecovery( + reason: err.localizedDescription, + partialFallback: bestEffort, + confirmedPrefix: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? confirmedPrefix : "", + customAudioData: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? tailAudio : nil, + generation: myGen + ) } } @@ -459,12 +485,21 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { completed = true lock.unlock() + let confirmedPrefix = session.confirmedTranscriptPrefix() + let tailAudio = session.untranscribedTailAudioData() + let hadTail = session.hasUntranscribedTailAudio let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines) session.abort() - if bestEffort.isEmpty && !InputBoxManager.shared.hasStreamedText && (self.hasSpokenInCurrentSession || pcmData.count > 16000) { - print("AppDelegate: Finalize timeout with empty text. Triggering Smart Audio Recovery...") - self.attemptFullAudioRecovery(reason: "Finalize timeout", generation: myGen) + if hadTail || (bestEffort.isEmpty && !InputBoxManager.shared.hasStreamedText && (self.hasSpokenInCurrentSession || pcmData.count > 16000)) { + print("AppDelegate: Finalize timeout with untranscribed tail/empty text. Triggering Smart Audio Recovery...") + self.attemptFullAudioRecovery( + reason: "Finalize timeout", + partialFallback: bestEffort, + confirmedPrefix: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? confirmedPrefix : "", + customAudioData: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? tailAudio : nil, + generation: myGen + ) } else { self.finishFinalization(text: bestEffort, error: nil, generation: myGen) } @@ -473,13 +508,20 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { DispatchQueue.main.asyncAfter(deadline: .now() + finalizeTimeout, execute: watchdog) } - private func attemptFullAudioRecovery(reason: String, partialFallback: String = "", generation: UInt64) { + private func attemptFullAudioRecovery( + reason: String, + partialFallback: String = "", + confirmedPrefix: String = "", + customAudioData: Data? = nil, + generation: UInt64 + ) { guard currentGeneration() == generation else { return } - let audioToRecover = self.lastRecordedAudio + let audioToRecover = customAudioData ?? self.lastRecordedAudio let audioSec = Double(audioToRecover.count) / 32000.0 guard audioToRecover.count > 6400 else { - finishFinalization(text: partialFallback.isEmpty ? nil : partialFallback, error: reason, generation: generation) + let fallbackText = !partialFallback.isEmpty ? partialFallback : confirmedPrefix + finishFinalization(text: fallbackText.isEmpty ? nil : fallbackText, error: reason, generation: generation) return } @@ -502,31 +544,33 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { guard let self = self, self.currentGeneration() == generation else { return } switch result { case .success(let recovered): - let trimmed = recovered.trimmingCharacters(in: .whitespacesAndNewlines) - if !trimmed.isEmpty { + let trimmedTail = recovered.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmedTail.isEmpty { + let combinedText = confirmedPrefix.isEmpty ? trimmedTail : (confirmedPrefix + " " + trimmedTail) VoiceCommandLogger.shared.logRecovery( audioBytes: audioToRecover.count, durationSec: audioSec, reason: reason, - outcome: "SUCCESS (Full Buffered Audio Recovered)", - recoveredText: trimmed + outcome: confirmedPrefix.isEmpty ? "SUCCESS (Full Buffered Audio Recovered)" : "SUCCESS (Confirmed Prefix + Tail Audio Recovered)", + recoveredText: combinedText ) - self.finishFinalization(text: trimmed, error: nil, generation: generation) + self.finishFinalization(text: combinedText, error: nil, generation: generation) return } case .failure: break } - if !partialFallback.isEmpty { + let fallbackToUse = !partialFallback.isEmpty ? partialFallback : confirmedPrefix + if !fallbackToUse.isEmpty { VoiceCommandLogger.shared.logRecovery( audioBytes: audioToRecover.count, durationSec: audioSec, reason: reason, outcome: "PARTIAL FALLBACK USED", - recoveredText: partialFallback + recoveredText: fallbackToUse ) - self.finishFinalization(text: partialFallback, error: nil, generation: generation) + self.finishFinalization(text: fallbackToUse, error: nil, generation: generation) } else { // Save to persistent offline queue so when internet comes back in a few minutes, it is automatically transcribed! OfflineAudioQueueManager.shared.enqueueFailedAudio(pcmData: audioToRecover) diff --git a/src/InputBoxManager.swift b/src/InputBoxManager.swift index b426023..c7ac11a 100644 --- a/src/InputBoxManager.swift +++ b/src/InputBoxManager.swift @@ -63,17 +63,19 @@ public final class InputBoxManager { public func streamDelta(text: String) { guard !text.isEmpty else { return } - // Suppress trailing end triggers ("and", "تمام", "end") from live typing - let triggerWords = text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } - if let lastRaw = triggerWords.last { - let last = InputBoxManager.stripPunctuation(lastRaw).lowercased() - if last == "and" || last == "تمام" || last == "end" { - if triggerWords.count <= 1 { - return // Drop completely if it was just the end word + // Suppress trailing end triggers ("and", "تمام", "end") ONLY in hands-free/toggle mode, never in Push-To-Talk + if HotkeyManager.shared.currentMode != .pushToTalk { + let triggerWords = text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } + if let lastRaw = triggerWords.last { + let last = InputBoxManager.stripPunctuation(lastRaw).lowercased() + if last == "and" || last == "تمام" || last == "end" { + if triggerWords.count <= 1 { + return + } + let textWithoutEnd = triggerWords.dropLast().joined(separator: " ") + " " + streamDelta(text: textWithoutEnd) + return } - let textWithoutEnd = triggerWords.dropLast().joined(separator: " ") + " " - streamDelta(text: textWithoutEnd) - return } } @@ -224,6 +226,7 @@ public final class InputBoxManager { // This guarantees that numbers (۱ -> 1), ellipses (... -> . . .), and punctuation tokens have identical word counts & tokens! let processedStreamed = rawStreamed.isEmpty ? "" : TextProcessor.shared.process(text: rawStreamed) let cleanStreamed = processedStreamed.trimmingCharacters(in: .whitespacesAndNewlines) + let streamedEndedWithSpace = rawStreamed.unicodeScalars.last.map { CharacterSet.whitespacesAndNewlines.contains($0) } ?? false pasteQueue.async { [weak self] in guard let self = self else { return } @@ -253,17 +256,14 @@ public final class InputBoxManager { // Case 1: Nothing was streamed live -> Paste entire text if cleanStreamed.isEmpty { - let pb = NSPasteboard.general - pb.clearContents() - pb.setString(textWithTrailing, forType: .string) - self.postCleanCmdV() + self.pasteSynchronously(textWithTrailing) restoreClipboardFullText() return } // Case 2: Exactly matches streamed text if cleanStreamed == trimmed { - if addTrailingSpace { + if addTrailingSpace && !streamedEndedWithSpace { self.pasteSynchronously(" ") } restoreClipboardFullText() @@ -272,9 +272,15 @@ public final class InputBoxManager { // Case 3: Streamed prefix is exact prefix of trimmed final text if trimmed.count > cleanStreamed.count && trimmed.hasPrefix(cleanStreamed) { - let suffix = String(trimmed.dropFirst(cleanStreamed.count)) - let suffixToPaste = addTrailingSpace ? (suffix + " ") : suffix - self.pasteSynchronously(suffixToPaste) + let rawSuffix = String(trimmed.dropFirst(cleanStreamed.count)) + let cleanSuffix = rawSuffix.trimmingCharacters(in: .whitespacesAndNewlines) + if !cleanSuffix.isEmpty { + let needsLeadingSpace = !streamedEndedWithSpace && (rawSuffix.first?.isWhitespace == true) + let suffixToPaste = (needsLeadingSpace ? " " : "") + cleanSuffix + (addTrailingSpace ? " " : "") + self.pasteSynchronously(suffixToPaste) + } else if addTrailingSpace && !streamedEndedWithSpace { + self.pasteSynchronously(" ") + } restoreClipboardFullText() return } @@ -374,12 +380,14 @@ public final class InputBoxManager { if startIndex < finalPairs.count { let remainingWords = finalPairs.dropFirst(startIndex).map { $0.raw } let suffix = remainingWords.joined(separator: " ") - let suffixToPaste = addTrailingSpace ? (" " + suffix + " ") : (" " + suffix) + let leadingSpace = streamedEndedWithSpace ? "" : " " + let trailingSpace = addTrailingSpace ? " " : "" + let suffixToPaste = leadingSpace + suffix + trailingSpace self.pasteSynchronously(suffixToPaste) restoreClipboardFullText() return } else { - if addTrailingSpace { + if addTrailingSpace && !streamedEndedWithSpace { self.pasteSynchronously(" ") } restoreClipboardFullText() @@ -390,13 +398,39 @@ public final class InputBoxManager { } } - /// Places text on NSPasteboard, posts Cmd+V, and waits 22ms so target app consumes this chunk BEFORE next clipboard write! + /// Places core word(s) on NSPasteboard via Cmd+V, while injecting leading/trailing spaces + /// via native Space keystrokes so apps that `.trim()` pasted clipboard text (like Hermes Desktop) + /// never glue words together or double-space! private func pasteSynchronously(_ text: String) { + guard !text.isEmpty else { return } + let horizontalSpaceSet = CharacterSet(charactersIn: " \t") + let hasLeadingSpace = text.unicodeScalars.first.map { horizontalSpaceSet.contains($0) } ?? false + let hasTrailingSpace = text.unicodeScalars.last.map { horizontalSpaceSet.contains($0) } ?? false + let core = text.trimmingCharacters(in: horizontalSpaceSet) + + if core.isEmpty { + if hasLeadingSpace || hasTrailingSpace { + postCleanSpace() + usleep(12000) + } + return + } + + if hasLeadingSpace { + postCleanSpace() + usleep(12000) + } + let pb = NSPasteboard.general pb.clearContents() - pb.setString(text, forType: .string) + pb.setString(core, forType: .string) postCleanCmdV() usleep(22000) // 22ms guard ensures browser/app reads NSPasteboard before next chunk overwrites it (~45 chunks/sec capacity) + + if hasTrailingSpace { + postCleanSpace() + usleep(12000) + } } private func postCleanCmdV() { @@ -411,4 +445,20 @@ public final class InputBoxManager { up.post(tap: .cgAnnotatedSessionEventTap) } } + + private func postCleanSpace() { + let src = CGEventSource(stateID: .privateState) + let spaceKeyCode: CGKeyCode = 0x31 // ' ' (kVK_Space) + var spaceChar: UniChar = 0x0020 + + if let down = CGEvent(keyboardEventSource: src, virtualKey: spaceKeyCode, keyDown: true), + let up = CGEvent(keyboardEventSource: src, virtualKey: spaceKeyCode, keyDown: false) { + down.flags = [] + up.flags = [] + down.keyboardSetUnicodeString(stringLength: 1, unicodeString: &spaceChar) + up.keyboardSetUnicodeString(stringLength: 1, unicodeString: &spaceChar) + down.post(tap: .cgAnnotatedSessionEventTap) + up.post(tap: .cgAnnotatedSessionEventTap) + } + } } diff --git a/src/SonioxClient.swift b/src/SonioxClient.swift index b83d0c0..d3908c6 100644 --- a/src/SonioxClient.swift +++ b/src/SonioxClient.swift @@ -35,10 +35,16 @@ public final class SonioxLiveSession { private var isAborted = false private let lock = NSLock() - // Transcripts & Stall tracking + // Transcripts & Stall tracking (Multi-segment continuous speech + checkpointed audio buffer) + private var archivedCommittedText: String = "" private var committedFinalTokens: [String] = [] private var currentNonFinalTokens: [String] = [] private var streamedPrefix: String = "" + private var currentSegmentAudioBuffer = Data() + private var lastConfirmedWordAudioByteOffset: Int = 0 + private var isReconnectingMidRecording = false + private var isFlushingReplayBuffer = false + private var segmentConnectTime: Date = Date() private var sentAudioBytes: Int = 0 public private(set) var totalReceivedTokensCount: Int = 0 public private(set) var hasSilentStall: Bool = false @@ -61,10 +67,46 @@ public final class SonioxLiveSession { isRecordingActive = active if active { lastDataTime = Date() + segmentConnectTime = Date() } lock.unlock() } + /// True when a mid-recording disconnect/stall occurred and there is >0.35s of unconfirmed tail audio waiting to be transcribed. + public var hasUntranscribedTailAudio: Bool { + lock.lock() + defer { lock.unlock() } + let tailCount = max(0, currentSegmentAudioBuffer.count - lastConfirmedWordAudioByteOffset) + let hadDrop = hasNetworkFailure || hasSilentStall || isReconnectingMidRecording || !isConnected + return hadDrop && tailCount > 11200 + } + + /// Returns the already-confirmed transcript prefix across all completed segments and finalized tokens. + public func confirmedTranscriptPrefix() -> String { + lock.lock() + let segCommitted = committedFinalTokens.joined().trimmingCharacters(in: .whitespacesAndNewlines) + let rawPrefix: String + if archivedCommittedText.isEmpty { + rawPrefix = segCommitted + } else if segCommitted.isEmpty { + rawPrefix = archivedCommittedText + } else { + rawPrefix = archivedCommittedText + " " + segCommitted + } + lock.unlock() + return SonioxLiveSession.sanitizeText(rawPrefix) + } + + /// Returns the slice of raw PCM audio from `lastConfirmedWordAudioByteOffset` to the end of `currentSegmentAudioBuffer`. + public func untranscribedTailAudioData() -> Data { + lock.lock() + defer { lock.unlock() } + guard currentSegmentAudioBuffer.count > lastConfirmedWordAudioByteOffset else { + return Data() + } + return currentSegmentAudioBuffer.subdata(in: lastConfirmedWordAudioByteOffset..1.5s of audio (48KB) was sent while speaking, but 0 frames arrived for >3.0s: - if connected && sent > 48000 && elapsedSinceData > 3.0 { + // If >1.0s of audio (32KB) was sent while speaking, but 0 frames arrived for >3.5s: + if connected && sent > 32000 && elapsedSinceData > 3.5 { lock.lock() hasSilentStall = true lock.unlock() DispatchQueue.main.async { [weak self] in self?.onNetworkDegraded?(true) } + if activeRec { + triggerMidRecordingReconnect(cleanArchive: false, reason: "Silent TCP/STT stall (\(String(format: "%.1f", elapsedSinceData))s without frames while speaking)") + } + } + } + + /// Seamlessly archives confirmed transcript tokens and reconnects a fresh WebSocket in the background + /// while the user continues holding Push-To-Talk / speaking for 5–10+ minutes! + public func triggerMidRecordingReconnect(cleanArchive: Bool, reason: String) { + lock.lock() + guard isRecordingActive, !isFinalizing, !isClosed, !isAborted, !isReconnectingMidRecording else { + lock.unlock() + return + } + isReconnectingMidRecording = true + isConnected = false + + let segCommitted = committedFinalTokens.joined() + let segNonFinal = currentNonFinalTokens.joined() + let toArchive = cleanArchive ? (segCommitted + segNonFinal) : segCommitted + let cleanToArchive = toArchive.trimmingCharacters(in: .whitespacesAndNewlines) + if !cleanToArchive.isEmpty { + if archivedCommittedText.isEmpty { + archivedCommittedText = cleanToArchive + } else { + archivedCommittedText += " " + cleanToArchive + } + } + committedFinalTokens.removeAll() + currentNonFinalTokens.removeAll() + + if cleanArchive { + currentSegmentAudioBuffer.removeAll(keepingCapacity: true) + lastConfirmedWordAudioByteOffset = 0 + } else if lastConfirmedWordAudioByteOffset > 0 && lastConfirmedWordAudioByteOffset <= currentSegmentAudioBuffer.count { + currentSegmentAudioBuffer.removeSubrange(0..= total { + isFlushingReplayBuffer = false + lock.unlock() + return + } + let chunkSize = 32000 // 1.0s chunk + let end = min(offset + chunkSize, total) + let slice = currentSegmentAudioBuffer.subdata(in: offset..150s when at a clean committed pause boundary so server never kills long 5–10m streams + let segAge = Date().timeIntervalSince(segmentConnectTime) + let canRotateProactively = segAge > 150.0 && currentNonFinalTokens.isEmpty && !committedFinalTokens.isEmpty lock.unlock() + if canRotateProactively { + triggerMidRecordingReconnect(cleanArchive: true, reason: "Proactive 150s continuous speech segment rollover") + return + } + let t0 = Date() let message = URLSessionWebSocketTask.Message.data(data) task.send(message) { [weak self] error in @@ -297,15 +539,19 @@ public final class SonioxLiveSession { if let error = error { print("SonioxLiveSession: Error streaming audio chunk:", error) self.lock.lock() - if !self.isFallbackSwitched && !self.isFinalizing && !self.isClosed && !self.isAborted { + if !self.isFallbackSwitched && !self.isFinalizing && !self.isClosed && !self.isAborted && self.archivedCommittedText.isEmpty && self.committedFinalTokens.isEmpty { self.lock.unlock() self.switchToFallback() return } self.hasNetworkFailure = true self.isConnected = false + let activeRec = self.isRecordingActive && !self.isFinalizing && !self.isClosed && !self.isAborted self.lock.unlock() self.onNetworkDegraded?(true) + if activeRec { + self.triggerMidRecordingReconnect(cleanArchive: false, reason: "Send audio chunk error: \(error.localizedDescription)") + } } else { self.lock.lock() self.sentAudioBytes += data.count @@ -320,13 +566,21 @@ public final class SonioxLiveSession { public func finalizeStream() { lock.lock() - guard !isFinalizing, !isClosed, !isAborted, let task = webSocketTask else { + guard !isFinalizing, !isClosed, !isAborted else { lock.unlock() return } isFinalizing = true + let connected = isConnected && !isReconnectingMidRecording + let task = webSocketTask lock.unlock() + guard connected, let task = task else { + print("SonioxLiveSession: finalizeStream called while socket disconnected/reconnecting — completing immediately for tail recovery.") + completeWithCurrentText() + return + } + print("SonioxLiveSession: Sending finalize packet...") // 1. Send trailing 300ms (9600 bytes at 16kHz 16-bit PCM) silence frame // so acoustic CTC model cleanly flushes and resolves trailing fast speech syllables @@ -403,14 +657,15 @@ public final class SonioxLiveSession { self.lock.lock() self.hasNetworkFailure = true self.isConnected = false - let activeRecording = self.isRecordingActive + let activeRecording = self.isRecordingActive && !self.isFinalizing && !self.isClosed && !self.isAborted self.lock.unlock() self.onNetworkDegraded?(true) if !activeRecording { self.completeWithCurrentText() } else { - print("SonioxLiveSession: Network dropped mid-recording. Audio is buffered for Smart Recovery upon key release.") + print("SonioxLiveSession: Network dropped mid-recording. Triggering automatic live reconnect...") + self.triggerMidRecordingReconnect(cleanArchive: false, reason: "WebSocket receive error: \(error.localizedDescription)") } } } @@ -465,21 +720,40 @@ public final class SonioxLiveSession { totalReceivedTokensCount += tokenCountDelta if !newFinals.isEmpty { committedFinalTokens.append(contentsOf: newFinals) + // Advance confirmed audio checkpoint in `currentSegmentAudioBuffer`: + // Keep a safety acoustic lookback proportional to unfinalized words (plus 0.8s = 25600 bytes) + let unfinalizedWords = newNonFinals.joined().components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }.count + let safetyLookbackBytes = max(25600, (unfinalizedWords + 1) * 16000) + if currentSegmentAudioBuffer.count > safetyLookbackBytes { + lastConfirmedWordAudioByteOffset = max(lastConfirmedWordAudioByteOffset, currentSegmentAudioBuffer.count - safetyLookbackBytes) + } } currentNonFinalTokens = newNonFinals let fullCommitted = committedFinalTokens.joined() let fullNonFinal = currentNonFinalTokens.joined() - let combined = fullCommitted + fullNonFinal + let segmentCombined = fullCommitted + fullNonFinal + let combined: String + if archivedCommittedText.isEmpty { + combined = segmentCombined + } else if segmentCombined.isEmpty { + combined = archivedCommittedText + } else { + let cleanSeg = segmentCombined.trimmingCharacters(in: .whitespaces) + combined = archivedCommittedText + (cleanSeg.isEmpty ? "" : (" " + cleanSeg)) + } - // Check for in-flight end trigger ("and", "تمام", "end") in combined text + // Check for in-flight end trigger ("and", "تمام", "end") ONLY in hands-free/toggle mode and ONLY on finalized tokens! + // Never trigger auto-stop during Push-To-Talk while user is physically holding the hotkey! var triggerDetected = false - let trimmedCombined = combined.trimmingCharacters(in: .whitespacesAndNewlines) - let currentWords = trimmedCombined.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } - if let lastRaw = currentWords.last { - let cleanLast = InputBoxManager.stripPunctuation(lastRaw).lowercased() - if cleanLast == "and" || cleanLast == "تمام" || cleanLast == "end" { - triggerDetected = true + if HotkeyManager.shared.currentMode != .pushToTalk && currentNonFinalTokens.isEmpty && !newFinals.isEmpty { + let trimmedCombined = combined.trimmingCharacters(in: .whitespacesAndNewlines) + let currentWords = trimmedCombined.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } + if let lastRaw = currentWords.last { + let cleanLast = InputBoxManager.stripPunctuation(lastRaw).lowercased() + if cleanLast == "and" || cleanLast == "تمام" || cleanLast == "end" { + triggerDetected = true + } } } @@ -492,27 +766,16 @@ public final class SonioxLiveSession { return raw.contains("\"finished\":true") || raw.contains("\"finished\": true") }() let isTerminatingMessage = gotFin || sessionEnded || sessionDone || isFinished || hasRawFinished + let stillActivelyRecording = self.isRecordingActive && !self.isFinalizing // Emit live streaming deltas in real time as the user speaks! - // Soniox batches `is_final: true` tokens only every 6-8 seconds, which would make live typing feel lagging and out of sync. - // However, completed words in `combined` (with a 1-word acoustic lookahead holdback) are 100% stable and appear within ~150ms! - // We stream `fullCommitted` PLUS any completed words in `fullNonFinal` except the trailing word currently being uttered. + // We stream completed words in `combined` with a 1-word acoustic lookahead holdback so every emitted chunk ends with a trailing space. var deltaToEmit: String? = nil if !triggerDetected && !isTerminatingMessage && !self.isFinalizing { - var stableCandidate = fullCommitted - if let lastSpaceRange = fullNonFinal.rangeOfCharacter(from: .whitespacesAndNewlines, options: .backwards) { - let completedNonFinal = String(fullNonFinal[.. streamedPrefix.count { @@ -521,9 +784,10 @@ public final class SonioxLiveSession { streamedPrefix = stableCandidate deltaToEmit = delta } else { - let common = stableCandidate.commonPrefix(with: streamedPrefix) - if stableCandidate.count > common.count { - let delta = String(stableCandidate.dropFirst(common.count)) + let streamedWordsCount = streamedPrefix.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }.count + let candidateWords = stableCandidate.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } + if candidateWords.count > streamedWordsCount { + let delta = candidateWords.dropFirst(streamedWordsCount).joined(separator: " ") + " " streamedPrefix = stableCandidate deltaToEmit = delta } @@ -549,7 +813,12 @@ public final class SonioxLiveSession { } if isTerminatingMessage { - completeWithCurrentText() + if stillActivelyRecording { + print("SonioxLiveSession: Server ended WebSocket segment while user is still speaking — seamlessly rotating to fresh segment...") + triggerMidRecordingReconnect(cleanArchive: true, reason: "Server ended WebSocket segment while recording is still active") + } else { + completeWithCurrentText() + } } } else { let sessionEnded = json["session_ended"] as? Bool ?? false @@ -560,7 +829,14 @@ public final class SonioxLiveSession { return raw.contains("\"finished\":true") || raw.contains("\"finished\": true") }() if sessionEnded || sessionDone || isFinished || hasRawFinished { - completeWithCurrentText() + lock.lock() + let stillActivelyRecording = self.isRecordingActive && !self.isFinalizing + lock.unlock() + if stillActivelyRecording { + triggerMidRecordingReconnect(cleanArchive: true, reason: "Server session_done while recording is still active") + } else { + completeWithCurrentText() + } } } } @@ -569,7 +845,15 @@ public final class SonioxLiveSession { lock.lock() let fullCommitted = committedFinalTokens.joined() let fullNonFinal = currentNonFinalTokens.joined() - let rawCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal) + let segCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal) + let rawCombined: String + if archivedCommittedText.isEmpty { + rawCombined = segCombined + } else if segCombined.isEmpty { + rawCombined = archivedCommittedText + } else { + rawCombined = archivedCommittedText + " " + segCombined.trimmingCharacters(in: .whitespaces) + } lock.unlock() return SonioxLiveSession.sanitizeText(rawCombined) } @@ -580,10 +864,23 @@ public final class SonioxLiveSession { lock.unlock() return } + let tailCount = max(0, currentSegmentAudioBuffer.count - lastConfirmedWordAudioByteOffset) + let hadDrop = hasNetworkFailure || hasSilentStall || isReconnectingMidRecording || !isConnected + let needsTailRecovery = hadDrop && tailCount > 11200 + isClosed = true + isReconnectingMidRecording = false let fullCommitted = committedFinalTokens.joined() let fullNonFinal = currentNonFinalTokens.joined() - let rawCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal) + let segCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal) + let rawCombined: String + if archivedCommittedText.isEmpty { + rawCombined = segCombined + } else if segCombined.isEmpty { + rawCombined = archivedCommittedText + } else { + rawCombined = archivedCommittedText + " " + segCombined.trimmingCharacters(in: .whitespaces) + } let cleaned = SonioxLiveSession.sanitizeText(rawCombined) let failureOccurred = hasNetworkFailure @@ -597,7 +894,9 @@ public final class SonioxLiveSession { stopKeepAliveHeartbeat() DispatchQueue.main.async { - if cleaned.isEmpty && failureOccurred { + if needsTailRecovery { + cb?(.failure(NSError(domain: "SonioxLiveSession", code: -103, userInfo: [NSLocalizedDescriptionKey: "Partial transcript with \(tailCount) bytes of untranscribed tail audio"]))) + } else if cleaned.isEmpty && failureOccurred { cb?(.failure(NSError(domain: "SonioxLiveSession", code: -101, userInfo: [NSLocalizedDescriptionKey: "WebSocket network failure with empty transcript"]))) } else { cb?(.success(cleaned)) @@ -613,7 +912,10 @@ public final class SonioxLiveSession { isConnected = false isRecordingActive = false isFinalizing = false + isReconnectingMidRecording = false + isFlushingReplayBuffer = false initialAudioBuffer.removeAll() + currentSegmentAudioBuffer.removeAll() onPartialText = nil onCommittedDelta = nil onEndTriggerDetected = nil @@ -656,6 +958,128 @@ public final class SonioxLiveSession { let durationSec = Double(pcmData.count) / 32000.0 print("SonioxLiveSession: Initiating Smart Burst Recovery: \(pcmData.count) bytes (\(String(format: "%.1f", durationSec))s audio, target: \(effectiveTarget ?? "orig"))...") + // For multi-minute recordings (>60s = 1,920,000 bytes), split into ~60s silence-aligned chunks + // so individual WebSocket burst connections never hit server buffer limits or timeouts! + let maxSingleBurstBytes = 1_920_000 + if pcmData.count > maxSingleBurstBytes { + let chunks = splitIntoSilenceAlignedChunks(pcmData: pcmData, targetChunkBytes: maxSingleBurstBytes) + print("SonioxLiveSession: Splitting long recovery audio (\(String(format: "%.1f", durationSec))s) into \(chunks.count) sequential chunks...") + var recoveredParts: [String] = [] + + func processChunk(at index: Int) { + if isCancelled() { + completion(.failure(NSError(domain: "SonioxLiveSession", code: -999, userInfo: [NSLocalizedDescriptionKey: "Cancelled by Escape"]))) + return + } + if index >= chunks.count { + let combinedText = recoveredParts.joined(separator: " ").trimmingCharacters(in: .whitespacesAndNewlines) + if !combinedText.isEmpty { + completion(.success(combinedText)) + } else { + completion(.failure(NSError(domain: "SonioxLiveSession", code: -104, userInfo: [NSLocalizedDescriptionKey: "Empty chunked recovery result"]))) + } + return + } + DispatchQueue.main.async { + onStatusUpdate?("PART \(index + 1)/\(chunks.count)") + } + retrySingleChunk( + pcmData: chunks[index], + languageHints: languageHints, + effectiveTarget: effectiveTarget, + isCancelled: isCancelled, + onProgress: onProgress, + onStatusUpdate: onStatusUpdate + ) { result in + switch result { + case .success(let partText): + let cleanPart = partText.trimmingCharacters(in: .whitespacesAndNewlines) + if !cleanPart.isEmpty { + recoveredParts.append(cleanPart) + } + processChunk(at: index + 1) + case .failure(let err): + if !recoveredParts.isEmpty { + // Salvage already-recovered chunks if a later chunk fails + let partial = recoveredParts.joined(separator: " ").trimmingCharacters(in: .whitespacesAndNewlines) + completion(.success(partial)) + } else { + completion(.failure(err)) + } + } + } + } + + processChunk(at: 0) + return + } + + retrySingleChunk( + pcmData: pcmData, + languageHints: languageHints, + effectiveTarget: effectiveTarget, + isCancelled: isCancelled, + onProgress: onProgress, + onStatusUpdate: onStatusUpdate, + completion: completion + ) + } + + /// Splits a multi-minute 16kHz 16-bit mono PCM buffer at low-energy silence boundaries near `targetChunkBytes`. + private static func splitIntoSilenceAlignedChunks(pcmData: Data, targetChunkBytes: Int) -> [Data] { + var chunks: [Data] = [] + var offset = 0 + let total = pcmData.count + let searchWindowBytes = 160_000 // Search within last 5.0s of each 60s window for lowest energy pause + let windowStepBytes = 3200 // 100ms window + + while offset < total { + let remaining = total - offset + if remaining <= targetChunkBytes + 160_000 { + chunks.append(pcmData.subdata(in: offset.. searchStart { + var minEnergy: Float = .greatestFiniteMagnitude + pcmData.withUnsafeBytes { rawBuf in + guard let basePtr = rawBuf.baseAddress else { return } + var pos = searchStart & ~1 // align to 2-byte Int16 sample + while pos < searchEnd { + let sampleCount = windowStepBytes / 2 + let ptr = basePtr.advanced(by: pos).assumingMemoryBound(to: Int16.self) + var sumSq: Float = 0 + for i in 0.. Bool, + onProgress: ((String) -> Void)?, + onStatusUpdate: ((String) -> Void)?, + completion: @escaping (Result) -> Void + ) { // Multi-stage resilient retry pipeline across dual endpoints with automatic backoff: // Attempt 1: Primary WS // Attempt 2: Fallback WS @@ -761,7 +1185,7 @@ public final class SonioxLiveSession { request.setValue(origin(for: endpoint), forHTTPHeaderField: "Origin") let durationSec = Double(pcmData.count) / 32000.0 - let watchdogTimeout = max(6.5, min(25.0, durationSec * 0.60 + 5.0)) + let watchdogTimeout = max(10.0, min(65.0, durationSec * 0.90 + 10.0)) request.timeoutInterval = 5.0 // Fast handshake timeout so dead endpoint fails over in <5s let config = URLSessionConfiguration.ephemeral diff --git a/src/main.swift b/src/main.swift index ef8e078..958177a 100644 --- a/src/main.swift +++ b/src/main.swift @@ -1,5 +1,8 @@ import Cocoa +setbuf(stdout, nil) +setbuf(stderr, nil) + func checkSingleInstance() { let myPid = ProcessInfo.processInfo.processIdentifier