Browse Source

fix(stt): continuous multi-segment reconnect, silence-aligned long audio chunking & dual space paste injection (v2.15)

main
Ali Alavi 2 weeks ago
parent
commit
36472fc2a2
  1. 2
      Info.plist
  2. 102
      src/AppDelegate.swift
  3. 94
      src/InputBoxManager.swift
  4. 512
      src/SonioxClient.swift
  5. 3
      src/main.swift

2
Info.plist

@ -19,7 +19,7 @@
<key>CFBundlePackageType</key> <key>CFBundlePackageType</key>
<string>APPL</string> <string>APPL</string>
<key>CFBundleShortVersionString</key> <key>CFBundleShortVersionString</key>
<string>2.14</string>
<string>2.15</string>
<key>CFBundleVersion</key> <key>CFBundleVersion</key>
<string>1</string> <string>1</string>
<key>LSMinimumSystemVersion</key> <key>LSMinimumSystemVersion</key>

102
src/AppDelegate.swift

@ -159,6 +159,9 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
PingMonitor.shared.onNetworkRestored = { [weak self] in PingMonitor.shared.onNetworkRestored = { [weak self] in
guard let self = self else { return } guard let self = self else { return }
if self.audioRecorder.isRecording, let active = self.activeSession, !active.isReady {
active.triggerMidRecordingReconnect(cleanArchive: false, reason: "Network restored while recording")
}
SonioxSessionPool.shared.reconnectAll() SonioxSessionPool.shared.reconnectAll()
let gen = self.currentGeneration() let gen = self.currentGeneration()
OfflineAudioQueueManager.shared.processPendingQueueIfNeeded(isCancelled: { [weak self] in OfflineAudioQueueManager.shared.processPendingQueueIfNeeded(isCancelled: { [weak self] in
@ -167,7 +170,10 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
}) })
} }
PingMonitor.shared.onInterfaceChanged = {
PingMonitor.shared.onInterfaceChanged = { [weak self] in
if let self = self, self.audioRecorder.isRecording {
self.activeSession?.triggerMidRecordingReconnect(cleanArchive: false, reason: "Network interface changed while recording")
}
SonioxSessionPool.shared.reconnectAll() SonioxSessionPool.shared.reconnectAll()
} }
@ -295,6 +301,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
session.onEndTriggerDetected = { [weak self] in session.onEndTriggerDetected = { [weak self] in
guard let self = self, self.currentGeneration() == myGen else { return } guard let self = self, self.currentGeneration() == myGen else { return }
guard HotkeyManager.shared.currentMode != .pushToTalk else { return }
print("AppDelegate: In-flight end trigger ('and'/'تمام') detected! Stopping recording immediately...") print("AppDelegate: In-flight end trigger ('and'/'تمام') detected! Stopping recording immediately...")
self.stopRecording() self.stopRecording()
} }
@ -306,14 +313,16 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
self.hasSpokenInCurrentSession = true self.hasSpokenInCurrentSession = true
self.lastVoiceDetectedTime = Date() self.lastVoiceDetectedTime = Date()
let deltaWords = delta.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastWord = deltaWords.last.map({ InputBoxManager.stripPunctuation($0).lowercased() }) {
if lastWord == "and" || lastWord == "تمام" || lastWord == "end" {
print("AppDelegate: In-speech end trigger detected ('\(lastWord)'). Finalizing speech...")
DispatchQueue.main.async {
self.stopRecording()
if HotkeyManager.shared.currentMode != .pushToTalk {
let deltaWords = delta.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastWord = deltaWords.last.map({ InputBoxManager.stripPunctuation($0).lowercased() }) {
if lastWord == "and" || lastWord == "تمام" || lastWord == "end" {
print("AppDelegate: In-speech end trigger detected ('\(lastWord)'). Finalizing speech...")
DispatchQueue.main.async {
self.stopRecording()
}
return
} }
return
} }
} }
@ -405,15 +414,24 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
session.setRecordingActive(false) session.setRecordingActive(false)
let audioSeconds = Double(pcmData.count) / 32000.0 let audioSeconds = Double(pcmData.count) / 32000.0
// FAST PATH FOR SILENT STALL / OFFLINE NETWORK:
// If user spoke (>0.8s) and socket is dead, disconnected, or had a silent TCP stall with 0 tokens,
// FAST PATH FOR SILENT STALL / OFFLINE NETWORK / UNTRANSCRIBED TAIL AUDIO:
// If user spoke (>0.8s) and socket is dead, disconnected, or has untranscribed tail audio from a mid-speech drop,
// do NOT wait 4.0s-10s for finalize timeout — jump straight into Smart Audio Recovery in 0ms! // do NOT wait 4.0s-10s for finalize timeout — jump straight into Smart Audio Recovery in 0ms!
if hasSpokenInCurrentSession && audioSeconds >= 0.8 { if hasSpokenInCurrentSession && audioSeconds >= 0.8 {
if !PingMonitor.shared.isPathSatisfied || !session.isAlive || (session.hasSilentStall && session.totalReceivedTokensCount == 0) {
print("AppDelegate: ⚡️ Fast-path recovery triggered immediately (Silent stall / dead socket detected during speech).")
if !PingMonitor.shared.isPathSatisfied || !session.isAlive || session.hasUntranscribedTailAudio || (session.hasSilentStall && session.totalReceivedTokensCount == 0) {
let confirmedPrefix = session.confirmedTranscriptPrefix()
let tailAudio = session.untranscribedTailAudioData()
let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines)
print("AppDelegate: ⚡️ Fast-path recovery triggered immediately (confirmedPrefix: \(confirmedPrefix.count) chars, tailAudio: \(tailAudio.count) bytes).")
session.abort() session.abort()
self.activeSession = nil self.activeSession = nil
attemptFullAudioRecovery(reason: "Fast-path stall detection", generation: myGen)
attemptFullAudioRecovery(
reason: "Fast-path stall/disconnect detection",
partialFallback: bestEffort,
confirmedPrefix: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? confirmedPrefix : "",
customAudioData: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? tailAudio : nil,
generation: myGen
)
return return
} }
} }
@ -441,9 +459,17 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
self.finishFinalization(text: text, error: nil, generation: myGen) self.finishFinalization(text: text, error: nil, generation: myGen)
} }
case .failure(let err): case .failure(let err):
let confirmedPrefix = session.confirmedTranscriptPrefix()
let tailAudio = session.untranscribedTailAudioData()
let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines) let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines)
print("AppDelegate: Live session error (\(err.localizedDescription)). Launching full buffered audio recovery...")
self.attemptFullAudioRecovery(reason: err.localizedDescription, partialFallback: bestEffort, generation: myGen)
print("AppDelegate: Live session error (\(err.localizedDescription)). Launching buffered audio recovery (prefix: \(confirmedPrefix.count) chars, tail: \(tailAudio.count) bytes)...")
self.attemptFullAudioRecovery(
reason: err.localizedDescription,
partialFallback: bestEffort,
confirmedPrefix: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? confirmedPrefix : "",
customAudioData: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? tailAudio : nil,
generation: myGen
)
} }
} }
@ -459,12 +485,21 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
completed = true completed = true
lock.unlock() lock.unlock()
let confirmedPrefix = session.confirmedTranscriptPrefix()
let tailAudio = session.untranscribedTailAudioData()
let hadTail = session.hasUntranscribedTailAudio
let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines) let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines)
session.abort() session.abort()
if bestEffort.isEmpty && !InputBoxManager.shared.hasStreamedText && (self.hasSpokenInCurrentSession || pcmData.count > 16000) {
print("AppDelegate: Finalize timeout with empty text. Triggering Smart Audio Recovery...")
self.attemptFullAudioRecovery(reason: "Finalize timeout", generation: myGen)
if hadTail || (bestEffort.isEmpty && !InputBoxManager.shared.hasStreamedText && (self.hasSpokenInCurrentSession || pcmData.count > 16000)) {
print("AppDelegate: Finalize timeout with untranscribed tail/empty text. Triggering Smart Audio Recovery...")
self.attemptFullAudioRecovery(
reason: "Finalize timeout",
partialFallback: bestEffort,
confirmedPrefix: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? confirmedPrefix : "",
customAudioData: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? tailAudio : nil,
generation: myGen
)
} else { } else {
self.finishFinalization(text: bestEffort, error: nil, generation: myGen) self.finishFinalization(text: bestEffort, error: nil, generation: myGen)
} }
@ -473,13 +508,20 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
DispatchQueue.main.asyncAfter(deadline: .now() + finalizeTimeout, execute: watchdog) DispatchQueue.main.asyncAfter(deadline: .now() + finalizeTimeout, execute: watchdog)
} }
private func attemptFullAudioRecovery(reason: String, partialFallback: String = "", generation: UInt64) {
private func attemptFullAudioRecovery(
reason: String,
partialFallback: String = "",
confirmedPrefix: String = "",
customAudioData: Data? = nil,
generation: UInt64
) {
guard currentGeneration() == generation else { return } guard currentGeneration() == generation else { return }
let audioToRecover = self.lastRecordedAudio
let audioToRecover = customAudioData ?? self.lastRecordedAudio
let audioSec = Double(audioToRecover.count) / 32000.0 let audioSec = Double(audioToRecover.count) / 32000.0
guard audioToRecover.count > 6400 else { guard audioToRecover.count > 6400 else {
finishFinalization(text: partialFallback.isEmpty ? nil : partialFallback, error: reason, generation: generation)
let fallbackText = !partialFallback.isEmpty ? partialFallback : confirmedPrefix
finishFinalization(text: fallbackText.isEmpty ? nil : fallbackText, error: reason, generation: generation)
return return
} }
@ -502,31 +544,33 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
guard let self = self, self.currentGeneration() == generation else { return } guard let self = self, self.currentGeneration() == generation else { return }
switch result { switch result {
case .success(let recovered): case .success(let recovered):
let trimmed = recovered.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty {
let trimmedTail = recovered.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmedTail.isEmpty {
let combinedText = confirmedPrefix.isEmpty ? trimmedTail : (confirmedPrefix + " " + trimmedTail)
VoiceCommandLogger.shared.logRecovery( VoiceCommandLogger.shared.logRecovery(
audioBytes: audioToRecover.count, audioBytes: audioToRecover.count,
durationSec: audioSec, durationSec: audioSec,
reason: reason, reason: reason,
outcome: "SUCCESS (Full Buffered Audio Recovered)",
recoveredText: trimmed
outcome: confirmedPrefix.isEmpty ? "SUCCESS (Full Buffered Audio Recovered)" : "SUCCESS (Confirmed Prefix + Tail Audio Recovered)",
recoveredText: combinedText
) )
self.finishFinalization(text: trimmed, error: nil, generation: generation)
self.finishFinalization(text: combinedText, error: nil, generation: generation)
return return
} }
case .failure: case .failure:
break break
} }
if !partialFallback.isEmpty {
let fallbackToUse = !partialFallback.isEmpty ? partialFallback : confirmedPrefix
if !fallbackToUse.isEmpty {
VoiceCommandLogger.shared.logRecovery( VoiceCommandLogger.shared.logRecovery(
audioBytes: audioToRecover.count, audioBytes: audioToRecover.count,
durationSec: audioSec, durationSec: audioSec,
reason: reason, reason: reason,
outcome: "PARTIAL FALLBACK USED", outcome: "PARTIAL FALLBACK USED",
recoveredText: partialFallback
recoveredText: fallbackToUse
) )
self.finishFinalization(text: partialFallback, error: nil, generation: generation)
self.finishFinalization(text: fallbackToUse, error: nil, generation: generation)
} else { } else {
// Save to persistent offline queue so when internet comes back in a few minutes, it is automatically transcribed! // Save to persistent offline queue so when internet comes back in a few minutes, it is automatically transcribed!
OfflineAudioQueueManager.shared.enqueueFailedAudio(pcmData: audioToRecover) OfflineAudioQueueManager.shared.enqueueFailedAudio(pcmData: audioToRecover)

94
src/InputBoxManager.swift

@ -63,17 +63,19 @@ public final class InputBoxManager {
public func streamDelta(text: String) { public func streamDelta(text: String) {
guard !text.isEmpty else { return } guard !text.isEmpty else { return }
// Suppress trailing end triggers ("and", "تمام", "end") from live typing
let triggerWords = text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastRaw = triggerWords.last {
let last = InputBoxManager.stripPunctuation(lastRaw).lowercased()
if last == "and" || last == "تمام" || last == "end" {
if triggerWords.count <= 1 {
return // Drop completely if it was just the end word
// Suppress trailing end triggers ("and", "تمام", "end") ONLY in hands-free/toggle mode, never in Push-To-Talk
if HotkeyManager.shared.currentMode != .pushToTalk {
let triggerWords = text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastRaw = triggerWords.last {
let last = InputBoxManager.stripPunctuation(lastRaw).lowercased()
if last == "and" || last == "تمام" || last == "end" {
if triggerWords.count <= 1 {
return
}
let textWithoutEnd = triggerWords.dropLast().joined(separator: " ") + " "
streamDelta(text: textWithoutEnd)
return
} }
let textWithoutEnd = triggerWords.dropLast().joined(separator: " ") + " "
streamDelta(text: textWithoutEnd)
return
} }
} }
@ -224,6 +226,7 @@ public final class InputBoxManager {
// This guarantees that numbers (۱ -> 1), ellipses (... -> . . .), and punctuation tokens have identical word counts & tokens! // This guarantees that numbers (۱ -> 1), ellipses (... -> . . .), and punctuation tokens have identical word counts & tokens!
let processedStreamed = rawStreamed.isEmpty ? "" : TextProcessor.shared.process(text: rawStreamed) let processedStreamed = rawStreamed.isEmpty ? "" : TextProcessor.shared.process(text: rawStreamed)
let cleanStreamed = processedStreamed.trimmingCharacters(in: .whitespacesAndNewlines) let cleanStreamed = processedStreamed.trimmingCharacters(in: .whitespacesAndNewlines)
let streamedEndedWithSpace = rawStreamed.unicodeScalars.last.map { CharacterSet.whitespacesAndNewlines.contains($0) } ?? false
pasteQueue.async { [weak self] in pasteQueue.async { [weak self] in
guard let self = self else { return } guard let self = self else { return }
@ -253,17 +256,14 @@ public final class InputBoxManager {
// Case 1: Nothing was streamed live -> Paste entire text // Case 1: Nothing was streamed live -> Paste entire text
if cleanStreamed.isEmpty { if cleanStreamed.isEmpty {
let pb = NSPasteboard.general
pb.clearContents()
pb.setString(textWithTrailing, forType: .string)
self.postCleanCmdV()
self.pasteSynchronously(textWithTrailing)
restoreClipboardFullText() restoreClipboardFullText()
return return
} }
// Case 2: Exactly matches streamed text // Case 2: Exactly matches streamed text
if cleanStreamed == trimmed { if cleanStreamed == trimmed {
if addTrailingSpace {
if addTrailingSpace && !streamedEndedWithSpace {
self.pasteSynchronously(" ") self.pasteSynchronously(" ")
} }
restoreClipboardFullText() restoreClipboardFullText()
@ -272,9 +272,15 @@ public final class InputBoxManager {
// Case 3: Streamed prefix is exact prefix of trimmed final text // Case 3: Streamed prefix is exact prefix of trimmed final text
if trimmed.count > cleanStreamed.count && trimmed.hasPrefix(cleanStreamed) { if trimmed.count > cleanStreamed.count && trimmed.hasPrefix(cleanStreamed) {
let suffix = String(trimmed.dropFirst(cleanStreamed.count))
let suffixToPaste = addTrailingSpace ? (suffix + " ") : suffix
self.pasteSynchronously(suffixToPaste)
let rawSuffix = String(trimmed.dropFirst(cleanStreamed.count))
let cleanSuffix = rawSuffix.trimmingCharacters(in: .whitespacesAndNewlines)
if !cleanSuffix.isEmpty {
let needsLeadingSpace = !streamedEndedWithSpace && (rawSuffix.first?.isWhitespace == true)
let suffixToPaste = (needsLeadingSpace ? " " : "") + cleanSuffix + (addTrailingSpace ? " " : "")
self.pasteSynchronously(suffixToPaste)
} else if addTrailingSpace && !streamedEndedWithSpace {
self.pasteSynchronously(" ")
}
restoreClipboardFullText() restoreClipboardFullText()
return return
} }
@ -374,12 +380,14 @@ public final class InputBoxManager {
if startIndex < finalPairs.count { if startIndex < finalPairs.count {
let remainingWords = finalPairs.dropFirst(startIndex).map { $0.raw } let remainingWords = finalPairs.dropFirst(startIndex).map { $0.raw }
let suffix = remainingWords.joined(separator: " ") let suffix = remainingWords.joined(separator: " ")
let suffixToPaste = addTrailingSpace ? (" " + suffix + " ") : (" " + suffix)
let leadingSpace = streamedEndedWithSpace ? "" : " "
let trailingSpace = addTrailingSpace ? " " : ""
let suffixToPaste = leadingSpace + suffix + trailingSpace
self.pasteSynchronously(suffixToPaste) self.pasteSynchronously(suffixToPaste)
restoreClipboardFullText() restoreClipboardFullText()
return return
} else { } else {
if addTrailingSpace {
if addTrailingSpace && !streamedEndedWithSpace {
self.pasteSynchronously(" ") self.pasteSynchronously(" ")
} }
restoreClipboardFullText() restoreClipboardFullText()
@ -390,13 +398,39 @@ public final class InputBoxManager {
} }
} }
/// Places text on NSPasteboard, posts Cmd+V, and waits 22ms so target app consumes this chunk BEFORE next clipboard write!
/// Places core word(s) on NSPasteboard via Cmd+V, while injecting leading/trailing spaces
/// via native Space keystrokes so apps that `.trim()` pasted clipboard text (like Hermes Desktop)
/// never glue words together or double-space!
private func pasteSynchronously(_ text: String) { private func pasteSynchronously(_ text: String) {
guard !text.isEmpty else { return }
let horizontalSpaceSet = CharacterSet(charactersIn: " \t")
let hasLeadingSpace = text.unicodeScalars.first.map { horizontalSpaceSet.contains($0) } ?? false
let hasTrailingSpace = text.unicodeScalars.last.map { horizontalSpaceSet.contains($0) } ?? false
let core = text.trimmingCharacters(in: horizontalSpaceSet)
if core.isEmpty {
if hasLeadingSpace || hasTrailingSpace {
postCleanSpace()
usleep(12000)
}
return
}
if hasLeadingSpace {
postCleanSpace()
usleep(12000)
}
let pb = NSPasteboard.general let pb = NSPasteboard.general
pb.clearContents() pb.clearContents()
pb.setString(text, forType: .string)
pb.setString(core, forType: .string)
postCleanCmdV() postCleanCmdV()
usleep(22000) // 22ms guard ensures browser/app reads NSPasteboard before next chunk overwrites it (~45 chunks/sec capacity) usleep(22000) // 22ms guard ensures browser/app reads NSPasteboard before next chunk overwrites it (~45 chunks/sec capacity)
if hasTrailingSpace {
postCleanSpace()
usleep(12000)
}
} }
private func postCleanCmdV() { private func postCleanCmdV() {
@ -411,4 +445,20 @@ public final class InputBoxManager {
up.post(tap: .cgAnnotatedSessionEventTap) up.post(tap: .cgAnnotatedSessionEventTap)
} }
} }
private func postCleanSpace() {
let src = CGEventSource(stateID: .privateState)
let spaceKeyCode: CGKeyCode = 0x31 // ' ' (kVK_Space)
var spaceChar: UniChar = 0x0020
if let down = CGEvent(keyboardEventSource: src, virtualKey: spaceKeyCode, keyDown: true),
let up = CGEvent(keyboardEventSource: src, virtualKey: spaceKeyCode, keyDown: false) {
down.flags = []
up.flags = []
down.keyboardSetUnicodeString(stringLength: 1, unicodeString: &spaceChar)
up.keyboardSetUnicodeString(stringLength: 1, unicodeString: &spaceChar)
down.post(tap: .cgAnnotatedSessionEventTap)
up.post(tap: .cgAnnotatedSessionEventTap)
}
}
} }

512
src/SonioxClient.swift

@ -35,10 +35,16 @@ public final class SonioxLiveSession {
private var isAborted = false private var isAborted = false
private let lock = NSLock() private let lock = NSLock()
// Transcripts & Stall tracking
// Transcripts & Stall tracking (Multi-segment continuous speech + checkpointed audio buffer)
private var archivedCommittedText: String = ""
private var committedFinalTokens: [String] = [] private var committedFinalTokens: [String] = []
private var currentNonFinalTokens: [String] = [] private var currentNonFinalTokens: [String] = []
private var streamedPrefix: String = "" private var streamedPrefix: String = ""
private var currentSegmentAudioBuffer = Data()
private var lastConfirmedWordAudioByteOffset: Int = 0
private var isReconnectingMidRecording = false
private var isFlushingReplayBuffer = false
private var segmentConnectTime: Date = Date()
private var sentAudioBytes: Int = 0 private var sentAudioBytes: Int = 0
public private(set) var totalReceivedTokensCount: Int = 0 public private(set) var totalReceivedTokensCount: Int = 0
public private(set) var hasSilentStall: Bool = false public private(set) var hasSilentStall: Bool = false
@ -61,10 +67,46 @@ public final class SonioxLiveSession {
isRecordingActive = active isRecordingActive = active
if active { if active {
lastDataTime = Date() lastDataTime = Date()
segmentConnectTime = Date()
} }
lock.unlock() lock.unlock()
} }
/// True when a mid-recording disconnect/stall occurred and there is >0.35s of unconfirmed tail audio waiting to be transcribed.
public var hasUntranscribedTailAudio: Bool {
lock.lock()
defer { lock.unlock() }
let tailCount = max(0, currentSegmentAudioBuffer.count - lastConfirmedWordAudioByteOffset)
let hadDrop = hasNetworkFailure || hasSilentStall || isReconnectingMidRecording || !isConnected
return hadDrop && tailCount > 11200
}
/// Returns the already-confirmed transcript prefix across all completed segments and finalized tokens.
public func confirmedTranscriptPrefix() -> String {
lock.lock()
let segCommitted = committedFinalTokens.joined().trimmingCharacters(in: .whitespacesAndNewlines)
let rawPrefix: String
if archivedCommittedText.isEmpty {
rawPrefix = segCommitted
} else if segCommitted.isEmpty {
rawPrefix = archivedCommittedText
} else {
rawPrefix = archivedCommittedText + " " + segCommitted
}
lock.unlock()
return SonioxLiveSession.sanitizeText(rawPrefix)
}
/// Returns the slice of raw PCM audio from `lastConfirmedWordAudioByteOffset` to the end of `currentSegmentAudioBuffer`.
public func untranscribedTailAudioData() -> Data {
lock.lock()
defer { lock.unlock() }
guard currentSegmentAudioBuffer.count > lastConfirmedWordAudioByteOffset else {
return Data()
}
return currentSegmentAudioBuffer.subdata(in: lastConfirmedWordAudioByteOffset..<currentSegmentAudioBuffer.count)
}
public var isReady: Bool { public var isReady: Bool {
lock.lock() lock.lock()
defer { lock.unlock() } defer { lock.unlock() }
@ -256,22 +298,197 @@ public final class SonioxLiveSession {
startReceiving() startReceiving()
} }
/// Called while user is actively speaking to detect half-open TCP silent stalls
/// Called while user is actively speaking to detect half-open TCP silent stalls and auto-reconnect seamlessly
public func checkSilentStallWhileSpeaking() { public func checkSilentStallWhileSpeaking() {
lock.lock() lock.lock()
let connected = isConnected && !isClosed && !isAborted let connected = isConnected && !isClosed && !isAborted
let activeRec = isRecordingActive && !isFinalizing && !isReconnectingMidRecording
let sent = sentAudioBytes let sent = sentAudioBytes
let elapsedSinceData = Date().timeIntervalSince(lastDataTime) let elapsedSinceData = Date().timeIntervalSince(lastDataTime)
lock.unlock() lock.unlock()
// If >1.5s of audio (48KB) was sent while speaking, but 0 frames arrived for >3.0s:
if connected && sent > 48000 && elapsedSinceData > 3.0 {
// If >1.0s of audio (32KB) was sent while speaking, but 0 frames arrived for >3.5s:
if connected && sent > 32000 && elapsedSinceData > 3.5 {
lock.lock() lock.lock()
hasSilentStall = true hasSilentStall = true
lock.unlock() lock.unlock()
DispatchQueue.main.async { [weak self] in DispatchQueue.main.async { [weak self] in
self?.onNetworkDegraded?(true) self?.onNetworkDegraded?(true)
} }
if activeRec {
triggerMidRecordingReconnect(cleanArchive: false, reason: "Silent TCP/STT stall (\(String(format: "%.1f", elapsedSinceData))s without frames while speaking)")
}
}
}
/// Seamlessly archives confirmed transcript tokens and reconnects a fresh WebSocket in the background
/// while the user continues holding Push-To-Talk / speaking for 5–10+ minutes!
public func triggerMidRecordingReconnect(cleanArchive: Bool, reason: String) {
lock.lock()
guard isRecordingActive, !isFinalizing, !isClosed, !isAborted, !isReconnectingMidRecording else {
lock.unlock()
return
}
isReconnectingMidRecording = true
isConnected = false
let segCommitted = committedFinalTokens.joined()
let segNonFinal = currentNonFinalTokens.joined()
let toArchive = cleanArchive ? (segCommitted + segNonFinal) : segCommitted
let cleanToArchive = toArchive.trimmingCharacters(in: .whitespacesAndNewlines)
if !cleanToArchive.isEmpty {
if archivedCommittedText.isEmpty {
archivedCommittedText = cleanToArchive
} else {
archivedCommittedText += " " + cleanToArchive
}
}
committedFinalTokens.removeAll()
currentNonFinalTokens.removeAll()
if cleanArchive {
currentSegmentAudioBuffer.removeAll(keepingCapacity: true)
lastConfirmedWordAudioByteOffset = 0
} else if lastConfirmedWordAudioByteOffset > 0 && lastConfirmedWordAudioByteOffset <= currentSegmentAudioBuffer.count {
currentSegmentAudioBuffer.removeSubrange(0..<lastConfirmedWordAudioByteOffset)
lastConfirmedWordAudioByteOffset = 0
}
if !archivedCommittedText.isEmpty {
streamedPrefix = archivedCommittedText + " "
}
let bufferedBytes = currentSegmentAudioBuffer.count
let archivedCount = archivedCommittedText.count
let oldTask = webSocketTask
let oldSession = urlSession
webSocketTask = nil
urlSession = nil
lock.unlock()
print("SonioxLiveSession: 🔄 Mid-recording auto-reconnect triggered (\(reason)). Archived \(archivedCount) chars, buffering \(bufferedBytes) bytes of unconfirmed audio...")
oldTask?.cancel(with: .goingAway, reason: nil)
oldSession?.invalidateAndCancel()
attemptMidRecordingConnect(attempt: 1)
}
private func attemptMidRecordingConnect(attempt: Int) {
lock.lock()
guard isRecordingActive, !isFinalizing, !isClosed, !isAborted else {
isReconnectingMidRecording = false
lock.unlock()
return
}
let endpoint = (attempt % 2 == 1) ? SonioxLiveSession.primaryWsBase : SonioxLiveSession.fallbackWsBase
let hints = languageHints.map { "language_hints=\($0)" }.joined(separator: "&")
var queryParams: [String] = []
if !hints.isEmpty { queryParams.append(hints) }
queryParams.append("enable_speaker_diarization=false")
queryParams.append("enable_language_identification=true")
queryParams.append("enable_endpoint_detection=false")
queryParams.append("providers=soniox")
if let target = targetLanguage, !target.isEmpty, target != "off" {
queryParams.append("target_language=\(target)")
}
let urlStr = "\(endpoint)?" + queryParams.joined(separator: "&")
guard let url = URL(string: urlStr) else {
isReconnectingMidRecording = false
lock.unlock()
return
}
var request = URLRequest(url: url)
request.setValue(SonioxLiveSession.userAgent, forHTTPHeaderField: "User-Agent")
request.setValue(SonioxLiveSession.origin(for: endpoint), forHTTPHeaderField: "Origin")
request.timeoutInterval = 6.0
let config = URLSessionConfiguration.default
config.waitsForConnectivity = false
config.requestCachePolicy = .reloadIgnoringLocalCacheData
let newSession = URLSession(configuration: config)
let newTask = newSession.webSocketTask(with: request)
self.urlSession = newSession
self.webSocketTask = newTask
lock.unlock()
newTask.resume()
newTask.sendPing { [weak self] err in
guard let self = self else { return }
self.lock.lock()
if err == nil && !self.isClosed && !self.isAborted && !self.isFinalizing {
self.isConnected = true
self.hasNetworkFailure = false
self.hasSilentStall = false
self.isReconnectingMidRecording = false
self.isFlushingReplayBuffer = true
self.lastDataTime = Date()
self.segmentConnectTime = Date()
self.sentAudioBytes = 0
let replaySnapshot = self.currentSegmentAudioBuffer
self.lock.unlock()
self.onNetworkDegraded?(false)
self.onConnectionStateChanged?(true)
print("SonioxLiveSession: ✅ Mid-recording reconnect succeeded on attempt \(attempt) (\(endpoint))! Replaying \(replaySnapshot.count) bytes of buffered speech...")
self.startReceiving()
self.flushReplayBufferToSocket(task: newTask, offset: 0)
} else {
let shouldRetry = self.isRecordingActive && !self.isFinalizing && !self.isClosed && !self.isAborted
self.webSocketTask = nil
self.urlSession = nil
self.lock.unlock()
newTask.cancel()
newSession.invalidateAndCancel()
if shouldRetry {
let delay = min(1.5, 0.5 + Double(min(attempt, 4)) * 0.25)
DispatchQueue.global(qos: .userInitiated).asyncAfter(deadline: .now() + delay) { [weak self] in
self?.attemptMidRecordingConnect(attempt: attempt + 1)
}
}
}
}
}
private func flushReplayBufferToSocket(task: URLSessionWebSocketTask, offset: Int) {
lock.lock()
guard isConnected, !isClosed, !isAborted, !isFinalizing, webSocketTask === task else {
isFlushingReplayBuffer = false
lock.unlock()
return
}
let total = currentSegmentAudioBuffer.count
if offset >= total {
isFlushingReplayBuffer = false
lock.unlock()
return
}
let chunkSize = 32000 // 1.0s chunk
let end = min(offset + chunkSize, total)
let slice = currentSegmentAudioBuffer.subdata(in: offset..<end)
lock.unlock()
task.send(.data(slice)) { [weak self] err in
guard let self = self else { return }
if let err = err {
print("SonioxLiveSession: Error flushing replay buffer:", err)
self.lock.lock()
self.isFlushingReplayBuffer = false
self.hasNetworkFailure = true
self.isConnected = false
let activeRec = self.isRecordingActive && !self.isFinalizing && !self.isClosed && !self.isAborted
self.lock.unlock()
if activeRec {
self.triggerMidRecordingReconnect(cleanArchive: false, reason: "Replay flush send error")
}
return
}
self.lock.lock()
self.sentAudioBytes += slice.count
self.lock.unlock()
self.flushReplayBufferToSocket(task: task, offset: end)
} }
} }
@ -281,15 +498,40 @@ public final class SonioxLiveSession {
lock.unlock() lock.unlock()
return return
} }
if !isConnected && initialAudioBuffer.count < 80 {
initialAudioBuffer.append(data)
currentSegmentAudioBuffer.append(data)
if !isConnected {
if initialAudioBuffer.count < 120 {
initialAudioBuffer.append(data)
}
let shouldReconnect = isRecordingActive && hasNetworkFailure && !isReconnectingMidRecording
lock.unlock()
if shouldReconnect {
triggerMidRecordingReconnect(cleanArchive: false, reason: "Audio chunk arrived while disconnected")
}
return
} }
if isReconnectingMidRecording || isFlushingReplayBuffer {
lock.unlock()
return
}
guard let task = webSocketTask else { guard let task = webSocketTask else {
lock.unlock() lock.unlock()
return return
} }
// Proactive segment rotation at >150s when at a clean committed pause boundary so server never kills long 5–10m streams
let segAge = Date().timeIntervalSince(segmentConnectTime)
let canRotateProactively = segAge > 150.0 && currentNonFinalTokens.isEmpty && !committedFinalTokens.isEmpty
lock.unlock() lock.unlock()
if canRotateProactively {
triggerMidRecordingReconnect(cleanArchive: true, reason: "Proactive 150s continuous speech segment rollover")
return
}
let t0 = Date() let t0 = Date()
let message = URLSessionWebSocketTask.Message.data(data) let message = URLSessionWebSocketTask.Message.data(data)
task.send(message) { [weak self] error in task.send(message) { [weak self] error in
@ -297,15 +539,19 @@ public final class SonioxLiveSession {
if let error = error { if let error = error {
print("SonioxLiveSession: Error streaming audio chunk:", error) print("SonioxLiveSession: Error streaming audio chunk:", error)
self.lock.lock() self.lock.lock()
if !self.isFallbackSwitched && !self.isFinalizing && !self.isClosed && !self.isAborted {
if !self.isFallbackSwitched && !self.isFinalizing && !self.isClosed && !self.isAborted && self.archivedCommittedText.isEmpty && self.committedFinalTokens.isEmpty {
self.lock.unlock() self.lock.unlock()
self.switchToFallback() self.switchToFallback()
return return
} }
self.hasNetworkFailure = true self.hasNetworkFailure = true
self.isConnected = false self.isConnected = false
let activeRec = self.isRecordingActive && !self.isFinalizing && !self.isClosed && !self.isAborted
self.lock.unlock() self.lock.unlock()
self.onNetworkDegraded?(true) self.onNetworkDegraded?(true)
if activeRec {
self.triggerMidRecordingReconnect(cleanArchive: false, reason: "Send audio chunk error: \(error.localizedDescription)")
}
} else { } else {
self.lock.lock() self.lock.lock()
self.sentAudioBytes += data.count self.sentAudioBytes += data.count
@ -320,13 +566,21 @@ public final class SonioxLiveSession {
public func finalizeStream() { public func finalizeStream() {
lock.lock() lock.lock()
guard !isFinalizing, !isClosed, !isAborted, let task = webSocketTask else {
guard !isFinalizing, !isClosed, !isAborted else {
lock.unlock() lock.unlock()
return return
} }
isFinalizing = true isFinalizing = true
let connected = isConnected && !isReconnectingMidRecording
let task = webSocketTask
lock.unlock() lock.unlock()
guard connected, let task = task else {
print("SonioxLiveSession: finalizeStream called while socket disconnected/reconnecting — completing immediately for tail recovery.")
completeWithCurrentText()
return
}
print("SonioxLiveSession: Sending finalize packet...") print("SonioxLiveSession: Sending finalize packet...")
// 1. Send trailing 300ms (9600 bytes at 16kHz 16-bit PCM) silence frame // 1. Send trailing 300ms (9600 bytes at 16kHz 16-bit PCM) silence frame
// so acoustic CTC model cleanly flushes and resolves trailing fast speech syllables // so acoustic CTC model cleanly flushes and resolves trailing fast speech syllables
@ -403,14 +657,15 @@ public final class SonioxLiveSession {
self.lock.lock() self.lock.lock()
self.hasNetworkFailure = true self.hasNetworkFailure = true
self.isConnected = false self.isConnected = false
let activeRecording = self.isRecordingActive
let activeRecording = self.isRecordingActive && !self.isFinalizing && !self.isClosed && !self.isAborted
self.lock.unlock() self.lock.unlock()
self.onNetworkDegraded?(true) self.onNetworkDegraded?(true)
if !activeRecording { if !activeRecording {
self.completeWithCurrentText() self.completeWithCurrentText()
} else { } else {
print("SonioxLiveSession: Network dropped mid-recording. Audio is buffered for Smart Recovery upon key release.")
print("SonioxLiveSession: Network dropped mid-recording. Triggering automatic live reconnect...")
self.triggerMidRecordingReconnect(cleanArchive: false, reason: "WebSocket receive error: \(error.localizedDescription)")
} }
} }
} }
@ -465,21 +720,40 @@ public final class SonioxLiveSession {
totalReceivedTokensCount += tokenCountDelta totalReceivedTokensCount += tokenCountDelta
if !newFinals.isEmpty { if !newFinals.isEmpty {
committedFinalTokens.append(contentsOf: newFinals) committedFinalTokens.append(contentsOf: newFinals)
// Advance confirmed audio checkpoint in `currentSegmentAudioBuffer`:
// Keep a safety acoustic lookback proportional to unfinalized words (plus 0.8s = 25600 bytes)
let unfinalizedWords = newNonFinals.joined().components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }.count
let safetyLookbackBytes = max(25600, (unfinalizedWords + 1) * 16000)
if currentSegmentAudioBuffer.count > safetyLookbackBytes {
lastConfirmedWordAudioByteOffset = max(lastConfirmedWordAudioByteOffset, currentSegmentAudioBuffer.count - safetyLookbackBytes)
}
} }
currentNonFinalTokens = newNonFinals currentNonFinalTokens = newNonFinals
let fullCommitted = committedFinalTokens.joined() let fullCommitted = committedFinalTokens.joined()
let fullNonFinal = currentNonFinalTokens.joined() let fullNonFinal = currentNonFinalTokens.joined()
let combined = fullCommitted + fullNonFinal
let segmentCombined = fullCommitted + fullNonFinal
let combined: String
if archivedCommittedText.isEmpty {
combined = segmentCombined
} else if segmentCombined.isEmpty {
combined = archivedCommittedText
} else {
let cleanSeg = segmentCombined.trimmingCharacters(in: .whitespaces)
combined = archivedCommittedText + (cleanSeg.isEmpty ? "" : (" " + cleanSeg))
}
// Check for in-flight end trigger ("and", "تمام", "end") in combined text
// Check for in-flight end trigger ("and", "تمام", "end") ONLY in hands-free/toggle mode and ONLY on finalized tokens!
// Never trigger auto-stop during Push-To-Talk while user is physically holding the hotkey!
var triggerDetected = false var triggerDetected = false
let trimmedCombined = combined.trimmingCharacters(in: .whitespacesAndNewlines)
let currentWords = trimmedCombined.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastRaw = currentWords.last {
let cleanLast = InputBoxManager.stripPunctuation(lastRaw).lowercased()
if cleanLast == "and" || cleanLast == "تمام" || cleanLast == "end" {
triggerDetected = true
if HotkeyManager.shared.currentMode != .pushToTalk && currentNonFinalTokens.isEmpty && !newFinals.isEmpty {
let trimmedCombined = combined.trimmingCharacters(in: .whitespacesAndNewlines)
let currentWords = trimmedCombined.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastRaw = currentWords.last {
let cleanLast = InputBoxManager.stripPunctuation(lastRaw).lowercased()
if cleanLast == "and" || cleanLast == "تمام" || cleanLast == "end" {
triggerDetected = true
}
} }
} }
@ -492,27 +766,16 @@ public final class SonioxLiveSession {
return raw.contains("\"finished\":true") || raw.contains("\"finished\": true") return raw.contains("\"finished\":true") || raw.contains("\"finished\": true")
}() }()
let isTerminatingMessage = gotFin || sessionEnded || sessionDone || isFinished || hasRawFinished let isTerminatingMessage = gotFin || sessionEnded || sessionDone || isFinished || hasRawFinished
let stillActivelyRecording = self.isRecordingActive && !self.isFinalizing
// Emit live streaming deltas in real time as the user speaks! // Emit live streaming deltas in real time as the user speaks!
// Soniox batches `is_final: true` tokens only every 6-8 seconds, which would make live typing feel lagging and out of sync.
// However, completed words in `combined` (with a 1-word acoustic lookahead holdback) are 100% stable and appear within ~150ms!
// We stream `fullCommitted` PLUS any completed words in `fullNonFinal` except the trailing word currently being uttered.
// We stream completed words in `combined` with a 1-word acoustic lookahead holdback so every emitted chunk ends with a trailing space.
var deltaToEmit: String? = nil var deltaToEmit: String? = nil
if !triggerDetected && !isTerminatingMessage && !self.isFinalizing { if !triggerDetected && !isTerminatingMessage && !self.isFinalizing {
var stableCandidate = fullCommitted
if let lastSpaceRange = fullNonFinal.rangeOfCharacter(from: .whitespacesAndNewlines, options: .backwards) {
let completedNonFinal = String(fullNonFinal[..<lastSpaceRange.upperBound])
let trailingToken = String(fullNonFinal[lastSpaceRange.upperBound...]).trimmingCharacters(in: .whitespacesAndNewlines)
if !trailingToken.isEmpty {
// At least one subsequent word has started, so `completedNonFinal` has full acoustic lookahead
stableCandidate = fullCommitted + completedNonFinal
} else {
// Trailing space at the very end of `fullNonFinal`: hold back the last word until the next word begins
let trimmedNF = completedNonFinal.trimmingCharacters(in: .whitespacesAndNewlines)
if let prevSpaceRange = trimmedNF.rangeOfCharacter(from: .whitespacesAndNewlines, options: .backwards) {
stableCandidate = fullCommitted + String(trimmedNF[..<prevSpaceRange.upperBound])
}
}
let rTrimmedCombined = combined.replacingOccurrences(of: #"\s+$"#, with: "", options: .regularExpression)
var stableCandidate = ""
if let lastSpaceRange = rTrimmedCombined.rangeOfCharacter(from: .whitespacesAndNewlines, options: .backwards) {
stableCandidate = String(rTrimmedCombined[..<lastSpaceRange.upperBound])
} }
if stableCandidate.count > streamedPrefix.count { if stableCandidate.count > streamedPrefix.count {
@ -521,9 +784,10 @@ public final class SonioxLiveSession {
streamedPrefix = stableCandidate streamedPrefix = stableCandidate
deltaToEmit = delta deltaToEmit = delta
} else { } else {
let common = stableCandidate.commonPrefix(with: streamedPrefix)
if stableCandidate.count > common.count {
let delta = String(stableCandidate.dropFirst(common.count))
let streamedWordsCount = streamedPrefix.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }.count
let candidateWords = stableCandidate.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if candidateWords.count > streamedWordsCount {
let delta = candidateWords.dropFirst(streamedWordsCount).joined(separator: " ") + " "
streamedPrefix = stableCandidate streamedPrefix = stableCandidate
deltaToEmit = delta deltaToEmit = delta
} }
@ -549,7 +813,12 @@ public final class SonioxLiveSession {
} }
if isTerminatingMessage { if isTerminatingMessage {
completeWithCurrentText()
if stillActivelyRecording {
print("SonioxLiveSession: Server ended WebSocket segment while user is still speaking — seamlessly rotating to fresh segment...")
triggerMidRecordingReconnect(cleanArchive: true, reason: "Server ended WebSocket segment while recording is still active")
} else {
completeWithCurrentText()
}
} }
} else { } else {
let sessionEnded = json["session_ended"] as? Bool ?? false let sessionEnded = json["session_ended"] as? Bool ?? false
@ -560,7 +829,14 @@ public final class SonioxLiveSession {
return raw.contains("\"finished\":true") || raw.contains("\"finished\": true") return raw.contains("\"finished\":true") || raw.contains("\"finished\": true")
}() }()
if sessionEnded || sessionDone || isFinished || hasRawFinished { if sessionEnded || sessionDone || isFinished || hasRawFinished {
completeWithCurrentText()
lock.lock()
let stillActivelyRecording = self.isRecordingActive && !self.isFinalizing
lock.unlock()
if stillActivelyRecording {
triggerMidRecordingReconnect(cleanArchive: true, reason: "Server session_done while recording is still active")
} else {
completeWithCurrentText()
}
} }
} }
} }
@ -569,7 +845,15 @@ public final class SonioxLiveSession {
lock.lock() lock.lock()
let fullCommitted = committedFinalTokens.joined() let fullCommitted = committedFinalTokens.joined()
let fullNonFinal = currentNonFinalTokens.joined() let fullNonFinal = currentNonFinalTokens.joined()
let rawCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal)
let segCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal)
let rawCombined: String
if archivedCommittedText.isEmpty {
rawCombined = segCombined
} else if segCombined.isEmpty {
rawCombined = archivedCommittedText
} else {
rawCombined = archivedCommittedText + " " + segCombined.trimmingCharacters(in: .whitespaces)
}
lock.unlock() lock.unlock()
return SonioxLiveSession.sanitizeText(rawCombined) return SonioxLiveSession.sanitizeText(rawCombined)
} }
@ -580,10 +864,23 @@ public final class SonioxLiveSession {
lock.unlock() lock.unlock()
return return
} }
let tailCount = max(0, currentSegmentAudioBuffer.count - lastConfirmedWordAudioByteOffset)
let hadDrop = hasNetworkFailure || hasSilentStall || isReconnectingMidRecording || !isConnected
let needsTailRecovery = hadDrop && tailCount > 11200
isClosed = true isClosed = true
isReconnectingMidRecording = false
let fullCommitted = committedFinalTokens.joined() let fullCommitted = committedFinalTokens.joined()
let fullNonFinal = currentNonFinalTokens.joined() let fullNonFinal = currentNonFinalTokens.joined()
let rawCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal)
let segCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal)
let rawCombined: String
if archivedCommittedText.isEmpty {
rawCombined = segCombined
} else if segCombined.isEmpty {
rawCombined = archivedCommittedText
} else {
rawCombined = archivedCommittedText + " " + segCombined.trimmingCharacters(in: .whitespaces)
}
let cleaned = SonioxLiveSession.sanitizeText(rawCombined) let cleaned = SonioxLiveSession.sanitizeText(rawCombined)
let failureOccurred = hasNetworkFailure let failureOccurred = hasNetworkFailure
@ -597,7 +894,9 @@ public final class SonioxLiveSession {
stopKeepAliveHeartbeat() stopKeepAliveHeartbeat()
DispatchQueue.main.async { DispatchQueue.main.async {
if cleaned.isEmpty && failureOccurred {
if needsTailRecovery {
cb?(.failure(NSError(domain: "SonioxLiveSession", code: -103, userInfo: [NSLocalizedDescriptionKey: "Partial transcript with \(tailCount) bytes of untranscribed tail audio"])))
} else if cleaned.isEmpty && failureOccurred {
cb?(.failure(NSError(domain: "SonioxLiveSession", code: -101, userInfo: [NSLocalizedDescriptionKey: "WebSocket network failure with empty transcript"]))) cb?(.failure(NSError(domain: "SonioxLiveSession", code: -101, userInfo: [NSLocalizedDescriptionKey: "WebSocket network failure with empty transcript"])))
} else { } else {
cb?(.success(cleaned)) cb?(.success(cleaned))
@ -613,7 +912,10 @@ public final class SonioxLiveSession {
isConnected = false isConnected = false
isRecordingActive = false isRecordingActive = false
isFinalizing = false isFinalizing = false
isReconnectingMidRecording = false
isFlushingReplayBuffer = false
initialAudioBuffer.removeAll() initialAudioBuffer.removeAll()
currentSegmentAudioBuffer.removeAll()
onPartialText = nil onPartialText = nil
onCommittedDelta = nil onCommittedDelta = nil
onEndTriggerDetected = nil onEndTriggerDetected = nil
@ -656,6 +958,128 @@ public final class SonioxLiveSession {
let durationSec = Double(pcmData.count) / 32000.0 let durationSec = Double(pcmData.count) / 32000.0
print("SonioxLiveSession: Initiating Smart Burst Recovery: \(pcmData.count) bytes (\(String(format: "%.1f", durationSec))s audio, target: \(effectiveTarget ?? "orig"))...") print("SonioxLiveSession: Initiating Smart Burst Recovery: \(pcmData.count) bytes (\(String(format: "%.1f", durationSec))s audio, target: \(effectiveTarget ?? "orig"))...")
// For multi-minute recordings (>60s = 1,920,000 bytes), split into ~60s silence-aligned chunks
// so individual WebSocket burst connections never hit server buffer limits or timeouts!
let maxSingleBurstBytes = 1_920_000
if pcmData.count > maxSingleBurstBytes {
let chunks = splitIntoSilenceAlignedChunks(pcmData: pcmData, targetChunkBytes: maxSingleBurstBytes)
print("SonioxLiveSession: Splitting long recovery audio (\(String(format: "%.1f", durationSec))s) into \(chunks.count) sequential chunks...")
var recoveredParts: [String] = []
func processChunk(at index: Int) {
if isCancelled() {
completion(.failure(NSError(domain: "SonioxLiveSession", code: -999, userInfo: [NSLocalizedDescriptionKey: "Cancelled by Escape"])))
return
}
if index >= chunks.count {
let combinedText = recoveredParts.joined(separator: " ").trimmingCharacters(in: .whitespacesAndNewlines)
if !combinedText.isEmpty {
completion(.success(combinedText))
} else {
completion(.failure(NSError(domain: "SonioxLiveSession", code: -104, userInfo: [NSLocalizedDescriptionKey: "Empty chunked recovery result"])))
}
return
}
DispatchQueue.main.async {
onStatusUpdate?("PART \(index + 1)/\(chunks.count)")
}
retrySingleChunk(
pcmData: chunks[index],
languageHints: languageHints,
effectiveTarget: effectiveTarget,
isCancelled: isCancelled,
onProgress: onProgress,
onStatusUpdate: onStatusUpdate
) { result in
switch result {
case .success(let partText):
let cleanPart = partText.trimmingCharacters(in: .whitespacesAndNewlines)
if !cleanPart.isEmpty {
recoveredParts.append(cleanPart)
}
processChunk(at: index + 1)
case .failure(let err):
if !recoveredParts.isEmpty {
// Salvage already-recovered chunks if a later chunk fails
let partial = recoveredParts.joined(separator: " ").trimmingCharacters(in: .whitespacesAndNewlines)
completion(.success(partial))
} else {
completion(.failure(err))
}
}
}
}
processChunk(at: 0)
return
}
retrySingleChunk(
pcmData: pcmData,
languageHints: languageHints,
effectiveTarget: effectiveTarget,
isCancelled: isCancelled,
onProgress: onProgress,
onStatusUpdate: onStatusUpdate,
completion: completion
)
}
/// Splits a multi-minute 16kHz 16-bit mono PCM buffer at low-energy silence boundaries near `targetChunkBytes`.
private static func splitIntoSilenceAlignedChunks(pcmData: Data, targetChunkBytes: Int) -> [Data] {
var chunks: [Data] = []
var offset = 0
let total = pcmData.count
let searchWindowBytes = 160_000 // Search within last 5.0s of each 60s window for lowest energy pause
let windowStepBytes = 3200 // 100ms window
while offset < total {
let remaining = total - offset
if remaining <= targetChunkBytes + 160_000 {
chunks.append(pcmData.subdata(in: offset..<total))
break
}
var bestCut = offset + targetChunkBytes
let searchStart = max(offset + targetChunkBytes - searchWindowBytes, offset + 64000)
let searchEnd = min(offset + targetChunkBytes + 32000, total - windowStepBytes)
if searchEnd > searchStart {
var minEnergy: Float = .greatestFiniteMagnitude
pcmData.withUnsafeBytes { rawBuf in
guard let basePtr = rawBuf.baseAddress else { return }
var pos = searchStart & ~1 // align to 2-byte Int16 sample
while pos < searchEnd {
let sampleCount = windowStepBytes / 2
let ptr = basePtr.advanced(by: pos).assumingMemoryBound(to: Int16.self)
var sumSq: Float = 0
for i in 0..<sampleCount {
let s = Float(ptr[i])
sumSq += s * s
}
if sumSq < minEnergy {
minEnergy = sumSq
bestCut = pos + (windowStepBytes / 2)
}
pos += windowStepBytes
}
}
}
bestCut = bestCut & ~1
chunks.append(pcmData.subdata(in: offset..<bestCut))
offset = bestCut
}
return chunks
}
private static func retrySingleChunk(
pcmData: Data,
languageHints: [String],
effectiveTarget: String?,
isCancelled: @escaping () -> Bool,
onProgress: ((String) -> Void)?,
onStatusUpdate: ((String) -> Void)?,
completion: @escaping (Result<String, Error>) -> Void
) {
// Multi-stage resilient retry pipeline across dual endpoints with automatic backoff: // Multi-stage resilient retry pipeline across dual endpoints with automatic backoff:
// Attempt 1: Primary WS // Attempt 1: Primary WS
// Attempt 2: Fallback WS // Attempt 2: Fallback WS
@ -761,7 +1185,7 @@ public final class SonioxLiveSession {
request.setValue(origin(for: endpoint), forHTTPHeaderField: "Origin") request.setValue(origin(for: endpoint), forHTTPHeaderField: "Origin")
let durationSec = Double(pcmData.count) / 32000.0 let durationSec = Double(pcmData.count) / 32000.0
let watchdogTimeout = max(6.5, min(25.0, durationSec * 0.60 + 5.0))
let watchdogTimeout = max(10.0, min(65.0, durationSec * 0.90 + 10.0))
request.timeoutInterval = 5.0 // Fast handshake timeout so dead endpoint fails over in <5s request.timeoutInterval = 5.0 // Fast handshake timeout so dead endpoint fails over in <5s
let config = URLSessionConfiguration.ephemeral let config = URLSessionConfiguration.ephemeral

3
src/main.swift

@ -1,5 +1,8 @@
import Cocoa import Cocoa
setbuf(stdout, nil)
setbuf(stderr, nil)
func checkSingleInstance() { func checkSingleInstance() {
let myPid = ProcessInfo.processInfo.processIdentifier let myPid = ProcessInfo.processInfo.processIdentifier

Loading…
Cancel
Save